Skip to content

Commit 1014a63

Browse files
committed
[test](iceberg) Expand capability, transaction and snapshot coverage
### What problem does this PR solve? Problem Summary: Add connector semantic tests, external regression cases and an operation coverage inventory. Preserve correct, individually disabled reproductions for known issues without changing production behavior. ### Release note None ### Check List (For Author) - Test: Unit Test; full connector reactor and FE Checkstyle passed. Groovy compilation passed; SQL execution pending managed environment. - Behavior changed: No - Does this need documentation: No
1 parent 2524893 commit 1014a63

12 files changed

Lines changed: 1965 additions & 0 deletions
Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.connector.iceberg;
19+
20+
import org.apache.doris.connector.iceberg.IcebergCatalogOps.CatalogBackedIcebergCatalogOps;
21+
import org.apache.doris.connector.spi.ConnectorColumn;
22+
import org.apache.doris.connector.spi.ConnectorType;
23+
import org.apache.doris.connector.spi.DorisConnectorException;
24+
25+
import org.apache.iceberg.HasTableOperations;
26+
import org.apache.iceberg.PartitionSpec;
27+
import org.apache.iceberg.Schema;
28+
import org.apache.iceberg.Table;
29+
import org.apache.iceberg.catalog.Namespace;
30+
import org.apache.iceberg.catalog.TableIdentifier;
31+
import org.apache.iceberg.inmemory.InMemoryCatalog;
32+
import org.apache.iceberg.types.Types;
33+
import org.junit.jupiter.api.Assertions;
34+
import org.junit.jupiter.api.Test;
35+
36+
import java.util.Collections;
37+
38+
public class IcebergLineageDdlCoverageTest {
39+
private static final TableIdentifier IDENTIFIER = TableIdentifier.of("db", "events");
40+
private static final IcebergTableHandle HANDLE = new IcebergTableHandle("db", "events");
41+
42+
@Test
43+
public void addDoesNotExposeDuplicateRowIdColumns() {
44+
assertAddPreservesUniqueMetadataNames("_row_id");
45+
}
46+
47+
@Test
48+
public void addDoesNotExposeDuplicateSequenceColumns() {
49+
assertAddPreservesUniqueMetadataNames("_last_updated_sequence_number");
50+
}
51+
52+
@Test
53+
public void renameDoesNotExposeDuplicateRowIdColumns() {
54+
assertRenamePreservesUniqueMetadataNames("_row_id");
55+
}
56+
57+
@Test
58+
public void renameDoesNotExposeDuplicateSequenceColumns() {
59+
assertRenamePreservesUniqueMetadataNames("_last_updated_sequence_number");
60+
}
61+
62+
private void assertAddPreservesUniqueMetadataNames(String reservedName) {
63+
InMemoryCatalog catalog = catalog();
64+
createV3(catalog);
65+
IcebergConnectorMetadata metadata = metadata(catalog);
66+
String previousMetadata = metadataLocation(catalog);
67+
DorisConnectorException failure = Assertions.assertThrows(DorisConnectorException.class,
68+
() -> metadata.addColumn(null, HANDLE,
69+
new ConnectorColumn(reservedName, ConnectorType.of("BIGINT"), "", true, null, false), null));
70+
Assertions.assertTrue(failure.getMessage().contains("reserved row lineage column"));
71+
Assertions.assertEquals(previousMetadata, metadataLocation(catalog));
72+
Assertions.assertNull(catalog.loadTable(IDENTIFIER).schema().findField(reservedName));
73+
assertUniqueColumnName(metadata, reservedName);
74+
}
75+
76+
private void assertRenamePreservesUniqueMetadataNames(String reservedName) {
77+
InMemoryCatalog catalog = catalog();
78+
createV3(catalog);
79+
IcebergConnectorMetadata metadata = metadata(catalog);
80+
String previousMetadata = metadataLocation(catalog);
81+
DorisConnectorException failure = Assertions.assertThrows(DorisConnectorException.class,
82+
() -> metadata.renameColumn(null, HANDLE, "id", reservedName));
83+
Assertions.assertTrue(failure.getMessage().contains("reserved row lineage column"));
84+
Assertions.assertEquals(previousMetadata, metadataLocation(catalog));
85+
Assertions.assertNotNull(catalog.loadTable(IDENTIFIER).schema().findField("id"));
86+
Assertions.assertNull(catalog.loadTable(IDENTIFIER).schema().findField(reservedName));
87+
assertUniqueColumnName(metadata, reservedName);
88+
}
89+
90+
private static void assertUniqueColumnName(IcebergConnectorMetadata metadata, String reservedName) {
91+
// A public schema must not contain both a physical user field and hidden metadata under one name.
92+
long matchingColumns = metadata.getTableSchema(null, HANDLE).getColumns().stream()
93+
.filter(column -> reservedName.equalsIgnoreCase(column.getName())).count();
94+
Assertions.assertEquals(1L, matchingColumns,
95+
"The connector exposed duplicate physical and hidden columns named " + reservedName);
96+
}
97+
98+
private static String metadataLocation(InMemoryCatalog catalog) {
99+
Table table = catalog.loadTable(IDENTIFIER);
100+
return ((HasTableOperations) table).operations().current().metadataFileLocation();
101+
}
102+
103+
private static InMemoryCatalog catalog() {
104+
InMemoryCatalog catalog = new InMemoryCatalog();
105+
catalog.initialize("repro", Collections.emptyMap());
106+
catalog.createNamespace(Namespace.of("db"));
107+
return catalog;
108+
}
109+
110+
private static void createV3(InMemoryCatalog catalog) {
111+
catalog.createTable(IDENTIFIER, new Schema(
112+
Types.NestedField.optional(1, "id", Types.LongType.get())),
113+
PartitionSpec.unpartitioned(), Collections.singletonMap("format-version", "3"));
114+
}
115+
116+
private static IcebergConnectorMetadata metadata(InMemoryCatalog catalog) {
117+
return new IcebergConnectorMetadata(new CatalogBackedIcebergCatalogOps(catalog),
118+
IcebergCatalogProperties.of(Collections.singletonMap(
119+
IcebergCatalogProperties.ICEBERG_CATALOG_TYPE, IcebergCatalogProperties.TYPE_REST)),
120+
new RecordingConnectorContext());
121+
}
122+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.connector.iceberg;
19+
20+
import org.apache.doris.connector.iceberg.IcebergCatalogOps.CatalogBackedIcebergCatalogOps;
21+
22+
import org.apache.iceberg.PartitionSpec;
23+
import org.apache.iceberg.Schema;
24+
import org.apache.iceberg.catalog.Namespace;
25+
import org.apache.iceberg.catalog.TableIdentifier;
26+
import org.apache.iceberg.inmemory.InMemoryCatalog;
27+
import org.apache.iceberg.types.Types;
28+
import org.junit.jupiter.api.Assertions;
29+
import org.junit.jupiter.api.Disabled;
30+
import org.junit.jupiter.api.Test;
31+
32+
import java.util.Collections;
33+
34+
public class IcebergLineageDdlKnownBugsTest {
35+
private static final TableIdentifier IDENTIFIER = TableIdentifier.of("db", "events");
36+
private static final IcebergTableHandle HANDLE = new IcebergTableHandle("db", "events");
37+
38+
@Test
39+
@Disabled("Known issue: a V2-to-V3 upgrade permits a user column to collide with row lineage.")
40+
public void upgradeDoesNotExposeDuplicateRowIdColumns() {
41+
assertUpgradePreservesUniqueMetadataNames("_row_id");
42+
}
43+
44+
@Test
45+
@Disabled("Known issue: a V2-to-V3 upgrade permits a user column to collide with row lineage.")
46+
public void upgradeDoesNotExposeDuplicateSequenceColumns() {
47+
assertUpgradePreservesUniqueMetadataNames("_last_updated_sequence_number");
48+
}
49+
50+
private void assertUpgradePreservesUniqueMetadataNames(String reservedName) {
51+
InMemoryCatalog catalog = catalog();
52+
catalog.createTable(IDENTIFIER, new Schema(
53+
Types.NestedField.optional(1, reservedName, Types.LongType.get())),
54+
PartitionSpec.unpartitioned(), Collections.singletonMap("format-version", "2"));
55+
IcebergConnectorMetadata metadata = metadata(catalog);
56+
metadata.setTableProperties(null, HANDLE, Collections.singletonMap("format-version", "3"));
57+
assertUniqueColumnName(metadata, reservedName);
58+
}
59+
60+
private static void assertUniqueColumnName(IcebergConnectorMetadata metadata, String reservedName) {
61+
// A public schema must not contain both a physical user field and hidden metadata under one name.
62+
long matchingColumns = metadata.getTableSchema(null, HANDLE).getColumns().stream()
63+
.filter(column -> reservedName.equalsIgnoreCase(column.getName())).count();
64+
Assertions.assertEquals(1L, matchingColumns,
65+
"The connector exposed duplicate physical and hidden columns named " + reservedName);
66+
}
67+
68+
private static InMemoryCatalog catalog() {
69+
InMemoryCatalog catalog = new InMemoryCatalog();
70+
catalog.initialize("repro", Collections.emptyMap());
71+
catalog.createNamespace(Namespace.of("db"));
72+
return catalog;
73+
}
74+
75+
private static IcebergConnectorMetadata metadata(InMemoryCatalog catalog) {
76+
return new IcebergConnectorMetadata(new CatalogBackedIcebergCatalogOps(catalog),
77+
IcebergCatalogProperties.of(Collections.singletonMap(
78+
IcebergCatalogProperties.ICEBERG_CATALOG_TYPE, IcebergCatalogProperties.TYPE_REST)),
79+
new RecordingConnectorContext());
80+
}
81+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.connector.iceberg;
19+
20+
import org.apache.iceberg.PartitionData;
21+
import org.apache.iceberg.PartitionSpec;
22+
import org.apache.iceberg.Schema;
23+
import org.apache.iceberg.transforms.Transform;
24+
import org.apache.iceberg.transforms.Transforms;
25+
import org.apache.iceberg.types.Type;
26+
import org.apache.iceberg.types.Types;
27+
import org.junit.jupiter.api.Assertions;
28+
import org.junit.jupiter.api.Test;
29+
30+
import java.math.BigDecimal;
31+
import java.nio.ByteBuffer;
32+
import java.time.LocalDate;
33+
import java.time.LocalDateTime;
34+
import java.time.ZoneId;
35+
import java.time.ZoneOffset;
36+
import java.util.Arrays;
37+
import java.util.List;
38+
39+
public class IcebergPartitionUtilsCoverageTest {
40+
@Test
41+
public void compoundPartitionWireFormatPreservesEscapingAndExactValues() {
42+
Schema schema = new Schema(
43+
Types.NestedField.optional(1, "text", Types.StringType.get()),
44+
Types.NestedField.optional(2, "nullable", Types.StringType.get()),
45+
Types.NestedField.optional(3, "amount", Types.DecimalType.of(38, 18)),
46+
Types.NestedField.optional(4, "date", Types.DateType.get()),
47+
Types.NestedField.optional(5, "timestamp", Types.TimestampNanoType.withoutZone()),
48+
Types.NestedField.optional(6, "binary", Types.BinaryType.get()));
49+
PartitionSpec spec = PartitionSpec.builderFor(schema).identity("text").identity("nullable")
50+
.identity("amount").identity("date").identity("timestamp").identity("binary").build();
51+
PartitionData original = new PartitionData(spec.partitionType());
52+
List<Object> values = Arrays.asList("null/\"quoted\"\\line\n\u0000雪", null,
53+
new BigDecimal("-99999999999999999999.000000000000000001"), -1,
54+
-1L, ByteBuffer.wrap(new byte[] {0, (byte) 0xff, (byte) 0x80, 34, 92}));
55+
for (int i = 0; i < values.size(); i++) {
56+
original.set(i, values.get(i));
57+
}
58+
59+
for (ZoneId zone : Arrays.asList(ZoneOffset.UTC, ZoneId.of("America/New_York"))) {
60+
List<String> encoded = IcebergPartitionUtils.parsePartitionValuesFromJson(
61+
IcebergPartitionUtils.getPartitionDataJson(original, spec, zone));
62+
Assertions.assertEquals(values.size(), encoded.size());
63+
Assertions.assertEquals(values.get(0), encoded.get(0));
64+
Assertions.assertNull(encoded.get(1));
65+
PartitionData decoded = new PartitionData(spec.partitionType());
66+
for (int i = 0; i < encoded.size(); i++) {
67+
decoded.set(i, IcebergPartitionUtils.parsePartitionValueFromString(
68+
encoded.get(i), spec.partitionType().fields().get(i).type(), zone));
69+
}
70+
// PartitionData stores binary as byte[], whose container equality compares array identities.
71+
for (int i = 0; i < original.size(); i++) {
72+
Assertions.assertEquals(original.get(i), decoded.get(i), "partition field " + i);
73+
}
74+
Assertions.assertEquals(18, ((BigDecimal) decoded.get(2)).scale());
75+
}
76+
}
77+
78+
@Test
79+
public void calendarRangesAgreeWithSdkTransformsAcrossEpochAndLeapBoundaries() {
80+
List<LocalDateTime> timestamps = Arrays.asList(
81+
LocalDateTime.of(1600, 2, 29, 23, 59, 59, 999999000),
82+
LocalDateTime.of(1900, 3, 1, 0, 0),
83+
LocalDateTime.of(1969, 12, 31, 23, 59, 59, 999999000),
84+
LocalDateTime.of(1970, 1, 1, 0, 0),
85+
LocalDateTime.of(2000, 2, 29, 23, 59, 59, 999999000),
86+
LocalDateTime.of(2100, 3, 1, 0, 0));
87+
for (Type type : Arrays.asList(Types.DateType.get(),
88+
Types.TimestampType.withoutZone(), Types.TimestampType.withZone())) {
89+
for (String name : Arrays.asList("year", "month", "day", "hour")) {
90+
if (type.typeId() == Type.TypeID.DATE && name.equals("hour")) {
91+
continue;
92+
}
93+
Transform<Object, Integer> transform = calendarTransform(name);
94+
for (LocalDateTime timestamp : timestamps) {
95+
int ordinal = transform.bind(type).apply(internalValue(type, timestamp));
96+
IcebergPartitionUtils.RangeBuild range = IcebergPartitionUtils.buildRange(
97+
name + "=" + ordinal, Integer.toString(ordinal), name, type, 100L, 200L);
98+
LocalDateTime lower = parseBound(range.getLowerBound().get(0), type);
99+
LocalDateTime upper = parseBound(range.getUpperBound().get(0), type);
100+
LocalDateTime row = type.typeId() == Type.TypeID.DATE
101+
? timestamp.toLocalDate().atStartOfDay() : timestamp;
102+
Assertions.assertFalse(row.isBefore(lower), name + " " + timestamp);
103+
Assertions.assertTrue(row.isBefore(upper), name + " " + timestamp);
104+
// Both ends must describe exactly one SDK transform ordinal, including negative ordinals.
105+
Assertions.assertEquals(ordinal, transform.bind(type).apply(internalValue(type, lower)));
106+
LocalDateTime last = type.typeId() == Type.TypeID.DATE
107+
? upper.minusDays(1) : upper.minusNanos(1000);
108+
Assertions.assertEquals(ordinal, transform.bind(type).apply(internalValue(type, last)));
109+
Assertions.assertEquals(ordinal + 1, transform.bind(type).apply(internalValue(type, upper)));
110+
}
111+
}
112+
}
113+
}
114+
115+
@Test
116+
public void binarySerializationDoesNotConsumeDirectOrReadOnlyBuffers() {
117+
ByteBuffer direct = ByteBuffer.allocateDirect(6);
118+
direct.put(new byte[] {99, 0, (byte) 0xfe, (byte) 0xff, 1, 99});
119+
direct.position(1);
120+
direct.limit(5);
121+
for (ByteBuffer buffer : Arrays.asList(direct, direct.asReadOnlyBuffer(), direct.slice())) {
122+
int position = buffer.position();
123+
int limit = buffer.limit();
124+
for (Type type : Arrays.asList(Types.BinaryType.get(), Types.FixedType.ofLength(4))) {
125+
String encoded = IcebergPartitionUtils.serializePartitionValue(type, buffer, ZoneOffset.UTC);
126+
Assertions.assertEquals("0x00feff01", encoded);
127+
Assertions.assertEquals(buffer, IcebergPartitionUtils.parsePartitionValueFromString(
128+
encoded, type, ZoneOffset.UTC));
129+
Assertions.assertEquals(position, buffer.position());
130+
Assertions.assertEquals(limit, buffer.limit());
131+
}
132+
}
133+
}
134+
135+
private static Transform<Object, Integer> calendarTransform(String name) {
136+
switch (name) {
137+
case "year":
138+
return Transforms.year();
139+
case "month":
140+
return Transforms.month();
141+
case "day":
142+
return Transforms.day();
143+
case "hour":
144+
return Transforms.hour();
145+
default:
146+
throw new IllegalArgumentException(name);
147+
}
148+
}
149+
150+
private static Object internalValue(Type type, LocalDateTime timestamp) {
151+
return type.typeId() == Type.TypeID.DATE ? (Object) (int) timestamp.toLocalDate().toEpochDay()
152+
: timestamp.toEpochSecond(ZoneOffset.UTC) * 1_000_000L + timestamp.getNano() / 1000;
153+
}
154+
155+
private static LocalDateTime parseBound(String bound, Type type) {
156+
return type.typeId() == Type.TypeID.DATE ? LocalDate.parse(bound).atStartOfDay()
157+
: LocalDateTime.parse(bound.replace("+00:00", "").replace(' ', 'T'));
158+
}
159+
}

0 commit comments

Comments
 (0)