|
| 1 | +// Licensed to the Apache Software Foundation (ASF) under one |
| 2 | +// or more contributor license agreements. See the NOTICE file |
| 3 | +// distributed with this work for additional information |
| 4 | +// regarding copyright ownership. The ASF licenses this file |
| 5 | +// to you under the Apache License, Version 2.0 (the |
| 6 | +// "License"); you may not use this file except in compliance |
| 7 | +// with the License. You may obtain a copy of the License at |
| 8 | +// |
| 9 | +// http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +// |
| 11 | +// Unless required by applicable law or agreed to in writing, |
| 12 | +// software distributed under the License is distributed on an |
| 13 | +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 14 | +// KIND, either express or implied. See the License for the |
| 15 | +// specific language governing permissions and limitations |
| 16 | +// under the License. |
| 17 | + |
| 18 | +package org.apache.doris.connector.iceberg; |
| 19 | + |
| 20 | +import org.apache.iceberg.PartitionData; |
| 21 | +import org.apache.iceberg.PartitionSpec; |
| 22 | +import org.apache.iceberg.Schema; |
| 23 | +import org.apache.iceberg.transforms.Transform; |
| 24 | +import org.apache.iceberg.transforms.Transforms; |
| 25 | +import org.apache.iceberg.types.Type; |
| 26 | +import org.apache.iceberg.types.Types; |
| 27 | +import org.junit.jupiter.api.Assertions; |
| 28 | +import org.junit.jupiter.api.Test; |
| 29 | + |
| 30 | +import java.math.BigDecimal; |
| 31 | +import java.nio.ByteBuffer; |
| 32 | +import java.time.LocalDate; |
| 33 | +import java.time.LocalDateTime; |
| 34 | +import java.time.ZoneId; |
| 35 | +import java.time.ZoneOffset; |
| 36 | +import java.util.Arrays; |
| 37 | +import java.util.List; |
| 38 | + |
| 39 | +public class IcebergPartitionUtilsCoverageTest { |
| 40 | + @Test |
| 41 | + public void compoundPartitionWireFormatPreservesEscapingAndExactValues() { |
| 42 | + Schema schema = new Schema( |
| 43 | + Types.NestedField.optional(1, "text", Types.StringType.get()), |
| 44 | + Types.NestedField.optional(2, "nullable", Types.StringType.get()), |
| 45 | + Types.NestedField.optional(3, "amount", Types.DecimalType.of(38, 18)), |
| 46 | + Types.NestedField.optional(4, "date", Types.DateType.get()), |
| 47 | + Types.NestedField.optional(5, "timestamp", Types.TimestampNanoType.withoutZone()), |
| 48 | + Types.NestedField.optional(6, "binary", Types.BinaryType.get())); |
| 49 | + PartitionSpec spec = PartitionSpec.builderFor(schema).identity("text").identity("nullable") |
| 50 | + .identity("amount").identity("date").identity("timestamp").identity("binary").build(); |
| 51 | + PartitionData original = new PartitionData(spec.partitionType()); |
| 52 | + List<Object> values = Arrays.asList("null/\"quoted\"\\line\n\u0000雪", null, |
| 53 | + new BigDecimal("-99999999999999999999.000000000000000001"), -1, |
| 54 | + -1L, ByteBuffer.wrap(new byte[] {0, (byte) 0xff, (byte) 0x80, 34, 92})); |
| 55 | + for (int i = 0; i < values.size(); i++) { |
| 56 | + original.set(i, values.get(i)); |
| 57 | + } |
| 58 | + |
| 59 | + for (ZoneId zone : Arrays.asList(ZoneOffset.UTC, ZoneId.of("America/New_York"))) { |
| 60 | + List<String> encoded = IcebergPartitionUtils.parsePartitionValuesFromJson( |
| 61 | + IcebergPartitionUtils.getPartitionDataJson(original, spec, zone)); |
| 62 | + Assertions.assertEquals(values.size(), encoded.size()); |
| 63 | + Assertions.assertEquals(values.get(0), encoded.get(0)); |
| 64 | + Assertions.assertNull(encoded.get(1)); |
| 65 | + PartitionData decoded = new PartitionData(spec.partitionType()); |
| 66 | + for (int i = 0; i < encoded.size(); i++) { |
| 67 | + decoded.set(i, IcebergPartitionUtils.parsePartitionValueFromString( |
| 68 | + encoded.get(i), spec.partitionType().fields().get(i).type(), zone)); |
| 69 | + } |
| 70 | + // PartitionData stores binary as byte[], whose container equality compares array identities. |
| 71 | + for (int i = 0; i < original.size(); i++) { |
| 72 | + Assertions.assertEquals(original.get(i), decoded.get(i), "partition field " + i); |
| 73 | + } |
| 74 | + Assertions.assertEquals(18, ((BigDecimal) decoded.get(2)).scale()); |
| 75 | + } |
| 76 | + } |
| 77 | + |
| 78 | + @Test |
| 79 | + public void calendarRangesAgreeWithSdkTransformsAcrossEpochAndLeapBoundaries() { |
| 80 | + List<LocalDateTime> timestamps = Arrays.asList( |
| 81 | + LocalDateTime.of(1600, 2, 29, 23, 59, 59, 999999000), |
| 82 | + LocalDateTime.of(1900, 3, 1, 0, 0), |
| 83 | + LocalDateTime.of(1969, 12, 31, 23, 59, 59, 999999000), |
| 84 | + LocalDateTime.of(1970, 1, 1, 0, 0), |
| 85 | + LocalDateTime.of(2000, 2, 29, 23, 59, 59, 999999000), |
| 86 | + LocalDateTime.of(2100, 3, 1, 0, 0)); |
| 87 | + for (Type type : Arrays.asList(Types.DateType.get(), |
| 88 | + Types.TimestampType.withoutZone(), Types.TimestampType.withZone())) { |
| 89 | + for (String name : Arrays.asList("year", "month", "day", "hour")) { |
| 90 | + if (type.typeId() == Type.TypeID.DATE && name.equals("hour")) { |
| 91 | + continue; |
| 92 | + } |
| 93 | + Transform<Object, Integer> transform = calendarTransform(name); |
| 94 | + for (LocalDateTime timestamp : timestamps) { |
| 95 | + int ordinal = transform.bind(type).apply(internalValue(type, timestamp)); |
| 96 | + IcebergPartitionUtils.RangeBuild range = IcebergPartitionUtils.buildRange( |
| 97 | + name + "=" + ordinal, Integer.toString(ordinal), name, type, 100L, 200L); |
| 98 | + LocalDateTime lower = parseBound(range.getLowerBound().get(0), type); |
| 99 | + LocalDateTime upper = parseBound(range.getUpperBound().get(0), type); |
| 100 | + LocalDateTime row = type.typeId() == Type.TypeID.DATE |
| 101 | + ? timestamp.toLocalDate().atStartOfDay() : timestamp; |
| 102 | + Assertions.assertFalse(row.isBefore(lower), name + " " + timestamp); |
| 103 | + Assertions.assertTrue(row.isBefore(upper), name + " " + timestamp); |
| 104 | + // Both ends must describe exactly one SDK transform ordinal, including negative ordinals. |
| 105 | + Assertions.assertEquals(ordinal, transform.bind(type).apply(internalValue(type, lower))); |
| 106 | + LocalDateTime last = type.typeId() == Type.TypeID.DATE |
| 107 | + ? upper.minusDays(1) : upper.minusNanos(1000); |
| 108 | + Assertions.assertEquals(ordinal, transform.bind(type).apply(internalValue(type, last))); |
| 109 | + Assertions.assertEquals(ordinal + 1, transform.bind(type).apply(internalValue(type, upper))); |
| 110 | + } |
| 111 | + } |
| 112 | + } |
| 113 | + } |
| 114 | + |
| 115 | + @Test |
| 116 | + public void binarySerializationDoesNotConsumeDirectOrReadOnlyBuffers() { |
| 117 | + ByteBuffer direct = ByteBuffer.allocateDirect(6); |
| 118 | + direct.put(new byte[] {99, 0, (byte) 0xfe, (byte) 0xff, 1, 99}); |
| 119 | + direct.position(1); |
| 120 | + direct.limit(5); |
| 121 | + for (ByteBuffer buffer : Arrays.asList(direct, direct.asReadOnlyBuffer(), direct.slice())) { |
| 122 | + int position = buffer.position(); |
| 123 | + int limit = buffer.limit(); |
| 124 | + for (Type type : Arrays.asList(Types.BinaryType.get(), Types.FixedType.ofLength(4))) { |
| 125 | + String encoded = IcebergPartitionUtils.serializePartitionValue(type, buffer, ZoneOffset.UTC); |
| 126 | + Assertions.assertEquals("0x00feff01", encoded); |
| 127 | + Assertions.assertEquals(buffer, IcebergPartitionUtils.parsePartitionValueFromString( |
| 128 | + encoded, type, ZoneOffset.UTC)); |
| 129 | + Assertions.assertEquals(position, buffer.position()); |
| 130 | + Assertions.assertEquals(limit, buffer.limit()); |
| 131 | + } |
| 132 | + } |
| 133 | + } |
| 134 | + |
| 135 | + private static Transform<Object, Integer> calendarTransform(String name) { |
| 136 | + switch (name) { |
| 137 | + case "year": |
| 138 | + return Transforms.year(); |
| 139 | + case "month": |
| 140 | + return Transforms.month(); |
| 141 | + case "day": |
| 142 | + return Transforms.day(); |
| 143 | + case "hour": |
| 144 | + return Transforms.hour(); |
| 145 | + default: |
| 146 | + throw new IllegalArgumentException(name); |
| 147 | + } |
| 148 | + } |
| 149 | + |
| 150 | + private static Object internalValue(Type type, LocalDateTime timestamp) { |
| 151 | + return type.typeId() == Type.TypeID.DATE ? (Object) (int) timestamp.toLocalDate().toEpochDay() |
| 152 | + : timestamp.toEpochSecond(ZoneOffset.UTC) * 1_000_000L + timestamp.getNano() / 1000; |
| 153 | + } |
| 154 | + |
| 155 | + private static LocalDateTime parseBound(String bound, Type type) { |
| 156 | + return type.typeId() == Type.TypeID.DATE ? LocalDate.parse(bound).atStartOfDay() |
| 157 | + : LocalDateTime.parse(bound.replace("+00:00", "").replace(' ', 'T')); |
| 158 | + } |
| 159 | +} |
0 commit comments