|
17 | 17 |
|
18 | 18 | package org.apache.doris.arrowflight; |
19 | 19 |
|
| 20 | +import org.apache.doris.catalog.PrimitiveType; |
20 | 21 | import org.apache.doris.common.jmockit.Deencapsulation; |
21 | 22 | import org.apache.doris.thrift.TColumnDesc; |
22 | 23 | import org.apache.doris.thrift.TPrimitiveType; |
|
26 | 27 | import org.apache.arrow.vector.ipc.ReadChannel; |
27 | 28 | import org.apache.arrow.vector.ipc.message.MessageSerializer; |
28 | 29 | import org.apache.arrow.vector.types.DateUnit; |
| 30 | +import org.apache.arrow.vector.types.FloatingPointPrecision; |
29 | 31 | import org.apache.arrow.vector.types.TimeUnit; |
30 | 32 | import org.apache.arrow.vector.types.pojo.ArrowType; |
31 | 33 | import org.apache.arrow.vector.types.pojo.Field; |
32 | 34 | import org.apache.arrow.vector.types.pojo.Schema; |
33 | 35 | import org.junit.jupiter.api.Assertions; |
34 | 36 | import org.junit.jupiter.api.Test; |
| 37 | +import org.junit.jupiter.params.ParameterizedTest; |
| 38 | +import org.junit.jupiter.params.provider.Arguments; |
| 39 | +import org.junit.jupiter.params.provider.MethodSource; |
35 | 40 |
|
36 | 41 | import java.io.ByteArrayInputStream; |
37 | 42 | import java.io.IOException; |
| 43 | +import java.lang.reflect.Method; |
38 | 44 | import java.nio.channels.Channels; |
39 | 45 | import java.util.Arrays; |
40 | 46 | import java.util.Collections; |
41 | 47 | import java.util.List; |
| 48 | +import java.util.Set; |
| 49 | +import java.util.stream.Collectors; |
| 50 | +import java.util.stream.Stream; |
42 | 51 |
|
43 | 52 | /** |
44 | 53 | * What {@code CommandGetTables} says a column is, against what the query that follows actually |
@@ -78,6 +87,119 @@ private static Field buildField(TColumnDesc columnDesc) { |
78 | 87 | return Deencapsulation.invoke(FlightSqlSchemaHelper.class, "buildField", DB, TABLE, columnDesc); |
79 | 88 | } |
80 | 89 |
|
| 90 | + private static ArrowType arrowType(PrimitiveType type, Integer precision, Integer scale) throws Exception { |
| 91 | + // Plain reflection rather than Deencapsulation: the descriptor of a column that has no precision |
| 92 | + // or scale hands the mapping null for both, and that null is part of what the table pins. |
| 93 | + Method getArrowType = FlightSqlSchemaHelper.class.getDeclaredMethod("getArrowType", |
| 94 | + PrimitiveType.class, Integer.class, Integer.class); |
| 95 | + getArrowType.setAccessible(true); |
| 96 | + return (ArrowType) getArrowType.invoke(null, type, precision, scale); |
| 97 | + } |
| 98 | + |
| 99 | + private static Arguments row(PrimitiveType type, ArrowType expected) { |
| 100 | + return Arguments.of(type, null, null, expected); |
| 101 | + } |
| 102 | + |
| 103 | + private static Arguments row(PrimitiveType type, int precision, int scale, ArrowType expected) { |
| 104 | + return Arguments.of(type, precision, scale, expected); |
| 105 | + } |
| 106 | + |
| 107 | + private static ArrowType timestamp(TimeUnit unit, String timezone) { |
| 108 | + return new ArrowType.Timestamp(unit, timezone); |
| 109 | + } |
| 110 | + |
| 111 | + /** |
| 112 | + * The mapping as it stands today, one row per {@link PrimitiveType} (plus one per precision / scale |
| 113 | + * band where the band picks the Arrow type). This is a record of the present, not of the ideal: the |
| 114 | + * rows marked "kept as is" are known to disagree with what BE emits and stay that way on purpose |
| 115 | + * until the BE Arrow type layer is reworked, after which the whole table is corrected in one step |
| 116 | + * against a golden shared with BE (tracked in #67577). Until then a change to any row is a |
| 117 | + * behaviour change that every {@code GetTables} client sees, and this test is what makes it |
| 118 | + * deliberate. |
| 119 | + */ |
| 120 | + private static Stream<Arguments> mapping() { |
| 121 | + return Stream.of( |
| 122 | + row(PrimitiveType.BOOLEAN, new ArrowType.Bool()), |
| 123 | + row(PrimitiveType.TINYINT, new ArrowType.Int(8, true)), |
| 124 | + row(PrimitiveType.SMALLINT, new ArrowType.Int(16, true)), |
| 125 | + row(PrimitiveType.INT, new ArrowType.Int(32, true)), |
| 126 | + row(PrimitiveType.BIGINT, new ArrowType.Int(64, true)), |
| 127 | + row(PrimitiveType.FLOAT, new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)), |
| 128 | + row(PrimitiveType.DOUBLE, new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), |
| 129 | + // BE writes a LARGEINT as its decimal text: it does not fit decimal128. |
| 130 | + row(PrimitiveType.LARGEINT, new ArrowType.Utf8()), |
| 131 | + row(PrimitiveType.CHAR, new ArrowType.Utf8()), |
| 132 | + row(PrimitiveType.VARCHAR, new ArrowType.Utf8()), |
| 133 | + row(PrimitiveType.STRING, new ArrowType.Utf8()), |
| 134 | + row(PrimitiveType.JSONB, new ArrowType.Utf8()), |
| 135 | + row(PrimitiveType.VARIANT, new ArrowType.Utf8()), |
| 136 | + // IPV4 rides in an int32 (parquet has no uint32); IPV6 is text. |
| 137 | + row(PrimitiveType.IPV4, new ArrowType.Int(32, true)), |
| 138 | + row(PrimitiveType.IPV6, new ArrowType.Utf8()), |
| 139 | + // The v1 date types stay text; DATEV2 is a day number, see dateV2IsDescribedAsDate32. |
| 140 | + row(PrimitiveType.DATE, new ArrowType.Utf8()), |
| 141 | + row(PrimitiveType.DATETIME, new ArrowType.Utf8()), |
| 142 | + row(PrimitiveType.DATEV2, new ArrowType.Date(DateUnit.DAY)), |
| 143 | + // DATETIMEV2 is a wall-clock value: a timezone-naive timestamp whose unit follows the scale, |
| 144 | + // with the bands' edges at scale 0 / 1 and 3 / 4. |
| 145 | + row(PrimitiveType.DATETIMEV2, 18, 0, timestamp(TimeUnit.SECOND, null)), |
| 146 | + row(PrimitiveType.DATETIMEV2, 19, 1, timestamp(TimeUnit.MILLISECOND, null)), |
| 147 | + row(PrimitiveType.DATETIMEV2, 21, 3, timestamp(TimeUnit.MILLISECOND, null)), |
| 148 | + row(PrimitiveType.DATETIMEV2, 22, 4, timestamp(TimeUnit.MICROSECOND, null)), |
| 149 | + row(PrimitiveType.DATETIMEV2, 24, 6, timestamp(TimeUnit.MICROSECOND, null)), |
| 150 | + row(PrimitiveType.TIMESTAMP_NS, 27, 9, timestamp(TimeUnit.NANOSECOND, null)), |
| 151 | + // The same bands as DATETIMEV2, but the timezone is the literal "UTC" where BE stamps |
| 152 | + // the session timezone. Kept as is (#67577). |
| 153 | + row(PrimitiveType.TIMESTAMPTZ, 18, 0, timestamp(TimeUnit.SECOND, "UTC")), |
| 154 | + row(PrimitiveType.TIMESTAMPTZ, 21, 3, timestamp(TimeUnit.MILLISECOND, "UTC")), |
| 155 | + row(PrimitiveType.TIMESTAMPTZ, 24, 6, timestamp(TimeUnit.MICROSECOND, "UTC")), |
| 156 | + // DECIMALV2 is always (27, 9) whatever the column declares; the v3 decimals carry theirs. |
| 157 | + row(PrimitiveType.DECIMALV2, 10, 2, new ArrowType.Decimal(27, 9, 128)), |
| 158 | + row(PrimitiveType.DECIMAL32, 9, 2, new ArrowType.Decimal(9, 2, 128)), |
| 159 | + row(PrimitiveType.DECIMAL64, 18, 4, new ArrowType.Decimal(18, 4, 128)), |
| 160 | + row(PrimitiveType.DECIMAL128, 38, 10, new ArrowType.Decimal(38, 10, 128)), |
| 161 | + row(PrimitiveType.DECIMAL256, 76, 20, new ArrowType.Decimal(76, 20, 256)), |
| 162 | + row(PrimitiveType.HLL, new ArrowType.Binary()), |
| 163 | + row(PrimitiveType.BITMAP, new ArrowType.Binary()), |
| 164 | + row(PrimitiveType.QUANTILE_STATE, new ArrowType.Binary()), |
| 165 | + // BE emits float64 for TIMEV2 and binary for VARBINARY and AGG_STATE; the schema says |
| 166 | + // Null for all three. Kept as is (#67577). |
| 167 | + row(PrimitiveType.TIMEV2, 18, 0, new ArrowType.Null()), |
| 168 | + row(PrimitiveType.VARBINARY, new ArrowType.Null()), |
| 169 | + row(PrimitiveType.AGG_STATE, new ArrowType.Null()), |
| 170 | + // The element types of a complex column live in the field's children, not in its type. |
| 171 | + row(PrimitiveType.ARRAY, new ArrowType.List()), |
| 172 | + row(PrimitiveType.MAP, new ArrowType.Map(false)), |
| 173 | + row(PrimitiveType.STRUCT, new ArrowType.Struct()), |
| 174 | + // Types that never name a stored column fall through to Null. |
| 175 | + row(PrimitiveType.INVALID_TYPE, new ArrowType.Null()), |
| 176 | + row(PrimitiveType.UNSUPPORTED, new ArrowType.Null()), |
| 177 | + row(PrimitiveType.NULL_TYPE, new ArrowType.Null()), |
| 178 | + row(PrimitiveType.LAMBDA_FUNCTION, new ArrowType.Null()), |
| 179 | + row(PrimitiveType.TEMPLATE, new ArrowType.Null()), |
| 180 | + row(PrimitiveType.BINARY, new ArrowType.Null())); |
| 181 | + } |
| 182 | + |
| 183 | + @ParameterizedTest(name = "{0}({1}, {2}) is described as {3}") |
| 184 | + @MethodSource("mapping") |
| 185 | + public void everyTypeIsDescribedAsToday(PrimitiveType type, Integer precision, Integer scale, |
| 186 | + ArrowType expected) throws Exception { |
| 187 | + Assertions.assertEquals(expected, arrowType(type, precision, scale)); |
| 188 | + } |
| 189 | + |
| 190 | + /** |
| 191 | + * A type this table does not know is a type whose schema nobody has looked at: a new |
| 192 | + * {@link PrimitiveType} must get a row here, and the row must say what BE emits for it. |
| 193 | + */ |
| 194 | + @Test |
| 195 | + public void everyPrimitiveTypeHasARow() { |
| 196 | + Set<PrimitiveType> covered = mapping().map(row -> (PrimitiveType) row.get()[0]) |
| 197 | + .collect(Collectors.toSet()); |
| 198 | + for (PrimitiveType type : PrimitiveType.values()) { |
| 199 | + Assertions.assertTrue(covered.contains(type), type + " has no row in the mapping table"); |
| 200 | + } |
| 201 | + } |
| 202 | + |
81 | 203 | /** |
82 | 204 | * BE writes a DATEV2 column as {@code arrow::Date32Type} -- a day number. {@link DateUnit#MILLISECOND} |
83 | 205 | * is date64, a different width and a different meaning, and a client that believes it renders and |
|
0 commit comments