Skip to content

Commit 600bdc7

Browse files
morningmanclaude
andcommitted
[test](arrow-flight) Pin the Doris-to-Arrow type mapping as it stands, one row per PrimitiveType
The schema in the table_schema column of Flight SQL GetTables is the only statement FE makes about the Arrow type of a column, and a client that types its columns from it reads the batches BE emits as that type. Until now only the nested-type rules and DATEV2 were covered; the scalar table itself, including the cells known to disagree with BE (TIMESTAMPTZ with a literal "UTC" zone, Null for TIMEV2 / VARBINARY / AGG_STATE, see apache#67577), could change without any test noticing. This records the table exactly as it is, one row per PrimitiveType plus one per precision / scale band where the band picks the Arrow unit, and fails when a PrimitiveType has no row, so a new type gets a deliberate mapping rather than the default. It is the baseline for extracting the mapping into a class of its own with no value changed. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
1 parent 14c957e commit 600bdc7

1 file changed

Lines changed: 122 additions & 0 deletions

File tree

‎fe/fe-core/src/test/java/org/apache/doris/arrowflight/FlightSqlSchemaHelperArrowTypeTest.java‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
package org.apache.doris.arrowflight;
1919

20+
import org.apache.doris.catalog.PrimitiveType;
2021
import org.apache.doris.common.jmockit.Deencapsulation;
2122
import org.apache.doris.thrift.TColumnDesc;
2223
import org.apache.doris.thrift.TPrimitiveType;
@@ -26,19 +27,27 @@
2627
import org.apache.arrow.vector.ipc.ReadChannel;
2728
import org.apache.arrow.vector.ipc.message.MessageSerializer;
2829
import org.apache.arrow.vector.types.DateUnit;
30+
import org.apache.arrow.vector.types.FloatingPointPrecision;
2931
import org.apache.arrow.vector.types.TimeUnit;
3032
import org.apache.arrow.vector.types.pojo.ArrowType;
3133
import org.apache.arrow.vector.types.pojo.Field;
3234
import org.apache.arrow.vector.types.pojo.Schema;
3335
import org.junit.jupiter.api.Assertions;
3436
import org.junit.jupiter.api.Test;
37+
import org.junit.jupiter.params.ParameterizedTest;
38+
import org.junit.jupiter.params.provider.Arguments;
39+
import org.junit.jupiter.params.provider.MethodSource;
3540

3641
import java.io.ByteArrayInputStream;
3742
import java.io.IOException;
43+
import java.lang.reflect.Method;
3844
import java.nio.channels.Channels;
3945
import java.util.Arrays;
4046
import java.util.Collections;
4147
import java.util.List;
48+
import java.util.Set;
49+
import java.util.stream.Collectors;
50+
import java.util.stream.Stream;
4251

4352
/**
4453
* What {@code CommandGetTables} says a column is, against what the query that follows actually
@@ -78,6 +87,119 @@ private static Field buildField(TColumnDesc columnDesc) {
7887
return Deencapsulation.invoke(FlightSqlSchemaHelper.class, "buildField", DB, TABLE, columnDesc);
7988
}
8089

90+
private static ArrowType arrowType(PrimitiveType type, Integer precision, Integer scale) throws Exception {
91+
// Plain reflection rather than Deencapsulation: the descriptor of a column that has no precision
92+
// or scale hands the mapping null for both, and that null is part of what the table pins.
93+
Method getArrowType = FlightSqlSchemaHelper.class.getDeclaredMethod("getArrowType",
94+
PrimitiveType.class, Integer.class, Integer.class);
95+
getArrowType.setAccessible(true);
96+
return (ArrowType) getArrowType.invoke(null, type, precision, scale);
97+
}
98+
99+
private static Arguments row(PrimitiveType type, ArrowType expected) {
100+
return Arguments.of(type, null, null, expected);
101+
}
102+
103+
private static Arguments row(PrimitiveType type, int precision, int scale, ArrowType expected) {
104+
return Arguments.of(type, precision, scale, expected);
105+
}
106+
107+
private static ArrowType timestamp(TimeUnit unit, String timezone) {
108+
return new ArrowType.Timestamp(unit, timezone);
109+
}
110+
111+
/**
112+
* The mapping as it stands today, one row per {@link PrimitiveType} (plus one per precision / scale
113+
* band where the band picks the Arrow type). This is a record of the present, not of the ideal: the
114+
* rows marked "kept as is" are known to disagree with what BE emits and stay that way on purpose
115+
* until the BE Arrow type layer is reworked, after which the whole table is corrected in one step
116+
* against a golden shared with BE (tracked in #67577). Until then a change to any row is a
117+
* behaviour change that every {@code GetTables} client sees, and this test is what makes it
118+
* deliberate.
119+
*/
120+
private static Stream<Arguments> mapping() {
121+
return Stream.of(
122+
row(PrimitiveType.BOOLEAN, new ArrowType.Bool()),
123+
row(PrimitiveType.TINYINT, new ArrowType.Int(8, true)),
124+
row(PrimitiveType.SMALLINT, new ArrowType.Int(16, true)),
125+
row(PrimitiveType.INT, new ArrowType.Int(32, true)),
126+
row(PrimitiveType.BIGINT, new ArrowType.Int(64, true)),
127+
row(PrimitiveType.FLOAT, new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)),
128+
row(PrimitiveType.DOUBLE, new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)),
129+
// BE writes a LARGEINT as its decimal text: it does not fit decimal128.
130+
row(PrimitiveType.LARGEINT, new ArrowType.Utf8()),
131+
row(PrimitiveType.CHAR, new ArrowType.Utf8()),
132+
row(PrimitiveType.VARCHAR, new ArrowType.Utf8()),
133+
row(PrimitiveType.STRING, new ArrowType.Utf8()),
134+
row(PrimitiveType.JSONB, new ArrowType.Utf8()),
135+
row(PrimitiveType.VARIANT, new ArrowType.Utf8()),
136+
// IPV4 rides in an int32 (parquet has no uint32); IPV6 is text.
137+
row(PrimitiveType.IPV4, new ArrowType.Int(32, true)),
138+
row(PrimitiveType.IPV6, new ArrowType.Utf8()),
139+
// The v1 date types stay text; DATEV2 is a day number, see dateV2IsDescribedAsDate32.
140+
row(PrimitiveType.DATE, new ArrowType.Utf8()),
141+
row(PrimitiveType.DATETIME, new ArrowType.Utf8()),
142+
row(PrimitiveType.DATEV2, new ArrowType.Date(DateUnit.DAY)),
143+
// DATETIMEV2 is a wall-clock value: a timezone-naive timestamp whose unit follows the scale,
144+
// with the bands' edges at scale 0 / 1 and 3 / 4.
145+
row(PrimitiveType.DATETIMEV2, 18, 0, timestamp(TimeUnit.SECOND, null)),
146+
row(PrimitiveType.DATETIMEV2, 19, 1, timestamp(TimeUnit.MILLISECOND, null)),
147+
row(PrimitiveType.DATETIMEV2, 21, 3, timestamp(TimeUnit.MILLISECOND, null)),
148+
row(PrimitiveType.DATETIMEV2, 22, 4, timestamp(TimeUnit.MICROSECOND, null)),
149+
row(PrimitiveType.DATETIMEV2, 24, 6, timestamp(TimeUnit.MICROSECOND, null)),
150+
row(PrimitiveType.TIMESTAMP_NS, 27, 9, timestamp(TimeUnit.NANOSECOND, null)),
151+
// The same bands as DATETIMEV2, but the timezone is the literal "UTC" where BE stamps
152+
// the session timezone. Kept as is (#67577).
153+
row(PrimitiveType.TIMESTAMPTZ, 18, 0, timestamp(TimeUnit.SECOND, "UTC")),
154+
row(PrimitiveType.TIMESTAMPTZ, 21, 3, timestamp(TimeUnit.MILLISECOND, "UTC")),
155+
row(PrimitiveType.TIMESTAMPTZ, 24, 6, timestamp(TimeUnit.MICROSECOND, "UTC")),
156+
// DECIMALV2 is always (27, 9) whatever the column declares; the v3 decimals carry theirs.
157+
row(PrimitiveType.DECIMALV2, 10, 2, new ArrowType.Decimal(27, 9, 128)),
158+
row(PrimitiveType.DECIMAL32, 9, 2, new ArrowType.Decimal(9, 2, 128)),
159+
row(PrimitiveType.DECIMAL64, 18, 4, new ArrowType.Decimal(18, 4, 128)),
160+
row(PrimitiveType.DECIMAL128, 38, 10, new ArrowType.Decimal(38, 10, 128)),
161+
row(PrimitiveType.DECIMAL256, 76, 20, new ArrowType.Decimal(76, 20, 256)),
162+
row(PrimitiveType.HLL, new ArrowType.Binary()),
163+
row(PrimitiveType.BITMAP, new ArrowType.Binary()),
164+
row(PrimitiveType.QUANTILE_STATE, new ArrowType.Binary()),
165+
// BE emits float64 for TIMEV2 and binary for VARBINARY and AGG_STATE; the schema says
166+
// Null for all three. Kept as is (#67577).
167+
row(PrimitiveType.TIMEV2, 18, 0, new ArrowType.Null()),
168+
row(PrimitiveType.VARBINARY, new ArrowType.Null()),
169+
row(PrimitiveType.AGG_STATE, new ArrowType.Null()),
170+
// The element types of a complex column live in the field's children, not in its type.
171+
row(PrimitiveType.ARRAY, new ArrowType.List()),
172+
row(PrimitiveType.MAP, new ArrowType.Map(false)),
173+
row(PrimitiveType.STRUCT, new ArrowType.Struct()),
174+
// Types that never name a stored column fall through to Null.
175+
row(PrimitiveType.INVALID_TYPE, new ArrowType.Null()),
176+
row(PrimitiveType.UNSUPPORTED, new ArrowType.Null()),
177+
row(PrimitiveType.NULL_TYPE, new ArrowType.Null()),
178+
row(PrimitiveType.LAMBDA_FUNCTION, new ArrowType.Null()),
179+
row(PrimitiveType.TEMPLATE, new ArrowType.Null()),
180+
row(PrimitiveType.BINARY, new ArrowType.Null()));
181+
}
182+
183+
@ParameterizedTest(name = "{0}({1}, {2}) is described as {3}")
184+
@MethodSource("mapping")
185+
public void everyTypeIsDescribedAsToday(PrimitiveType type, Integer precision, Integer scale,
186+
ArrowType expected) throws Exception {
187+
Assertions.assertEquals(expected, arrowType(type, precision, scale));
188+
}
189+
190+
/**
191+
* A type this table does not know is a type whose schema nobody has looked at: a new
192+
* {@link PrimitiveType} must get a row here, and the row must say what BE emits for it.
193+
*/
194+
@Test
195+
public void everyPrimitiveTypeHasARow() {
196+
Set<PrimitiveType> covered = mapping().map(row -> (PrimitiveType) row.get()[0])
197+
.collect(Collectors.toSet());
198+
for (PrimitiveType type : PrimitiveType.values()) {
199+
Assertions.assertTrue(covered.contains(type), type + " has no row in the mapping table");
200+
}
201+
}
202+
81203
/**
82204
* BE writes a DATEV2 column as {@code arrow::Date32Type} -- a day number. {@link DateUnit#MILLISECOND}
83205
* is date64, a different width and a different meaning, and a client that believes it renders and

0 commit comments

Comments
 (0)