Skip to content

Commit 2371c92

Browse files
morningmanclaude
andcommitted
[refactor](arrow-flight) Extract the Doris-to-Arrow type mapping into DorisArrowTypeMapping
FE is about to hand clients more Arrow schemas than the one in GetTables: the result and parameter schemas of prepared statements, ExecuteSchema. Each of them must describe a column exactly as the others do, or a client that types its columns from one of them fails to read the batches another promised. Until now the only mapping lived as private methods of FlightSqlSchemaHelper, where a second caller could only copy it. This moves getArrowType, the column-descriptor overload and the field builder with its nested-type rules into org.apache.doris.arrow. DorisArrowTypeMapping, unchanged: the switch, the children rules and the Flight SQL column metadata are byte for byte what they were, and the table-driven test recorded in the previous commit passes against the new class with no row touched. The cells known to disagree with BE (the literal "UTC" zone on TIMESTAMPTZ, Null for TIMEV2 / VARBINARY / AGG_STATE) are carried over as they are: they are fixed in one step against a golden shared with BE once the BE Arrow type layer has been reworked, not one call site at a time (apache#67577). FlightSqlSchemaHelper keeps the GetTables plumbing and the schema serialization; the tests split the same way, the mapping and nesting cases into DorisArrowTypeMappingTest, the serialized-schema round trips into FlightSqlSchemaHelperSerializedSchemaTest. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
1 parent 600bdc7 commit 2371c92

4 files changed

Lines changed: 347 additions & 250 deletions

File tree

Lines changed: 229 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,229 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.arrow;
19+
20+
import org.apache.doris.catalog.PrimitiveType;
21+
import org.apache.doris.thrift.TColumnDesc;
22+
23+
import org.apache.arrow.flight.sql.FlightSqlColumnMetadata;
24+
import org.apache.arrow.vector.ZeroVector;
25+
import org.apache.arrow.vector.complex.BaseRepeatedValueVector;
26+
import org.apache.arrow.vector.complex.MapVector;
27+
import org.apache.arrow.vector.types.DateUnit;
28+
import org.apache.arrow.vector.types.FloatingPointPrecision;
29+
import org.apache.arrow.vector.types.TimeUnit;
30+
import org.apache.arrow.vector.types.pojo.ArrowType;
31+
import org.apache.arrow.vector.types.pojo.Field;
32+
import org.apache.arrow.vector.types.pojo.FieldType;
33+
34+
import java.util.ArrayList;
35+
import java.util.Arrays;
36+
import java.util.Collections;
37+
import java.util.List;
38+
import java.util.Map;
39+
40+
/**
41+
* How FE describes a Doris column as an Arrow field.
42+
*
43+
* <p>This is the one place FE turns a Doris type into an Arrow type. Every Arrow schema FE hands a
44+
* client -- today the {@code table_schema} column of Flight SQL {@code GetTables}, next the result
45+
* and parameter schemas of prepared statements -- is built here, so that a client which types its
46+
* columns from one of them reads the batches BE emits as that type. The values mirror
47+
* {@code convert_to_arrow_type} in {@code be/src/format/arrow/arrow_row_batch.cpp}, which alone
48+
* decides the shape of the data on the wire; the cells known to disagree with it (a literal
49+
* {@code "UTC"} zone on TIMESTAMPTZ, {@code Null} for TIMEV2 / VARBINARY / AGG_STATE) are kept
50+
* deliberately until the BE Arrow type layer is reworked, and are then corrected here, once, against
51+
* a golden shared with BE (#67577). {@code DorisArrowTypeMappingTest} records every cell as it
52+
* stands, so a change to any of them is a change to that test as well.
53+
*/
54+
public final class DorisArrowTypeMapping {
55+
56+
private DorisArrowTypeMapping() {
57+
}
58+
59+
/**
60+
* The Arrow type of a Doris type; {@code precision} and {@code scale} are those of the column and
61+
* are null for a type that has none.
62+
*/
63+
public static ArrowType toArrowType(PrimitiveType primitiveType, Integer precision, Integer scale) {
64+
switch (primitiveType) {
65+
case BOOLEAN:
66+
return new ArrowType.Bool();
67+
case TINYINT:
68+
return new ArrowType.Int(8, true);
69+
case SMALLINT:
70+
return new ArrowType.Int(16, true);
71+
case INT:
72+
case IPV4:
73+
return new ArrowType.Int(32, true);
74+
case BIGINT:
75+
return new ArrowType.Int(64, true);
76+
case FLOAT:
77+
return new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE);
78+
case DOUBLE:
79+
return new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE);
80+
case LARGEINT:
81+
case VARCHAR:
82+
case STRING:
83+
case CHAR:
84+
case DATETIME:
85+
case DATE:
86+
case JSONB:
87+
case IPV6:
88+
case VARIANT:
89+
return new ArrowType.Utf8();
90+
case DATEV2:
91+
// DAY, not MILLISECOND: BE writes a DATEV2 column as arrow::Date32Type (a day number),
92+
// so a MILLISECOND unit here describes the metadata as date64 while the data that
93+
// follows is date32. A client that trusts this schema -- one reading through the ADBC
94+
// Flight SQL driver does -- then types the column as a datetime and fails the read.
95+
return new ArrowType.Date(DateUnit.DAY);
96+
case DATETIMEV2:
97+
if (scale > 3) {
98+
return new ArrowType.Timestamp(TimeUnit.MICROSECOND, null);
99+
} else if (scale > 0) {
100+
return new ArrowType.Timestamp(TimeUnit.MILLISECOND, null);
101+
} else {
102+
return new ArrowType.Timestamp(TimeUnit.SECOND, null);
103+
}
104+
case TIMESTAMP_NS:
105+
return new ArrowType.Timestamp(TimeUnit.NANOSECOND, null);
106+
case TIMESTAMPTZ:
107+
if (scale > 3) {
108+
return new ArrowType.Timestamp(TimeUnit.MICROSECOND, "UTC");
109+
} else if (scale > 0) {
110+
return new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC");
111+
} else {
112+
return new ArrowType.Timestamp(TimeUnit.SECOND, "UTC");
113+
}
114+
case DECIMAL32:
115+
case DECIMAL64:
116+
case DECIMAL128:
117+
return new ArrowType.Decimal(precision, scale, 128);
118+
case DECIMAL256:
119+
return new ArrowType.Decimal(precision, scale, 256);
120+
case DECIMALV2:
121+
return new ArrowType.Decimal(27, 9, 128);
122+
case HLL:
123+
case BITMAP:
124+
case QUANTILE_STATE:
125+
return new ArrowType.Binary();
126+
case MAP:
127+
return new ArrowType.Map(false);
128+
case ARRAY:
129+
return new ArrowType.List();
130+
case STRUCT:
131+
return new ArrowType.Struct();
132+
default:
133+
return new ArrowType.Null();
134+
}
135+
}
136+
137+
/** The Arrow type of a column as {@code describeTables} reports it. */
138+
public static ArrowType toArrowType(TColumnDesc desc) {
139+
PrimitiveType primitiveType = PrimitiveType.fromThrift(desc.getColumnType());
140+
Integer precision = desc.isSetColumnPrecision() ? desc.getColumnPrecision() : null;
141+
Integer scale = desc.isSetColumnScale() ? desc.getColumnScale() : null;
142+
return toArrowType(primitiveType, precision, scale);
143+
}
144+
145+
/**
146+
* One column of {@code dbName.tableName} as an Arrow field, with its nested types described down
147+
* to the leaves and the Flight SQL column metadata a client's {@code ResultSetMetaData} reads.
148+
*/
149+
public static Field toField(String dbName, String tableName, TColumnDesc desc) {
150+
ArrowType arrowType = toArrowType(desc);
151+
return new Field(desc.getColumnName(),
152+
new FieldType(desc.isIsAllowNull(), arrowType, null,
153+
flightSqlColumnMetadata(dbName, tableName, desc)),
154+
children(dbName, tableName, desc, arrowType));
155+
}
156+
157+
/**
158+
* The Arrow children of a complex column, built from the descriptor's own children.
159+
*
160+
* <p>These are not decoration. An Arrow ARRAY/MAP/STRUCT type carries its element types in its
161+
* children and nowhere else, so a placeholder child says the column is an array OF NOTHING --
162+
* and BE emits the real element type in the data ({@code convert_to_arrow_type}: ListType(item),
163+
* MapType(key, value), StructType(fields)), which leaves the schema describing one thing and the
164+
* batch carrying another. A client that types its columns from this schema (one reading through
165+
* the ADBC Flight SQL driver does) then rejects the column outright.
166+
*
167+
* <p>{@code describeTables} already reports the tree -- {@code Column.createChildrenColumn} names
168+
* an array's element "item" and a map's pair "key"/"value", which is what Arrow calls them too.
169+
* When it reports none, the old placeholders are kept rather than an empty child list: a source
170+
* that cannot describe its nested types is no worse off than before.
171+
*/
172+
private static List<Field> children(String dbName, String tableName, TColumnDesc desc,
173+
ArrowType arrowType) {
174+
List<TColumnDesc> children = desc.isSetChildren() ? desc.getChildren() : Collections.emptyList();
175+
switch (arrowType.getTypeID()) {
176+
case List:
177+
case LargeList:
178+
case FixedSizeList:
179+
if (children.size() != 1) {
180+
return Collections.singletonList(
181+
Field.notNullable(BaseRepeatedValueVector.DATA_VECTOR_NAME,
182+
ZeroVector.INSTANCE.getField().getType()));
183+
}
184+
return Collections.singletonList(toField(dbName, tableName, children.get(0)));
185+
case Map:
186+
// Arrow spells a map as list<entries: struct<key, value>>, with the entries struct and
187+
// the key both non-nullable -- the descriptor's key nullability is not carried over,
188+
// because an Arrow map with a nullable key is not a valid schema.
189+
if (children.size() != 2) {
190+
return Collections.singletonList(
191+
Field.notNullable(MapVector.DATA_VECTOR_NAME, new ArrowType.List()));
192+
}
193+
Field key = toField(dbName, tableName, children.get(0));
194+
Field value = toField(dbName, tableName, children.get(1));
195+
Field entries = new Field(MapVector.DATA_VECTOR_NAME,
196+
new FieldType(false, new ArrowType.Struct(), null),
197+
Arrays.asList(new Field(key.getName(),
198+
new FieldType(false, key.getType(), null), key.getChildren()),
199+
value));
200+
return Collections.singletonList(entries);
201+
case Struct:
202+
if (children.isEmpty()) {
203+
return Collections.emptyList();
204+
}
205+
List<Field> structFields = new ArrayList<>(children.size());
206+
for (TColumnDesc child : children) {
207+
structFields.add(toField(dbName, tableName, child));
208+
}
209+
return structFields;
210+
default:
211+
return null;
212+
}
213+
}
214+
215+
private static Map<String, String> flightSqlColumnMetadata(final String dbName, final String tableName,
216+
final TColumnDesc desc) {
217+
final FlightSqlColumnMetadata.Builder columnMetadataBuilder = new FlightSqlColumnMetadata.Builder().schemaName(
218+
dbName).tableName(tableName).typeName(PrimitiveType.fromThrift(desc.getColumnType()).toString())
219+
.isAutoIncrement(false).isCaseSensitive(false).isReadOnly(true).isSearchable(true);
220+
221+
if (desc.isSetColumnPrecision()) {
222+
columnMetadataBuilder.precision(desc.getColumnPrecision());
223+
}
224+
if (desc.isSetColumnScale()) {
225+
columnMetadataBuilder.scale(desc.getColumnScale());
226+
}
227+
return columnMetadataBuilder.build().getMetadataMap();
228+
}
229+
}

0 commit comments

Comments
 (0)