|
| 1 | +// Licensed to the Apache Software Foundation (ASF) under one |
| 2 | +// or more contributor license agreements. See the NOTICE file |
| 3 | +// distributed with this work for additional information |
| 4 | +// regarding copyright ownership. The ASF licenses this file |
| 5 | +// to you under the Apache License, Version 2.0 (the |
| 6 | +// "License"); you may not use this file except in compliance |
| 7 | +// with the License. You may obtain a copy of the License at |
| 8 | +// |
| 9 | +// http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +// |
| 11 | +// Unless required by applicable law or agreed to in writing, |
| 12 | +// software distributed under the License is distributed on an |
| 13 | +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 14 | +// KIND, either express or implied. See the License for the |
| 15 | +// specific language governing permissions and limitations |
| 16 | +// under the License. |
| 17 | + |
| 18 | +package org.apache.doris.arrow; |
| 19 | + |
| 20 | +import org.apache.doris.catalog.PrimitiveType; |
| 21 | +import org.apache.doris.thrift.TColumnDesc; |
| 22 | + |
| 23 | +import org.apache.arrow.flight.sql.FlightSqlColumnMetadata; |
| 24 | +import org.apache.arrow.vector.ZeroVector; |
| 25 | +import org.apache.arrow.vector.complex.BaseRepeatedValueVector; |
| 26 | +import org.apache.arrow.vector.complex.MapVector; |
| 27 | +import org.apache.arrow.vector.types.DateUnit; |
| 28 | +import org.apache.arrow.vector.types.FloatingPointPrecision; |
| 29 | +import org.apache.arrow.vector.types.TimeUnit; |
| 30 | +import org.apache.arrow.vector.types.pojo.ArrowType; |
| 31 | +import org.apache.arrow.vector.types.pojo.Field; |
| 32 | +import org.apache.arrow.vector.types.pojo.FieldType; |
| 33 | + |
| 34 | +import java.util.ArrayList; |
| 35 | +import java.util.Arrays; |
| 36 | +import java.util.Collections; |
| 37 | +import java.util.List; |
| 38 | +import java.util.Map; |
| 39 | + |
| 40 | +/** |
| 41 | + * How FE describes a Doris column as an Arrow field. |
| 42 | + * |
| 43 | + * <p>This is the one place FE turns a Doris type into an Arrow type. Every Arrow schema FE hands a |
| 44 | + * client -- today the {@code table_schema} column of Flight SQL {@code GetTables}, next the result |
| 45 | + * and parameter schemas of prepared statements -- is built here, so that a client which types its |
| 46 | + * columns from one of them reads the batches BE emits as that type. The values mirror |
| 47 | + * {@code convert_to_arrow_type} in {@code be/src/format/arrow/arrow_row_batch.cpp}, which alone |
| 48 | + * decides the shape of the data on the wire; the cells known to disagree with it (a literal |
| 49 | + * {@code "UTC"} zone on TIMESTAMPTZ, {@code Null} for TIMEV2 / VARBINARY / AGG_STATE) are kept |
| 50 | + * deliberately until the BE Arrow type layer is reworked, and are then corrected here, once, against |
| 51 | + * a golden shared with BE (#67577). {@code DorisArrowTypeMappingTest} records every cell as it |
| 52 | + * stands, so a change to any of them is a change to that test as well. |
| 53 | + */ |
| 54 | +public final class DorisArrowTypeMapping { |
| 55 | + |
| 56 | + private DorisArrowTypeMapping() { |
| 57 | + } |
| 58 | + |
| 59 | + /** |
| 60 | + * The Arrow type of a Doris type; {@code precision} and {@code scale} are those of the column and |
| 61 | + * are null for a type that has none. |
| 62 | + */ |
| 63 | + public static ArrowType toArrowType(PrimitiveType primitiveType, Integer precision, Integer scale) { |
| 64 | + switch (primitiveType) { |
| 65 | + case BOOLEAN: |
| 66 | + return new ArrowType.Bool(); |
| 67 | + case TINYINT: |
| 68 | + return new ArrowType.Int(8, true); |
| 69 | + case SMALLINT: |
| 70 | + return new ArrowType.Int(16, true); |
| 71 | + case INT: |
| 72 | + case IPV4: |
| 73 | + return new ArrowType.Int(32, true); |
| 74 | + case BIGINT: |
| 75 | + return new ArrowType.Int(64, true); |
| 76 | + case FLOAT: |
| 77 | + return new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE); |
| 78 | + case DOUBLE: |
| 79 | + return new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE); |
| 80 | + case LARGEINT: |
| 81 | + case VARCHAR: |
| 82 | + case STRING: |
| 83 | + case CHAR: |
| 84 | + case DATETIME: |
| 85 | + case DATE: |
| 86 | + case JSONB: |
| 87 | + case IPV6: |
| 88 | + case VARIANT: |
| 89 | + return new ArrowType.Utf8(); |
| 90 | + case DATEV2: |
| 91 | + // DAY, not MILLISECOND: BE writes a DATEV2 column as arrow::Date32Type (a day number), |
| 92 | + // so a MILLISECOND unit here describes the metadata as date64 while the data that |
| 93 | + // follows is date32. A client that trusts this schema -- one reading through the ADBC |
| 94 | + // Flight SQL driver does -- then types the column as a datetime and fails the read. |
| 95 | + return new ArrowType.Date(DateUnit.DAY); |
| 96 | + case DATETIMEV2: |
| 97 | + if (scale > 3) { |
| 98 | + return new ArrowType.Timestamp(TimeUnit.MICROSECOND, null); |
| 99 | + } else if (scale > 0) { |
| 100 | + return new ArrowType.Timestamp(TimeUnit.MILLISECOND, null); |
| 101 | + } else { |
| 102 | + return new ArrowType.Timestamp(TimeUnit.SECOND, null); |
| 103 | + } |
| 104 | + case TIMESTAMP_NS: |
| 105 | + return new ArrowType.Timestamp(TimeUnit.NANOSECOND, null); |
| 106 | + case TIMESTAMPTZ: |
| 107 | + if (scale > 3) { |
| 108 | + return new ArrowType.Timestamp(TimeUnit.MICROSECOND, "UTC"); |
| 109 | + } else if (scale > 0) { |
| 110 | + return new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC"); |
| 111 | + } else { |
| 112 | + return new ArrowType.Timestamp(TimeUnit.SECOND, "UTC"); |
| 113 | + } |
| 114 | + case DECIMAL32: |
| 115 | + case DECIMAL64: |
| 116 | + case DECIMAL128: |
| 117 | + return new ArrowType.Decimal(precision, scale, 128); |
| 118 | + case DECIMAL256: |
| 119 | + return new ArrowType.Decimal(precision, scale, 256); |
| 120 | + case DECIMALV2: |
| 121 | + return new ArrowType.Decimal(27, 9, 128); |
| 122 | + case HLL: |
| 123 | + case BITMAP: |
| 124 | + case QUANTILE_STATE: |
| 125 | + return new ArrowType.Binary(); |
| 126 | + case MAP: |
| 127 | + return new ArrowType.Map(false); |
| 128 | + case ARRAY: |
| 129 | + return new ArrowType.List(); |
| 130 | + case STRUCT: |
| 131 | + return new ArrowType.Struct(); |
| 132 | + default: |
| 133 | + return new ArrowType.Null(); |
| 134 | + } |
| 135 | + } |
| 136 | + |
| 137 | + /** The Arrow type of a column as {@code describeTables} reports it. */ |
| 138 | + public static ArrowType toArrowType(TColumnDesc desc) { |
| 139 | + PrimitiveType primitiveType = PrimitiveType.fromThrift(desc.getColumnType()); |
| 140 | + Integer precision = desc.isSetColumnPrecision() ? desc.getColumnPrecision() : null; |
| 141 | + Integer scale = desc.isSetColumnScale() ? desc.getColumnScale() : null; |
| 142 | + return toArrowType(primitiveType, precision, scale); |
| 143 | + } |
| 144 | + |
| 145 | + /** |
| 146 | + * One column of {@code dbName.tableName} as an Arrow field, with its nested types described down |
| 147 | + * to the leaves and the Flight SQL column metadata a client's {@code ResultSetMetaData} reads. |
| 148 | + */ |
| 149 | + public static Field toField(String dbName, String tableName, TColumnDesc desc) { |
| 150 | + ArrowType arrowType = toArrowType(desc); |
| 151 | + return new Field(desc.getColumnName(), |
| 152 | + new FieldType(desc.isIsAllowNull(), arrowType, null, |
| 153 | + flightSqlColumnMetadata(dbName, tableName, desc)), |
| 154 | + children(dbName, tableName, desc, arrowType)); |
| 155 | + } |
| 156 | + |
| 157 | + /** |
| 158 | + * The Arrow children of a complex column, built from the descriptor's own children. |
| 159 | + * |
| 160 | + * <p>These are not decoration. An Arrow ARRAY/MAP/STRUCT type carries its element types in its |
| 161 | + * children and nowhere else, so a placeholder child says the column is an array OF NOTHING -- |
| 162 | + * and BE emits the real element type in the data ({@code convert_to_arrow_type}: ListType(item), |
| 163 | + * MapType(key, value), StructType(fields)), which leaves the schema describing one thing and the |
| 164 | + * batch carrying another. A client that types its columns from this schema (one reading through |
| 165 | + * the ADBC Flight SQL driver does) then rejects the column outright. |
| 166 | + * |
| 167 | + * <p>{@code describeTables} already reports the tree -- {@code Column.createChildrenColumn} names |
| 168 | + * an array's element "item" and a map's pair "key"/"value", which is what Arrow calls them too. |
| 169 | + * When it reports none, the old placeholders are kept rather than an empty child list: a source |
| 170 | + * that cannot describe its nested types is no worse off than before. |
| 171 | + */ |
| 172 | + private static List<Field> children(String dbName, String tableName, TColumnDesc desc, |
| 173 | + ArrowType arrowType) { |
| 174 | + List<TColumnDesc> children = desc.isSetChildren() ? desc.getChildren() : Collections.emptyList(); |
| 175 | + switch (arrowType.getTypeID()) { |
| 176 | + case List: |
| 177 | + case LargeList: |
| 178 | + case FixedSizeList: |
| 179 | + if (children.size() != 1) { |
| 180 | + return Collections.singletonList( |
| 181 | + Field.notNullable(BaseRepeatedValueVector.DATA_VECTOR_NAME, |
| 182 | + ZeroVector.INSTANCE.getField().getType())); |
| 183 | + } |
| 184 | + return Collections.singletonList(toField(dbName, tableName, children.get(0))); |
| 185 | + case Map: |
| 186 | + // Arrow spells a map as list<entries: struct<key, value>>, with the entries struct and |
| 187 | + // the key both non-nullable -- the descriptor's key nullability is not carried over, |
| 188 | + // because an Arrow map with a nullable key is not a valid schema. |
| 189 | + if (children.size() != 2) { |
| 190 | + return Collections.singletonList( |
| 191 | + Field.notNullable(MapVector.DATA_VECTOR_NAME, new ArrowType.List())); |
| 192 | + } |
| 193 | + Field key = toField(dbName, tableName, children.get(0)); |
| 194 | + Field value = toField(dbName, tableName, children.get(1)); |
| 195 | + Field entries = new Field(MapVector.DATA_VECTOR_NAME, |
| 196 | + new FieldType(false, new ArrowType.Struct(), null), |
| 197 | + Arrays.asList(new Field(key.getName(), |
| 198 | + new FieldType(false, key.getType(), null), key.getChildren()), |
| 199 | + value)); |
| 200 | + return Collections.singletonList(entries); |
| 201 | + case Struct: |
| 202 | + if (children.isEmpty()) { |
| 203 | + return Collections.emptyList(); |
| 204 | + } |
| 205 | + List<Field> structFields = new ArrayList<>(children.size()); |
| 206 | + for (TColumnDesc child : children) { |
| 207 | + structFields.add(toField(dbName, tableName, child)); |
| 208 | + } |
| 209 | + return structFields; |
| 210 | + default: |
| 211 | + return null; |
| 212 | + } |
| 213 | + } |
| 214 | + |
| 215 | + private static Map<String, String> flightSqlColumnMetadata(final String dbName, final String tableName, |
| 216 | + final TColumnDesc desc) { |
| 217 | + final FlightSqlColumnMetadata.Builder columnMetadataBuilder = new FlightSqlColumnMetadata.Builder().schemaName( |
| 218 | + dbName).tableName(tableName).typeName(PrimitiveType.fromThrift(desc.getColumnType()).toString()) |
| 219 | + .isAutoIncrement(false).isCaseSensitive(false).isReadOnly(true).isSearchable(true); |
| 220 | + |
| 221 | + if (desc.isSetColumnPrecision()) { |
| 222 | + columnMetadataBuilder.precision(desc.getColumnPrecision()); |
| 223 | + } |
| 224 | + if (desc.isSetColumnScale()) { |
| 225 | + columnMetadataBuilder.scale(desc.getColumnScale()); |
| 226 | + } |
| 227 | + return columnMetadataBuilder.build().getMetadataMap(); |
| 228 | + } |
| 229 | +} |
0 commit comments