Skip to content

Commit d80877e

Browse files
committed
[fix](hudi) Preserve UUID mapping through schema normalization
1 parent 6f00812 commit d80877e

6 files changed

Lines changed: 348 additions & 7 deletions

File tree

‎fe/be-java-extensions/hadoop-hudi-scanner/pom.xml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,19 @@ under the License.
3737
</properties>
3838

3939
<dependencies>
40+
<!-- Match preload-extensions so Hive's bundled format classes cannot shadow the test reader. -->
41+
<dependency>
42+
<groupId>org.apache.parquet</groupId>
43+
<artifactId>parquet-hadoop-bundle</artifactId>
44+
<version>${parquet.version}</version>
45+
<scope>test</scope>
46+
</dependency>
47+
<!-- Production supplies Thrift through the shared loader; standalone Parquet tests need it here. -->
48+
<dependency>
49+
<groupId>org.apache.thrift</groupId>
50+
<artifactId>libthrift</artifactId>
51+
<scope>test</scope>
52+
</dependency>
4053
<dependency>
4154
<groupId>org.apache.doris</groupId>
4255
<artifactId>java-common</artifactId>

‎fe/be-java-extensions/hadoop-hudi-scanner/src/main/java/org/apache/doris/hudi/HadoopHudiColumnValue.java‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,16 +31,20 @@
3131
import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector;
3232
import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector;
3333
import org.apache.hadoop.io.LongWritable;
34+
import org.apache.hadoop.io.Text;
3435

3536
import java.math.BigDecimal;
3637
import java.math.BigInteger;
38+
import java.nio.ByteBuffer;
39+
import java.nio.charset.StandardCharsets;
3740
import java.sql.Timestamp;
3841
import java.time.Instant;
3942
import java.time.LocalDate;
4043
import java.time.LocalDateTime;
4144
import java.time.ZoneId;
4245
import java.util.List;
4346
import java.util.Map;
47+
import java.util.UUID;
4448

4549
public class HadoopHudiColumnValue implements ColumnValue {
4650
private ColumnType dorisType;
@@ -105,6 +109,26 @@ public byte[] getBytes() {
105109
return (byte[]) inspectObject();
106110
}
107111

112+
@Override
113+
public UUID getUuid() {
114+
// Hudi normalizes UUID schemas to FIXED(16), but older log records can still carry Text.
115+
if (fieldData instanceof Text) {
116+
return UUID.fromString(fieldData.toString());
117+
}
118+
Object value = inspectObject();
119+
if (value instanceof byte[]) {
120+
byte[] bytes = (byte[]) value;
121+
if (bytes.length == 16) {
122+
// Avro fixed UUIDs use network byte order, independently of the JNI vector layout.
123+
ByteBuffer buffer = ByteBuffer.wrap(bytes);
124+
return new UUID(buffer.getLong(), buffer.getLong());
125+
}
126+
// A binary Hive inspector also exposes original Parquet STRING values as UTF-8 bytes.
127+
return UUID.fromString(new String(bytes, StandardCharsets.UTF_8));
128+
}
129+
return UUID.fromString(value.toString());
130+
}
131+
108132

109133
@Override
110134
public byte getByte() {

‎fe/be-java-extensions/hadoop-hudi-scanner/src/main/java/org/apache/doris/hudi/HadoopHudiJniScanner.java‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@ public class HadoopHudiJniScanner extends JniScanner {
8080
private final String[] requiredFields;
8181
private List<Integer> requiredColumnIds;
8282
private ColumnType[] requiredTypes;
83+
private final String[] requiredTypeNames;
8384

8485
// Hadoop info
8586
private RecordReader<NullWritable, ArrayWritable> reader;
@@ -111,6 +112,7 @@ public HadoopHudiJniScanner(int fetchSize, Map<String, String> params) {
111112

112113
this.hudiColumnNames = params.get("hudi_column_names");
113114
this.hudiColumnTypes = params.get("hudi_column_types").split("#");
115+
this.requiredTypeNames = params.get("columns_types").split("#");
114116
// Required fields will be empty when only partition fields are selected
115117
// This is because partition fields are not stored in the data files
116118
if (!params.get("required_fields").equals("")) {
@@ -216,19 +218,15 @@ private void initRequiredColumnsAndTypes() {
216218
.boxed()
217219
.collect(Collectors.toMap(i -> splitHudiColumnNames[i], i -> i));
218220

219-
Map<String, String> hudiColNameToType =
220-
IntStream.range(0, splitHudiColumnNames.length)
221-
.boxed()
222-
.collect(Collectors.toMap(i -> splitHudiColumnNames[i], i -> hudiColumnTypes[i]));
223-
224221
requiredTypes = new ColumnType[requiredFields.length];
225222
for (int i = 0; i < requiredFields.length; i++) {
226223
String requiredField = requiredFields[i];
227-
if (!hudiColNameToType.containsKey(requiredField)) {
224+
if (!hudiColNameToIdx.containsKey(requiredField)) {
228225
throw new IllegalArgumentException(
229226
"Required field " + requiredField + " not found in Hudi column names: " + splitHudiColumnNames);
230227
}
231-
requiredTypes[i] = ColumnType.parseType(requiredField, hudiColNameToType.get(requiredField));
228+
// JNI buffers must match BE's requested types; Hive carriers lose UUID logical metadata.
229+
requiredTypes[i] = ColumnType.parseType(requiredField, requiredTypeNames[i]);
232230
}
233231

234232
requiredColumnIds = Arrays.stream(requiredFields)
Lines changed: 220 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,220 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.hudi;
19+
20+
import org.apache.doris.common.jni.utils.OffHeap;
21+
import org.apache.doris.common.jni.vec.ColumnType;
22+
import org.apache.doris.common.jni.vec.VectorColumn;
23+
24+
import org.apache.avro.LogicalTypes;
25+
import org.apache.avro.Schema;
26+
import org.apache.avro.generic.GenericData;
27+
import org.apache.avro.generic.GenericRecord;
28+
import org.apache.hadoop.conf.Configuration;
29+
import org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat;
30+
import org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe;
31+
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
32+
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
33+
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
34+
import org.apache.hadoop.io.ArrayWritable;
35+
import org.apache.hadoop.io.BytesWritable;
36+
import org.apache.hadoop.io.NullWritable;
37+
import org.apache.hadoop.io.Text;
38+
import org.apache.hadoop.mapred.FileSplit;
39+
import org.apache.hadoop.mapred.JobConf;
40+
import org.apache.hadoop.mapred.RecordReader;
41+
import org.apache.hadoop.mapred.Reporter;
42+
import org.apache.hudi.hadoop.utils.HoodieRealtimeRecordReaderUtils;
43+
import org.apache.parquet.avro.AvroParquetWriter;
44+
import org.apache.parquet.hadoop.ParquetWriter;
45+
import org.junit.Assert;
46+
import org.junit.Test;
47+
48+
import java.lang.reflect.Field;
49+
import java.lang.reflect.Method;
50+
import java.nio.ByteBuffer;
51+
import java.nio.charset.StandardCharsets;
52+
import java.nio.file.Files;
53+
import java.nio.file.Path;
54+
import java.time.ZoneOffset;
55+
import java.util.Arrays;
56+
import java.util.HashMap;
57+
import java.util.Map;
58+
import java.util.Properties;
59+
import java.util.UUID;
60+
61+
public class HudiUuidTransportTest {
62+
private static final UUID UUID_VALUE = UUID.fromString("fedcba98-7654-3210-8123-456789abcdef");
63+
64+
private byte[] uuidBytes() {
65+
return ByteBuffer.allocate(16).putLong(UUID_VALUE.getMostSignificantBits())
66+
.putLong(UUID_VALUE.getLeastSignificantBits()).array();
67+
}
68+
69+
@Test
70+
public void testBinaryAndTextUuidCarriers() {
71+
HadoopHudiColumnValue value = new HadoopHudiColumnValue(ZoneOffset.UTC);
72+
value.setField(ColumnType.parseType("id", "uuid"),
73+
PrimitiveObjectInspectorFactory.writableBinaryObjectInspector);
74+
BytesWritable padded = new BytesWritable(uuidBytes());
75+
padded.setCapacity(64);
76+
for (Object carrier : Arrays.asList(padded, new Text(UUID_VALUE.toString()),
77+
new BytesWritable(UUID_VALUE.toString().getBytes(StandardCharsets.UTF_8)))) {
78+
value.setRow(carrier);
79+
Assert.assertEquals(UUID_VALUE, value.getUuid());
80+
}
81+
value.setField(ColumnType.parseType("id", "uuid"),
82+
PrimitiveObjectInspectorFactory.javaByteArrayObjectInspector);
83+
value.setRow(uuidBytes());
84+
Assert.assertEquals(UUID_VALUE, value.getUuid());
85+
value.setField(ColumnType.parseType("id", "uuid"),
86+
PrimitiveObjectInspectorFactory.javaStringObjectInspector);
87+
value.setRow(UUID_VALUE.toString());
88+
Assert.assertEquals(UUID_VALUE, value.getUuid());
89+
value.setField(ColumnType.parseType("id", "uuid"),
90+
PrimitiveObjectInspectorFactory.javaByteArrayObjectInspector);
91+
value.setRow(new byte[8]);
92+
Assert.assertThrows(IllegalArgumentException.class, value::getUuid);
93+
}
94+
95+
@Test
96+
public void testNestedUuidThroughHudiWritableSerdeAndJniVector() throws Exception {
97+
OffHeap.setTesting();
98+
for (boolean fixed : new boolean[] {false, true}) {
99+
Schema uuid = LogicalTypes.uuid().addToSchema(fixed
100+
? Schema.createFixed("uuid_fixed", null, null, 16) : Schema.create(Schema.Type.STRING));
101+
Schema nullable = Schema.createUnion(Schema.create(Schema.Type.NULL), uuid);
102+
Schema array = Schema.createArray(nullable);
103+
Schema record = Schema.createRecord("uuid_record", null, null, false);
104+
record.setFields(Arrays.asList(new Schema.Field("ids", array),
105+
new Schema.Field("by_name", Schema.createMap(nullable)), new Schema.Field("id", nullable)));
106+
Object id = fixed ? new GenericData.Fixed(uuid, uuidBytes()) : UUID_VALUE.toString();
107+
GenericData.Record row = new GenericData.Record(record);
108+
row.put("ids", new GenericData.Array<>(array, Arrays.asList(id, null)));
109+
Map<String, Object> byName = new HashMap<>();
110+
byName.put("present", id);
111+
byName.put("absent", null);
112+
row.put("by_name", byName);
113+
row.put("id", null);
114+
ParquetHiveSerDe serde = new ParquetHiveSerDe();
115+
Properties properties = new Properties();
116+
properties.setProperty("columns", "value");
117+
properties.setProperty("columns.types", "struct<ids:array<binary>,by_name:map<string,binary>,id:binary>");
118+
serde.initialize(new Configuration(false), properties);
119+
StructObjectInspector inspector = (StructObjectInspector) serde.getObjectInspector();
120+
StructField field = inspector.getStructFieldRef("value");
121+
HadoopHudiColumnValue value = new HadoopHudiColumnValue(ZoneOffset.UTC);
122+
ColumnType type = ColumnType.parseType("value", "struct<ids:array<uuid>,by_name:map<string,uuid>,id:uuid>");
123+
value.setField(type, field.getFieldObjectInspector());
124+
value.setRow(HoodieRealtimeRecordReaderUtils.avroToArrayWritable(row, record));
125+
VectorColumn column = VectorColumn.createWritableColumn(type, 1);
126+
try {
127+
column.appendValue(value);
128+
Map<?, ?> restored = column.getStructColumn(0, 1)[0];
129+
Assert.assertEquals(Arrays.asList(UUID_VALUE, null), restored.get("ids"));
130+
Assert.assertEquals(UUID_VALUE, ((Map<?, ?>) restored.get("by_name")).get("present"));
131+
Assert.assertNull(((Map<?, ?>) restored.get("by_name")).get("absent"));
132+
Assert.assertNull(restored.get("id"));
133+
} finally {
134+
column.close();
135+
}
136+
}
137+
}
138+
139+
@Test
140+
public void testParquetStringAndFixedUuidWithNormalizedHiveSchema() throws Exception {
141+
Path directory = Files.createTempDirectory("hudi-uuid");
142+
try {
143+
for (boolean fixed : new boolean[] {false, true}) {
144+
Path file = directory.resolve(fixed + ".parquet");
145+
Schema uuid = LogicalTypes.uuid().addToSchema(fixed
146+
? Schema.createFixed("uuid_fixed", null, null, 16) : Schema.create(Schema.Type.STRING));
147+
Schema record = Schema.createRecord("uuid_record", null, null, false);
148+
record.setFields(Arrays.asList(new Schema.Field("id", uuid)));
149+
GenericRecord row = new GenericData.Record(record);
150+
row.put("id", fixed ? new GenericData.Fixed(uuid, uuidBytes()) : UUID_VALUE.toString());
151+
JobConf conf = new JobConf(false);
152+
conf.set("fs.file.impl", "org.apache.hadoop.fs.RawLocalFileSystem");
153+
org.apache.hadoop.fs.Path hadoopPath = new org.apache.hadoop.fs.Path(file.toUri());
154+
try (ParquetWriter<GenericRecord> writer = AvroParquetWriter.<GenericRecord>builder(hadoopPath)
155+
.withSchema(record).withConf(conf).build()) {
156+
writer.write(row);
157+
}
158+
conf.set("columns", "id");
159+
conf.set("columns.types", "binary");
160+
conf.set("hive.io.file.read.all.columns", "true");
161+
Properties properties = new Properties();
162+
properties.setProperty("columns", "id");
163+
properties.setProperty("columns.types", "binary");
164+
ParquetHiveSerDe serde = new ParquetHiveSerDe();
165+
serde.initialize(conf, properties);
166+
StructObjectInspector inspector = (StructObjectInspector) serde.getObjectInspector();
167+
StructField field = inspector.getStructFieldRef("id");
168+
try (RecordReader<NullWritable, ArrayWritable> reader = new MapredParquetInputFormat()
169+
.getRecordReader(new FileSplit(hadoopPath, 0, Files.size(file), new String[0]),
170+
conf, Reporter.NULL)) {
171+
ArrayWritable writable = reader.createValue();
172+
Assert.assertTrue(reader.next(reader.createKey(), writable));
173+
HadoopHudiColumnValue value = new HadoopHudiColumnValue(ZoneOffset.UTC);
174+
value.setField(ColumnType.parseType("id", "uuid"), field.getFieldObjectInspector());
175+
value.setRow(inspector.getStructFieldData(serde.deserialize(writable), field));
176+
Assert.assertEquals(UUID_VALUE, value.getUuid());
177+
Assert.assertFalse(reader.next(reader.createKey(), writable));
178+
} finally {
179+
Files.deleteIfExists(file);
180+
}
181+
}
182+
} finally {
183+
try (java.util.stream.Stream<Path> files = Files.list(directory)) {
184+
for (Path file : (Iterable<Path>) files::iterator) {
185+
Files.deleteIfExists(file);
186+
}
187+
}
188+
Files.delete(directory);
189+
}
190+
}
191+
192+
@Test
193+
public void testScannerUsesRequestedJniTypesInsteadOfHiveCarriers() throws Exception {
194+
Map<String, String> params = new HashMap<>();
195+
params.put("data_file_length", "0");
196+
params.put("serde", ParquetHiveSerDe.class.getName());
197+
params.put("required_fields", "ids,id");
198+
params.put("columns_types", "array<uuid>#uuid");
199+
params.put("hudi_column_names", "id,unused,ids");
200+
params.put("hudi_column_types", "binary#string#array<binary>");
201+
HadoopHudiJniScanner scanner = new HadoopHudiJniScanner(1, params);
202+
Method init = HadoopHudiJniScanner.class.getDeclaredMethod("initRequiredColumnsAndTypes");
203+
init.setAccessible(true);
204+
init.invoke(scanner);
205+
Field types = HadoopHudiJniScanner.class.getDeclaredField("requiredTypes");
206+
types.setAccessible(true);
207+
ColumnType[] actual = (ColumnType[]) types.get(scanner);
208+
Assert.assertEquals(ColumnType.Type.UUID, actual[0].getChildTypes().get(0).getType());
209+
Assert.assertEquals(ColumnType.Type.UUID, actual[1].getType());
210+
Method properties = HadoopHudiJniScanner.class.getDeclaredMethod("getReaderProperties");
211+
properties.setAccessible(true);
212+
Assert.assertEquals("binary,string,array<binary>",
213+
((Properties) properties.invoke(scanner)).getProperty("columns.types"));
214+
params.put("required_fields", "");
215+
params.put("columns_types", "");
216+
HadoopHudiJniScanner emptyScanner = new HadoopHudiJniScanner(1, params);
217+
init.invoke(emptyScanner);
218+
Assert.assertEquals(0, ((ColumnType[]) types.get(emptyScanner)).length);
219+
}
220+
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiUtils.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,10 @@ public static String convertAvroToHiveType(Schema schema) {
100100
return "string";
101101
case FIXED:
102102
case BYTES:
103+
// Keep normalized UUID bytes intact until the JNI scanner decodes their logical type.
104+
if (logicalType != null && "uuid".equals(logicalType.getName())) {
105+
return "binary";
106+
}
103107
if (logicalType instanceof LogicalTypes.Decimal) {
104108
LogicalTypes.Decimal decimalType = (LogicalTypes.Decimal) logicalType;
105109
return String.format("decimal(%d,%d)", decimalType.getPrecision(), decimalType.getScale());
@@ -233,6 +237,10 @@ private static Type handleLongType(LogicalType logicalType) {
233237
}
234238

235239
private static Type handleFixedOrBytesType(LogicalType logicalType) {
240+
// Hudi's InternalSchema round trip rewrites Avro STRING UUIDs as annotated FIXED(16).
241+
if (logicalType != null && "uuid".equals(logicalType.getName())) {
242+
return Type.UUID;
243+
}
236244
if (logicalType instanceof LogicalTypes.Decimal) {
237245
int precision = ((LogicalTypes.Decimal) logicalType).getPrecision();
238246
int scale = ((LogicalTypes.Decimal) logicalType).getScale();

0 commit comments

Comments
 (0)