|
| 1 | +// Licensed to the Apache Software Foundation (ASF) under one |
| 2 | +// or more contributor license agreements. See the NOTICE file |
| 3 | +// distributed with this work for additional information |
| 4 | +// regarding copyright ownership. The ASF licenses this file |
| 5 | +// to you under the Apache License, Version 2.0 (the |
| 6 | +// "License"); you may not use this file except in compliance |
| 7 | +// with the License. You may obtain a copy of the License at |
| 8 | +// |
| 9 | +// http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +// |
| 11 | +// Unless required by applicable law or agreed to in writing, |
| 12 | +// software distributed under the License is distributed on an |
| 13 | +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 14 | +// KIND, either express or implied. See the License for the |
| 15 | +// specific language governing permissions and limitations |
| 16 | +// under the License. |
| 17 | + |
| 18 | +package org.apache.doris.hudi; |
| 19 | + |
| 20 | +import org.apache.doris.common.jni.utils.OffHeap; |
| 21 | +import org.apache.doris.common.jni.vec.ColumnType; |
| 22 | +import org.apache.doris.common.jni.vec.VectorColumn; |
| 23 | + |
| 24 | +import org.apache.avro.LogicalTypes; |
| 25 | +import org.apache.avro.Schema; |
| 26 | +import org.apache.avro.generic.GenericData; |
| 27 | +import org.apache.avro.generic.GenericRecord; |
| 28 | +import org.apache.hadoop.conf.Configuration; |
| 29 | +import org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat; |
| 30 | +import org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe; |
| 31 | +import org.apache.hadoop.hive.serde2.objectinspector.StructField; |
| 32 | +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; |
| 33 | +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; |
| 34 | +import org.apache.hadoop.io.ArrayWritable; |
| 35 | +import org.apache.hadoop.io.BytesWritable; |
| 36 | +import org.apache.hadoop.io.NullWritable; |
| 37 | +import org.apache.hadoop.io.Text; |
| 38 | +import org.apache.hadoop.mapred.FileSplit; |
| 39 | +import org.apache.hadoop.mapred.JobConf; |
| 40 | +import org.apache.hadoop.mapred.RecordReader; |
| 41 | +import org.apache.hadoop.mapred.Reporter; |
| 42 | +import org.apache.hudi.hadoop.utils.HoodieRealtimeRecordReaderUtils; |
| 43 | +import org.apache.parquet.avro.AvroParquetWriter; |
| 44 | +import org.apache.parquet.hadoop.ParquetWriter; |
| 45 | +import org.junit.Assert; |
| 46 | +import org.junit.Test; |
| 47 | + |
| 48 | +import java.lang.reflect.Field; |
| 49 | +import java.lang.reflect.Method; |
| 50 | +import java.nio.ByteBuffer; |
| 51 | +import java.nio.charset.StandardCharsets; |
| 52 | +import java.nio.file.Files; |
| 53 | +import java.nio.file.Path; |
| 54 | +import java.time.ZoneOffset; |
| 55 | +import java.util.Arrays; |
| 56 | +import java.util.HashMap; |
| 57 | +import java.util.Map; |
| 58 | +import java.util.Properties; |
| 59 | +import java.util.UUID; |
| 60 | + |
| 61 | +public class HudiUuidTransportTest { |
| 62 | + private static final UUID UUID_VALUE = UUID.fromString("fedcba98-7654-3210-8123-456789abcdef"); |
| 63 | + |
| 64 | + private byte[] uuidBytes() { |
| 65 | + return ByteBuffer.allocate(16).putLong(UUID_VALUE.getMostSignificantBits()) |
| 66 | + .putLong(UUID_VALUE.getLeastSignificantBits()).array(); |
| 67 | + } |
| 68 | + |
| 69 | + @Test |
| 70 | + public void testBinaryAndTextUuidCarriers() { |
| 71 | + HadoopHudiColumnValue value = new HadoopHudiColumnValue(ZoneOffset.UTC); |
| 72 | + value.setField(ColumnType.parseType("id", "uuid"), |
| 73 | + PrimitiveObjectInspectorFactory.writableBinaryObjectInspector); |
| 74 | + BytesWritable padded = new BytesWritable(uuidBytes()); |
| 75 | + padded.setCapacity(64); |
| 76 | + for (Object carrier : Arrays.asList(padded, new Text(UUID_VALUE.toString()), |
| 77 | + new BytesWritable(UUID_VALUE.toString().getBytes(StandardCharsets.UTF_8)))) { |
| 78 | + value.setRow(carrier); |
| 79 | + Assert.assertEquals(UUID_VALUE, value.getUuid()); |
| 80 | + } |
| 81 | + value.setField(ColumnType.parseType("id", "uuid"), |
| 82 | + PrimitiveObjectInspectorFactory.javaByteArrayObjectInspector); |
| 83 | + value.setRow(uuidBytes()); |
| 84 | + Assert.assertEquals(UUID_VALUE, value.getUuid()); |
| 85 | + value.setField(ColumnType.parseType("id", "uuid"), |
| 86 | + PrimitiveObjectInspectorFactory.javaStringObjectInspector); |
| 87 | + value.setRow(UUID_VALUE.toString()); |
| 88 | + Assert.assertEquals(UUID_VALUE, value.getUuid()); |
| 89 | + value.setField(ColumnType.parseType("id", "uuid"), |
| 90 | + PrimitiveObjectInspectorFactory.javaByteArrayObjectInspector); |
| 91 | + value.setRow(new byte[8]); |
| 92 | + Assert.assertThrows(IllegalArgumentException.class, value::getUuid); |
| 93 | + } |
| 94 | + |
| 95 | + @Test |
| 96 | + public void testNestedUuidThroughHudiWritableSerdeAndJniVector() throws Exception { |
| 97 | + OffHeap.setTesting(); |
| 98 | + for (boolean fixed : new boolean[] {false, true}) { |
| 99 | + Schema uuid = LogicalTypes.uuid().addToSchema(fixed |
| 100 | + ? Schema.createFixed("uuid_fixed", null, null, 16) : Schema.create(Schema.Type.STRING)); |
| 101 | + Schema nullable = Schema.createUnion(Schema.create(Schema.Type.NULL), uuid); |
| 102 | + Schema array = Schema.createArray(nullable); |
| 103 | + Schema record = Schema.createRecord("uuid_record", null, null, false); |
| 104 | + record.setFields(Arrays.asList(new Schema.Field("ids", array), |
| 105 | + new Schema.Field("by_name", Schema.createMap(nullable)), new Schema.Field("id", nullable))); |
| 106 | + Object id = fixed ? new GenericData.Fixed(uuid, uuidBytes()) : UUID_VALUE.toString(); |
| 107 | + GenericData.Record row = new GenericData.Record(record); |
| 108 | + row.put("ids", new GenericData.Array<>(array, Arrays.asList(id, null))); |
| 109 | + Map<String, Object> byName = new HashMap<>(); |
| 110 | + byName.put("present", id); |
| 111 | + byName.put("absent", null); |
| 112 | + row.put("by_name", byName); |
| 113 | + row.put("id", null); |
| 114 | + ParquetHiveSerDe serde = new ParquetHiveSerDe(); |
| 115 | + Properties properties = new Properties(); |
| 116 | + properties.setProperty("columns", "value"); |
| 117 | + properties.setProperty("columns.types", "struct<ids:array<binary>,by_name:map<string,binary>,id:binary>"); |
| 118 | + serde.initialize(new Configuration(false), properties); |
| 119 | + StructObjectInspector inspector = (StructObjectInspector) serde.getObjectInspector(); |
| 120 | + StructField field = inspector.getStructFieldRef("value"); |
| 121 | + HadoopHudiColumnValue value = new HadoopHudiColumnValue(ZoneOffset.UTC); |
| 122 | + ColumnType type = ColumnType.parseType("value", "struct<ids:array<uuid>,by_name:map<string,uuid>,id:uuid>"); |
| 123 | + value.setField(type, field.getFieldObjectInspector()); |
| 124 | + value.setRow(HoodieRealtimeRecordReaderUtils.avroToArrayWritable(row, record)); |
| 125 | + VectorColumn column = VectorColumn.createWritableColumn(type, 1); |
| 126 | + try { |
| 127 | + column.appendValue(value); |
| 128 | + Map<?, ?> restored = column.getStructColumn(0, 1)[0]; |
| 129 | + Assert.assertEquals(Arrays.asList(UUID_VALUE, null), restored.get("ids")); |
| 130 | + Assert.assertEquals(UUID_VALUE, ((Map<?, ?>) restored.get("by_name")).get("present")); |
| 131 | + Assert.assertNull(((Map<?, ?>) restored.get("by_name")).get("absent")); |
| 132 | + Assert.assertNull(restored.get("id")); |
| 133 | + } finally { |
| 134 | + column.close(); |
| 135 | + } |
| 136 | + } |
| 137 | + } |
| 138 | + |
| 139 | + @Test |
| 140 | + public void testParquetStringAndFixedUuidWithNormalizedHiveSchema() throws Exception { |
| 141 | + Path directory = Files.createTempDirectory("hudi-uuid"); |
| 142 | + try { |
| 143 | + for (boolean fixed : new boolean[] {false, true}) { |
| 144 | + Path file = directory.resolve(fixed + ".parquet"); |
| 145 | + Schema uuid = LogicalTypes.uuid().addToSchema(fixed |
| 146 | + ? Schema.createFixed("uuid_fixed", null, null, 16) : Schema.create(Schema.Type.STRING)); |
| 147 | + Schema record = Schema.createRecord("uuid_record", null, null, false); |
| 148 | + record.setFields(Arrays.asList(new Schema.Field("id", uuid))); |
| 149 | + GenericRecord row = new GenericData.Record(record); |
| 150 | + row.put("id", fixed ? new GenericData.Fixed(uuid, uuidBytes()) : UUID_VALUE.toString()); |
| 151 | + JobConf conf = new JobConf(false); |
| 152 | + conf.set("fs.file.impl", "org.apache.hadoop.fs.RawLocalFileSystem"); |
| 153 | + org.apache.hadoop.fs.Path hadoopPath = new org.apache.hadoop.fs.Path(file.toUri()); |
| 154 | + try (ParquetWriter<GenericRecord> writer = AvroParquetWriter.<GenericRecord>builder(hadoopPath) |
| 155 | + .withSchema(record).withConf(conf).build()) { |
| 156 | + writer.write(row); |
| 157 | + } |
| 158 | + conf.set("columns", "id"); |
| 159 | + conf.set("columns.types", "binary"); |
| 160 | + conf.set("hive.io.file.read.all.columns", "true"); |
| 161 | + Properties properties = new Properties(); |
| 162 | + properties.setProperty("columns", "id"); |
| 163 | + properties.setProperty("columns.types", "binary"); |
| 164 | + ParquetHiveSerDe serde = new ParquetHiveSerDe(); |
| 165 | + serde.initialize(conf, properties); |
| 166 | + StructObjectInspector inspector = (StructObjectInspector) serde.getObjectInspector(); |
| 167 | + StructField field = inspector.getStructFieldRef("id"); |
| 168 | + try (RecordReader<NullWritable, ArrayWritable> reader = new MapredParquetInputFormat() |
| 169 | + .getRecordReader(new FileSplit(hadoopPath, 0, Files.size(file), new String[0]), |
| 170 | + conf, Reporter.NULL)) { |
| 171 | + ArrayWritable writable = reader.createValue(); |
| 172 | + Assert.assertTrue(reader.next(reader.createKey(), writable)); |
| 173 | + HadoopHudiColumnValue value = new HadoopHudiColumnValue(ZoneOffset.UTC); |
| 174 | + value.setField(ColumnType.parseType("id", "uuid"), field.getFieldObjectInspector()); |
| 175 | + value.setRow(inspector.getStructFieldData(serde.deserialize(writable), field)); |
| 176 | + Assert.assertEquals(UUID_VALUE, value.getUuid()); |
| 177 | + Assert.assertFalse(reader.next(reader.createKey(), writable)); |
| 178 | + } finally { |
| 179 | + Files.deleteIfExists(file); |
| 180 | + } |
| 181 | + } |
| 182 | + } finally { |
| 183 | + try (java.util.stream.Stream<Path> files = Files.list(directory)) { |
| 184 | + for (Path file : (Iterable<Path>) files::iterator) { |
| 185 | + Files.deleteIfExists(file); |
| 186 | + } |
| 187 | + } |
| 188 | + Files.delete(directory); |
| 189 | + } |
| 190 | + } |
| 191 | + |
| 192 | + @Test |
| 193 | + public void testScannerUsesRequestedJniTypesInsteadOfHiveCarriers() throws Exception { |
| 194 | + Map<String, String> params = new HashMap<>(); |
| 195 | + params.put("data_file_length", "0"); |
| 196 | + params.put("serde", ParquetHiveSerDe.class.getName()); |
| 197 | + params.put("required_fields", "ids,id"); |
| 198 | + params.put("columns_types", "array<uuid>#uuid"); |
| 199 | + params.put("hudi_column_names", "id,unused,ids"); |
| 200 | + params.put("hudi_column_types", "binary#string#array<binary>"); |
| 201 | + HadoopHudiJniScanner scanner = new HadoopHudiJniScanner(1, params); |
| 202 | + Method init = HadoopHudiJniScanner.class.getDeclaredMethod("initRequiredColumnsAndTypes"); |
| 203 | + init.setAccessible(true); |
| 204 | + init.invoke(scanner); |
| 205 | + Field types = HadoopHudiJniScanner.class.getDeclaredField("requiredTypes"); |
| 206 | + types.setAccessible(true); |
| 207 | + ColumnType[] actual = (ColumnType[]) types.get(scanner); |
| 208 | + Assert.assertEquals(ColumnType.Type.UUID, actual[0].getChildTypes().get(0).getType()); |
| 209 | + Assert.assertEquals(ColumnType.Type.UUID, actual[1].getType()); |
| 210 | + Method properties = HadoopHudiJniScanner.class.getDeclaredMethod("getReaderProperties"); |
| 211 | + properties.setAccessible(true); |
| 212 | + Assert.assertEquals("binary,string,array<binary>", |
| 213 | + ((Properties) properties.invoke(scanner)).getProperty("columns.types")); |
| 214 | + params.put("required_fields", ""); |
| 215 | + params.put("columns_types", ""); |
| 216 | + HadoopHudiJniScanner emptyScanner = new HadoopHudiJniScanner(1, params); |
| 217 | + init.invoke(emptyScanner); |
| 218 | + Assert.assertEquals(0, ((ColumnType[]) types.get(emptyScanner)).length); |
| 219 | + } |
| 220 | +} |
0 commit comments