Skip to content

Commit 28d406c

Browse files
committed
[improvement](paimon) Read Variant through Paimon JNI
### What problem does this PR solve? Issue Number: close apache#65086 Related PR: apache#66334, apache#66547, apache#66626 Problem Summary: Forward-port Variant V2 decoding across the Paimon Java scanner and native JNI bridge. The reader propagates nested access paths, materializes encoded Variant bytes into ColumnVariantV2, and accepts Paimon required-value leaves for fallback-only Variant arrays. ### Release note Improve Paimon Variant reads with nested projection and native Variant V2 materialization. ### Check List (For Author) - Test: Unit Test - Added Java vector/projection and BE Parquet schema/reader coverage; BE format/check, header hygiene, and FE checkstyle passed. Final compilation and test execution are deferred until all forward-port picks are complete. - Behavior changed: Yes, Paimon Variant reads support nested projection and fallback-array schemas. - Does this need documentation: No
1 parent fd06842 commit 28d406c

21 files changed

Lines changed: 1016 additions & 19 deletions

File tree

‎be/src/format/jni/jni_data_bridge.cpp‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
#include "core/column/column_string.h"
3030
#include "core/column/column_struct.h"
3131
#include "core/column/column_varbinary.h"
32+
#include "core/column/variant_v2/column_variant_v2.h"
3233
#include "core/data_type/data_type_array.h"
3334
#include "core/data_type/data_type_map.h"
3435
#include "core/data_type/data_type_nullable.h"
@@ -149,6 +150,9 @@ Status JniDataBridge::fill_column(TableMetaAddress& address, ColumnPtr& doris_co
149150
case PrimitiveType::TYPE_VARBINARY:
150151
status = _fill_varbinary_column(address, data_column, num_rows);
151152
break;
153+
case PrimitiveType::TYPE_VARIANT:
154+
status = _fill_variant_v2_column(address, data_column, num_rows);
155+
break;
152156
default:
153157
status = Status::InvalidArgument("Unsupported type {} in jni scanner",
154158
data_type->get_name());
@@ -180,6 +184,26 @@ Status JniDataBridge::_fill_varbinary_column(TableMetaAddress& address,
180184
return Status::OK();
181185
}
182186

187+
Status JniDataBridge::_fill_variant_v2_column(TableMetaAddress& address,
188+
MutableColumnPtr& doris_column, size_t num_rows) {
189+
const auto metadata_count = static_cast<size_t>(address.next_meta_as_long());
190+
const auto* metadata_offsets = reinterpret_cast<const uint32_t*>(address.next_meta_as_ptr());
191+
const auto* metadata_bytes = reinterpret_cast<const char*>(address.next_meta_as_ptr());
192+
const auto* metadata_ids = reinterpret_cast<const uint32_t*>(address.next_meta_as_ptr());
193+
const auto* value_offsets = reinterpret_cast<const uint32_t*>(address.next_meta_as_ptr());
194+
const auto* value_bytes = reinterpret_cast<const char*>(address.next_meta_as_ptr());
195+
196+
auto& variant_column = assert_cast<ColumnVariantV2&>(*doris_column);
197+
variant_column.insert_encoded_rows({
198+
.metadata_bytes = {metadata_bytes, metadata_offsets[metadata_count]},
199+
.metadata_offsets = {metadata_offsets, metadata_count + 1},
200+
.meta_ids = {metadata_ids, num_rows},
201+
.value_bytes = {value_bytes, value_offsets[num_rows]},
202+
.value_offsets = {value_offsets, num_rows + 1},
203+
});
204+
return Status::OK();
205+
}
206+
183207
Status JniDataBridge::_fill_string_column(TableMetaAddress& address, MutableColumnPtr& doris_column,
184208
size_t num_rows) {
185209
auto& string_col = static_cast<ColumnString&>(*doris_column);
@@ -361,6 +385,8 @@ std::string JniDataBridge::get_jni_type(const DataTypePtr& data_type) {
361385
}
362386
case TYPE_VARBINARY:
363387
return "varbinary";
388+
case TYPE_VARIANT:
389+
return "variant";
364390
// bitmap, hll, quantile_state, jsonb are transferred as strings via JNI
365391
case TYPE_BITMAP:
366392
[[fallthrough]];
@@ -438,6 +464,8 @@ std::string JniDataBridge::get_jni_type_with_different_string(const DataTypePtr&
438464
<< assert_cast<const DataTypeVarbinary*>(remove_nullable(data_type).get())->len()
439465
<< ")";
440466
return buffer.str();
467+
case TYPE_VARIANT:
468+
return "variant";
441469
case TYPE_DECIMALV2: {
442470
buffer << "decimalv2(" << DecimalV2Value::PRECISION << "," << DecimalV2Value::SCALE << ")";
443471
return buffer.str();

‎be/src/format/jni/jni_data_bridge.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,9 @@ class JniDataBridge {
154154
static Status _fill_varbinary_column(TableMetaAddress& address, MutableColumnPtr& doris_column,
155155
size_t num_rows);
156156

157+
static Status _fill_variant_v2_column(TableMetaAddress& address, MutableColumnPtr& doris_column,
158+
size_t num_rows);
159+
157160
static Status _fill_array_column(TableMetaAddress& address, MutableColumnPtr& doris_column,
158161
const DataTypePtr& data_type, size_t num_rows);
159162

‎be/src/format_v2/jni/paimon_jni_reader.cpp‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ constexpr std::string_view HADOOP_OPTION_PREFIX = "hadoop.";
3232
constexpr std::string_view DORIS_ENABLE_JNI_IO_MANAGER = "jni.enable_jni_io_manager";
3333
constexpr std::string_view DORIS_JNI_IO_MANAGER_TMP_DIR = "jni.io_manager.tmp_dir";
3434
constexpr std::string_view PAIMON_JNI_SCANNER_IO_TMP_DIR = "paimon_jni_scanner_io_tmp";
35+
constexpr std::string_view VARIANT_ACCESS_PATH_PREFIX = "variant_access_path.";
3536

3637
const std::string* get_paimon_predicate(const TFileScanRangeParams* scan_params,
3738
const TPaimonFileDesc& paimon_params) {
@@ -135,6 +136,14 @@ Status PaimonJniReader::build_scanner_params(std::map<std::string, std::string>*
135136
(*params)[std::string(HADOOP_OPTION_PREFIX) + kv.first] = kv.second;
136137
}
137138
}
139+
for (size_t column_idx = 0; column_idx < _projected_columns.size(); ++column_idx) {
140+
const auto& access_paths = _projected_columns[column_idx].variant_access_paths;
141+
for (size_t path_idx = 0; path_idx < access_paths.size(); ++path_idx) {
142+
(*params)[std::string(VARIANT_ACCESS_PATH_PREFIX) + std::to_string(column_idx) + "." +
143+
std::to_string(path_idx)] =
144+
JniDataBridge::encode_schema_values(access_paths[path_idx]);
145+
}
146+
}
138147
// TODO: Remove legacy split-level paimon_predicate, paimon_options and hadoop_conf from thrift
139148
// after the minimum supported FE always sends their scan-level replacements.
140149
return Status::OK();

‎be/src/format_v2/parquet/native_schema_desc.cpp‎

Lines changed: 27 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -286,7 +286,7 @@ class ScopedBoolOverride {
286286

287287
Status validate_variant_layout(const NativeFieldSchema& group_field,
288288
std::optional<int8_t> specification_version,
289-
bool allow_optional_shredded_metadata) {
289+
bool allow_paimon_shredded_layout) {
290290
if (specification_version.has_value() && *specification_version != 1) {
291291
return Status::NotSupported("Parquet Variant specification version {} is not supported",
292292
*specification_version);
@@ -332,7 +332,7 @@ Status validate_variant_layout(const NativeFieldSchema& group_field,
332332
// unannotated overrides; row materialization still rejects null metadata for a non-null value.
333333
const bool valid_metadata_repetition =
334334
metadata_repetition == tparquet::FieldRepetitionType::REQUIRED ||
335-
(allow_optional_shredded_metadata && typed_value != nullptr &&
335+
(allow_paimon_shredded_layout && typed_value != nullptr &&
336336
metadata_repetition == tparquet::FieldRepetitionType::OPTIONAL);
337337
if (!metadata->children.empty() || metadata->physical_type != tparquet::Type::BYTE_ARRAY ||
338338
!valid_metadata_repetition) {
@@ -360,8 +360,17 @@ Status validate_variant_layout(const NativeFieldSchema& group_field,
360360
std::function<Status(const NativeFieldSchema&)> validate_typed_value;
361361
std::function<Status(const NativeFieldSchema&, WrapperContext)> validate_wrapper;
362362
validate_wrapper = [&](const NativeFieldSchema& wrapper, WrapperContext context) -> Status {
363-
if (!wrapper.parquet_schema.__isset.repetition_type ||
364-
wrapper.parquet_schema.repetition_type != tparquet::FieldRepetitionType::REQUIRED) {
363+
const bool valid_wrapper_repetition =
364+
wrapper.parquet_schema.__isset.repetition_type &&
365+
(wrapper.parquet_schema.repetition_type ==
366+
tparquet::FieldRepetitionType::REQUIRED ||
367+
(allow_paimon_shredded_layout && wrapper.parquet_schema.repetition_type ==
368+
tparquet::FieldRepetitionType::OPTIONAL));
369+
// The Parquet Variant specification requires wrapper groups. Paimon's unannotated
370+
// physical carrier makes them optional, so accept that representation only through the
371+
// table-format override. Materialization still rejects an actually null array element;
372+
// an absent object wrapper represents a missing key.
373+
if (!valid_wrapper_repetition) {
365374
return Status::Corruption("Parquet Variant shredded wrapper {} must be required",
366375
wrapper.name);
367376
}
@@ -399,13 +408,23 @@ Status validate_variant_layout(const NativeFieldSchema& group_field,
399408
"Parquet Variant object wrapper {} requires an optional value child",
400409
wrapper.name);
401410
}
411+
// Paimon makes this leaf required because a fallback-only array element has no typed
412+
// carrier; keep the exception scoped to that exact unannotated layout.
413+
const bool allow_required_fallback = allow_paimon_shredded_layout &&
414+
context == WrapperContext::ARRAY_ELEMENT &&
415+
typed == nullptr;
416+
const bool valid_fallback_repetition =
417+
fallback != nullptr && fallback->parquet_schema.__isset.repetition_type &&
418+
(fallback->parquet_schema.repetition_type ==
419+
tparquet::FieldRepetitionType::OPTIONAL ||
420+
(allow_required_fallback && fallback->parquet_schema.repetition_type ==
421+
tparquet::FieldRepetitionType::REQUIRED));
402422
if (fallback != nullptr &&
403423
(!fallback->children.empty() || fallback->physical_type != tparquet::Type::BYTE_ARRAY ||
404-
!fallback->parquet_schema.__isset.repetition_type ||
405-
fallback->parquet_schema.repetition_type != tparquet::FieldRepetitionType::OPTIONAL)) {
424+
!valid_fallback_repetition)) {
406425
return Status::Corruption(
407-
"Parquet Variant wrapper {} value must be an optional BYTE_ARRAY",
408-
wrapper.name);
426+
"Parquet Variant wrapper {} value must be an {} BYTE_ARRAY", wrapper.name,
427+
allow_required_fallback ? "optional or required" : "optional");
409428
}
410429
if (typed != nullptr) {
411430
if (!typed->parquet_schema.__isset.repetition_type ||

‎be/src/format_v2/parquet/native_schema_desc.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ struct NativeFieldSchema {
9292

9393
Status validate_variant_layout(const NativeFieldSchema& group_field,
9494
std::optional<int8_t> specification_version = std::nullopt,
95-
bool allow_optional_shredded_metadata = false);
95+
bool allow_paimon_shredded_layout = false);
9696

9797
// V2 owns this schema tree and parser so footer/schema planning never invokes the V1 reader path.
9898
class NativeFieldDescriptor {

‎be/test/format_v2/parquet/parquet_schema_test.cpp‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -306,6 +306,59 @@ TEST(ParquetSchemaTest, AppliesPaimonShreddedVariantOverrideWithOptionalMetadata
306306
EXPECT_EQ(fields[0]->kind, ParquetColumnSchemaKind::VARIANT);
307307
}
308308

309+
TEST(ParquetSchemaTest, AppliesPaimonFallbackOnlyArrayWithRequiredValue) {
310+
auto schema = shredded_array_variant_schema(true, false);
311+
schema[1].__isset.logicalType = false;
312+
schema[2].__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL);
313+
schema[6].__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL);
314+
schema[7].__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED);
315+
316+
NativeFieldDescriptor descriptor;
317+
ASSERT_TRUE(descriptor.parse_from_thrift(schema).ok());
318+
std::vector<std::unique_ptr<ParquetColumnSchema>> fields;
319+
ASSERT_TRUE(build_parquet_column_schema(descriptor, &fields).ok());
320+
321+
const std::vector overrides {format::LocalColumnIndex::top_level(format::LocalColumnId(0))};
322+
const auto status = apply_variant_schema_overrides(descriptor, overrides, &fields);
323+
ASSERT_TRUE(status.ok()) << status;
324+
ASSERT_EQ(fields.size(), 1);
325+
EXPECT_EQ(fields[0]->kind, ParquetColumnSchemaKind::VARIANT);
326+
}
327+
328+
TEST(ParquetSchemaTest, RejectsRequiredShreddedFallbackOutsidePaimonFallbackOnlyArray) {
329+
const auto expect_override_corruption = [](std::vector<tparquet::SchemaElement> schema) {
330+
NativeFieldDescriptor descriptor;
331+
ASSERT_TRUE(descriptor.parse_from_thrift(schema).ok());
332+
std::vector<std::unique_ptr<ParquetColumnSchema>> fields;
333+
ASSERT_TRUE(build_parquet_column_schema(descriptor, &fields).ok());
334+
const std::vector overrides {format::LocalColumnIndex::top_level(format::LocalColumnId(0))};
335+
const auto status = apply_variant_schema_overrides(descriptor, overrides, &fields);
336+
EXPECT_TRUE(status.is<ErrorCode::CORRUPTION>()) << status;
337+
};
338+
339+
auto paimon_object = shredded_object_variant_schema();
340+
paimon_object[1].__isset.logicalType = false;
341+
paimon_object[2].__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL);
342+
paimon_object[5].__set_num_children(1);
343+
paimon_object[5].__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL);
344+
paimon_object[6].__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED);
345+
paimon_object.pop_back();
346+
expect_override_corruption(std::move(paimon_object));
347+
348+
auto paimon_array_with_typed_value = shredded_array_variant_schema(true, true);
349+
paimon_array_with_typed_value[1].__isset.logicalType = false;
350+
paimon_array_with_typed_value[2].__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL);
351+
paimon_array_with_typed_value[6].__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL);
352+
paimon_array_with_typed_value[7].__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED);
353+
expect_override_corruption(std::move(paimon_array_with_typed_value));
354+
355+
auto annotated_array = shredded_array_variant_schema(true, false);
356+
annotated_array[7].__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED);
357+
NativeFieldDescriptor descriptor;
358+
const auto annotated_status = descriptor.parse_from_thrift(annotated_array);
359+
EXPECT_TRUE(annotated_status.is<ErrorCode::CORRUPTION>()) << annotated_status;
360+
}
361+
309362
TEST(ParquetSchemaTest, RejectsMalformedUnannotatedVariantOverride) {
310363
auto schema = unshredded_variant_schema();
311364
schema[1].__isset.logicalType = false;

‎be/test/format_v2/parquet/variant_column_reader_test.cpp‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,24 @@ ParquetColumnSchema shredded_array_schema() {
178178
return schema;
179179
}
180180

181+
ParquetColumnSchema shredded_fallback_only_array_schema() {
182+
auto schema = unshredded_schema();
183+
auto typed = std::make_unique<ParquetColumnSchema>();
184+
typed->name = "typed_value";
185+
typed->kind = ParquetColumnSchemaKind::LIST;
186+
auto element = std::make_unique<ParquetColumnSchema>();
187+
element->name = "element";
188+
element->kind = ParquetColumnSchemaKind::STRUCT;
189+
auto value = std::make_unique<ParquetColumnSchema>();
190+
value->name = "value";
191+
value->kind = ParquetColumnSchemaKind::PRIMITIVE;
192+
value->type = std::make_shared<DataTypeString>();
193+
element->children.push_back(std::move(value));
194+
typed->children.push_back(std::move(element));
195+
schema.children.push_back(std::move(typed));
196+
return schema;
197+
}
198+
181199
ParquetColumnSchema shredded_mixed_array_schema() {
182200
auto schema = shredded_array_schema();
183201
auto* element = schema.children.back()->children[0].get();
@@ -2355,6 +2373,47 @@ TEST(VariantColumnReaderTest, MaterializesShreddedArrayElements) {
23552373
EXPECT_EQ(value.array_at(1).get_int(), 4);
23562374
}
23572375

2376+
TEST(VariantColumnReaderTest, MaterializesFallbackOnlyArrayFromRequiredValueLeaf) {
2377+
const StringRef metadata(VARIANT_EMPTY_METADATA.data(), VARIANT_EMPTY_METADATA.size());
2378+
const std::array<char, 2> first_value {
2379+
static_cast<char>(static_cast<uint8_t>(VariantPrimitiveId::INT8)
2380+
<< VARIANT_VALUE_HEADER_SHIFT),
2381+
3};
2382+
const std::array<char, 2> second_value {
2383+
static_cast<char>(static_cast<uint8_t>(VariantPrimitiveId::INT8)
2384+
<< VARIANT_VALUE_HEADER_SHIFT),
2385+
4};
2386+
2387+
auto values = ColumnString::create();
2388+
values->insert_data(first_value.data(), first_value.size());
2389+
values->insert_data(second_value.data(), second_value.size());
2390+
MutableColumns wrapper_fields;
2391+
wrapper_fields.push_back(std::move(values));
2392+
auto wrappers = ColumnStruct::create(std::move(wrapper_fields));
2393+
auto elements = ColumnNullable::create(std::move(wrappers), ColumnUInt8::create(2, 0));
2394+
auto offsets = ColumnArray::ColumnOffsets::create();
2395+
offsets->insert_value(2);
2396+
auto array = ColumnArray::create(std::move(elements), std::move(offsets));
2397+
2398+
const std::array<char, 1> ignored {0};
2399+
MutableColumns root_fields;
2400+
root_fields.push_back(nullable_strings({metadata}, {0}));
2401+
root_fields.push_back(nullable_strings({{ignored.data(), 0}}, {1}));
2402+
root_fields.push_back(ColumnNullable::create(std::move(array), ColumnUInt8::create(1, 0)));
2403+
auto physical = root_wrapper(std::move(root_fields));
2404+
2405+
auto output = make_nullable(std::make_shared<DataTypeVariantV2>())->create_column();
2406+
const auto status =
2407+
materialize_variant_rows(shredded_fallback_only_array_schema(), *physical, output);
2408+
ASSERT_TRUE(status.ok()) << status;
2409+
const auto& nullable = assert_cast<const ColumnNullable&>(*output);
2410+
const auto& variants = assert_cast<const ColumnVariantV2&>(nullable.get_nested_column());
2411+
const VariantRef value = variants.get_value_ref(0);
2412+
ASSERT_EQ(value.num_elements(), 2);
2413+
EXPECT_EQ(value.array_at(0).get_int(), 3);
2414+
EXPECT_EQ(value.array_at(1).get_int(), 4);
2415+
}
2416+
23582417
TEST(VariantColumnReaderTest, RejectsCorruptShreddedWrappersWithoutCrashing) {
23592418
const std::array<char, 2> int_seven {
23602419
static_cast<char>(static_cast<uint8_t>(VariantPrimitiveId::INT8)

‎fe/be-java-extensions/jni-spi/src/main/java/org/apache/doris/jni/spi/vec/ColumnType.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ public enum Type {
6464
IPV6(16),
6565
STRING(-1),
6666
VARBINARY(-1),
67+
VARIANT(-1),
6768
ARRAY(-1),
6869
MAP(-1),
6970
STRUCT(-1);
@@ -158,6 +159,10 @@ public boolean isVarbinaryType() {
158159
return type == Type.BINARY || type == Type.VARBINARY;
159160
}
160161

162+
public boolean isVariantType() {
163+
return type == Type.VARIANT;
164+
}
165+
161166
public boolean isComplexType() {
162167
return type == Type.ARRAY || type == Type.MAP || type == Type.STRUCT;
163168
}
@@ -251,6 +256,10 @@ public int metaSize() {
251256
case VARCHAR:
252257
// [const | nullMap | offsets | data ]
253258
return 4;
259+
case VARIANT:
260+
// [const | nullMap | metadata count | metadata offsets | metadata bytes
261+
// | metadata ids | value offsets | value bytes]
262+
return 8;
254263
default:
255264
// [const | nullMap | data]
256265
return 3;
@@ -349,6 +358,9 @@ private static ColumnType parseType(String columnName, String hiveType, boolean
349358
case "varbinary":
350359
type = Type.VARBINARY;
351360
break;
361+
case "variant":
362+
type = Type.VARIANT;
363+
break;
352364
default:
353365
if (lowerCaseType.startsWith("timestamptz")) {
354366
type = Type.TIMESTAMPTZ;

‎fe/be-java-extensions/jni-spi/src/main/java/org/apache/doris/jni/spi/vec/ColumnValue.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,14 @@ default boolean canGetCharAsBytes() {
7777

7878
byte[] getBytes();
7979

80+
default byte[] getVariantMetadata() {
81+
throw new UnsupportedOperationException("Variant metadata is not available");
82+
}
83+
84+
default byte[] getVariantValue() {
85+
throw new UnsupportedOperationException("Variant value is not available");
86+
}
87+
8088
void unpackArray(List<ColumnValue> values);
8189

8290
void unpackMap(List<ColumnValue> keys, List<ColumnValue> values);

0 commit comments

Comments
 (0)