Skip to content

Commit 0306190

Browse files
committed
update file types
1 parent a7da26c commit 0306190

17 files changed

Lines changed: 718 additions & 112 deletions

File tree

‎be/src/format/jni/jni_data_bridge.cpp‎

Lines changed: 3 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -46,34 +46,6 @@
4646

4747
namespace doris {
4848

49-
namespace {
50-
51-
// FILE keeps its Doris type in the destination block. The JNI wire layout is the same
52-
// six-child layout as STRUCT, so Java can reuse VectorColumn's existing nested transport.
53-
std::string jni_file_type(const DataTypePtr& type, bool encoded_names) {
54-
const auto& file_type = assert_cast<const DataTypeFile&>(*remove_nullable(type));
55-
std::ostringstream buffer;
56-
buffer << "struct<";
57-
for (size_t i = 0; i < DataTypeFile::FIELD_COUNT; ++i) {
58-
if (i != 0) {
59-
buffer << ",";
60-
}
61-
if (encoded_names) {
62-
std::string encoded_name;
63-
base64_encode(file_type.get_element_name(i), &encoded_name);
64-
buffer << "$" << encoded_name;
65-
} else {
66-
buffer << file_type.get_element_name(i);
67-
}
68-
// Type lengths remain authoritative on BE; Java only needs the binary value layout.
69-
buffer << ":" << JniDataBridge::get_jni_type(file_type.get_element(i));
70-
}
71-
buffer << ">";
72-
return buffer.str();
73-
}
74-
75-
} // namespace
76-
7749
#define FOR_FIXED_LENGTH_TYPES(M) \
7850
M(PrimitiveType::TYPE_TINYINT, ColumnInt8, Int8) \
7951
M(PrimitiveType::TYPE_BOOLEAN, ColumnUInt8, UInt8) \
@@ -337,7 +309,7 @@ std::string JniDataBridge::get_jni_type(const DataTypePtr& data_type) {
337309
std::ostringstream buffer;
338310
switch (type->get_primitive_type()) {
339311
case TYPE_FILE:
340-
return jni_file_type(data_type, false);
312+
return "file";
341313
case TYPE_BOOLEAN:
342314
return "boolean";
343315
case TYPE_TINYINT:
@@ -445,7 +417,7 @@ std::string JniDataBridge::get_jni_type_with_different_string(const DataTypePtr&
445417
std::ostringstream buffer;
446418
switch (data_type->get_primitive_type()) {
447419
case TYPE_FILE:
448-
return jni_file_type(data_type, false);
420+
return "file";
449421
case TYPE_BOOLEAN:
450422
return "boolean";
451423
case TYPE_TINYINT:
@@ -579,7 +551,7 @@ std::string JniDataBridge::encode_string_list(const std::vector<std::string>& va
579551
std::string JniDataBridge::get_jni_type_with_encoded_struct_fields(const DataTypePtr& data_type) {
580552
switch (data_type->get_primitive_type()) {
581553
case TYPE_FILE:
582-
return jni_file_type(data_type, true);
554+
return "file";
583555
case TYPE_STRUCT: {
584556
const auto* type_struct =
585557
assert_cast<const DataTypeStruct*>(remove_nullable(data_type).get());

‎be/src/format/jni/jni_data_bridge.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,8 @@ class JniDataBridge {
130130

131131
/**
132132
* Convert a Doris DataType to its JNI type string representation.
133-
* e.g., TYPE_INT -> "int", TYPE_DECIMAL128I -> "decimal128(p,s)"
133+
* e.g., TYPE_INT -> "int", TYPE_DECIMAL128I -> "decimal128(p,s)", TYPE_FILE -> "file".
134+
* FILE retains its logical type; both sides use the canonical six-child physical layout.
134135
*/
135136
static std::string get_jni_type(const DataTypePtr& data_type);
136137

‎be/src/format_v2/jni/paimon_jni_reader.cpp‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,6 @@ std::string PaimonJniReader::connector_class() const {
8888
Status PaimonJniReader::build_scanner_params(std::map<std::string, std::string>* params) const {
8989
DORIS_CHECK(params != nullptr);
9090
DORIS_CHECK(_scan_params != nullptr);
91-
DORIS_CHECK(_runtime_state != nullptr);
9291
params->clear();
9392

9493
const auto& paimon_params = _current_range.table_format_params.paimon_params;
@@ -103,8 +102,9 @@ Status PaimonJniReader::build_scanner_params(std::map<std::string, std::string>*
103102
!_scan_params->serialized_table_cache_key.empty()
104103
? _scan_params->serialized_table_cache_key
105104
: generate_uuid_string();
106-
(*params)["paimon_blob_max_size"] =
107-
std::to_string(_runtime_state->query_options().paimon_blob_max_size);
105+
(*params)["paimon_blob_max_size"] = std::to_string(
106+
_runtime_state != nullptr ? _runtime_state->query_options().paimon_blob_max_size
107+
: TQueryOptions().paimon_blob_max_size);
108108

109109
if (_scan_params->__isset.paimon_options && !_scan_params->paimon_options.empty()) {
110110
for (const auto& kv : _scan_params->paimon_options) {
@@ -123,6 +123,7 @@ Status PaimonJniReader::build_scanner_params(std::map<std::string, std::string>*
123123
auto enable_io_manager_it = params->find(enable_io_manager_key);
124124
if (enable_io_manager_it != params->end() && iequal(enable_io_manager_it->second, "true") &&
125125
params->find(io_manager_tmp_dir_key) == params->end()) {
126+
DORIS_CHECK(_runtime_state != nullptr);
126127
// Keep Format V2 consistent with the legacy JNI path. Paimon creates and
127128
// removes its own paimon-* child directory under these Doris storage-root scoped
128129
// parent directories.

‎be/test/format/jni/jni_data_bridge_test.cpp‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,16 @@
2222
#include <array>
2323
#include <memory>
2424

25+
#include "core/column/column_file.h"
2526
#include "core/column/column_nullable.h"
27+
#include "core/column/column_varbinary.h"
2628
#include "core/column/variant_v2/column_variant_v2.h"
29+
#include "core/data_type/data_type_array.h"
30+
#include "core/data_type/data_type_file.h"
31+
#include "core/data_type/data_type_map.h"
2732
#include "core/data_type/data_type_nullable.h"
33+
#include "core/data_type/data_type_string.h"
34+
#include "core/data_type/data_type_struct.h"
2835
#include "core/data_type/data_type_variant_v2.h"
2936

3037
namespace doris {
@@ -35,6 +42,81 @@ long address_of(T* pointer) {
3542
return reinterpret_cast<long>(pointer);
3643
}
3744

45+
TEST(JniDataBridgeTest, FileTypePreservesLogicalJniNameInNestedTypes) {
46+
DataTypePtr file_type = std::make_shared<DataTypeFile>();
47+
for (const auto& type : {file_type, make_nullable(file_type)}) {
48+
EXPECT_EQ(JniDataBridge::get_jni_type(type), "file");
49+
EXPECT_EQ(JniDataBridge::get_jni_type_with_different_string(type), "file");
50+
EXPECT_EQ(JniDataBridge::get_jni_type_with_encoded_struct_fields(type), "file");
51+
}
52+
auto array = std::make_shared<DataTypeArray>(make_nullable(file_type));
53+
auto map = std::make_shared<DataTypeMap>(std::make_shared<DataTypeString>(),
54+
make_nullable(file_type));
55+
auto nested = std::make_shared<DataTypeStruct>(DataTypes {make_nullable(file_type)},
56+
Strings {"image"});
57+
EXPECT_EQ(JniDataBridge::get_jni_type(array), "array<file>");
58+
EXPECT_EQ(JniDataBridge::get_jni_type(map), "map<string,file>");
59+
EXPECT_EQ(JniDataBridge::get_jni_type(nested), "struct<image:file>");
60+
EXPECT_EQ(JniDataBridge::get_jni_type_with_encoded_struct_fields(nested),
61+
"struct<$aW1hZ2U=:file>");
62+
}
63+
64+
TEST(JniDataBridgeTest, FillFileFromCanonicalJavaChildren) {
65+
// Rows: inline payload, descriptor, SQL NULL. All six child columns must
66+
// retain the same row count, including null placeholders.
67+
std::array<bool, 3> file_nulls {false, false, true};
68+
std::array<bool, 3> descriptor_nulls {true, false, true};
69+
std::array<bool, 3> inline_nulls {false, true, true};
70+
std::array<bool, 3> all_nulls {true, true, true};
71+
std::string uri = "file:/data.blob";
72+
std::array<int32_t, 3> uri_offsets {0, static_cast<int32_t>(uri.size()),
73+
static_cast<int32_t>(uri.size())};
74+
std::array<int64_t, 3> blob_offsets {0, 17, 0};
75+
std::array<int64_t, 3> blob_sizes {0, 11, 0};
76+
std::array<int32_t, 3> empty_offsets {0, 0, 0};
77+
std::string payload = "hello world";
78+
// Java -> BE VARBINARY entries carry length and pointer, not C++ StringView.
79+
std::array<int64_t, 6> inline_values {
80+
static_cast<int64_t>(payload.size()), address_of(payload.data()), 0, 0, 0, 0};
81+
std::array<long, 16> meta {
82+
address_of(file_nulls.data()),
83+
address_of(descriptor_nulls.data()),
84+
address_of(uri_offsets.data()),
85+
address_of(uri.data()),
86+
address_of(descriptor_nulls.data()),
87+
address_of(blob_offsets.data()),
88+
address_of(descriptor_nulls.data()),
89+
address_of(blob_sizes.data()),
90+
address_of(all_nulls.data()),
91+
address_of(empty_offsets.data()),
92+
address_of(uri.data()),
93+
address_of(all_nulls.data()),
94+
address_of(empty_offsets.data()),
95+
address_of(uri.data()),
96+
address_of(inline_nulls.data()),
97+
address_of(inline_values.data()),
98+
};
99+
JniDataBridge::TableMetaAddress address(address_of(meta.data()));
100+
DataTypePtr type = make_nullable(std::make_shared<DataTypeFile>());
101+
ColumnPtr column = type->create_column();
102+
ASSERT_TRUE(JniDataBridge::fill_column(address, column, type, 3).ok());
103+
const auto& nullable = assert_cast<const ColumnNullable&>(*column);
104+
EXPECT_EQ(nullable.get_null_map_data(), NullMap({0, 0, 1}));
105+
const auto& file = assert_cast<const ColumnFile&>(nullable.get_nested_column());
106+
ASSERT_EQ(file.size(), 3);
107+
for (size_t i = 0; i < DataTypeFile::FIELD_COUNT; ++i) {
108+
EXPECT_EQ(file.get_column(i).size(), 3);
109+
}
110+
const auto& uri_column =
111+
assert_cast<const ColumnNullable&>(file.get_column(DataTypeFile::URI_FIELD_INDEX));
112+
EXPECT_EQ(uri_column.get_null_map_data(), NullMap({1, 0, 1}));
113+
EXPECT_EQ(uri_column.get_nested_column().get_data_at(1).to_string(), uri);
114+
const auto& inline_column =
115+
assert_cast<const ColumnNullable&>(file.get_column(DataTypeFile::INLINE_FIELD_INDEX));
116+
EXPECT_EQ(inline_column.get_null_map_data(), NullMap({0, 1, 1}));
117+
EXPECT_EQ(inline_column.get_nested_column().get_data_at(0).to_string(), payload);
118+
}
119+
38120
TEST(JniDataBridgeTest, VariantTypeUsesVariantJniName) {
39121
auto variant_type = std::make_shared<DataTypeVariantV2>();
40122
auto nullable_variant_type = make_nullable(variant_type);

‎be/test/format_v2/jni/paimon_jni_reader_test.cpp‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,33 @@ TEST(PaimonJniReaderTest, UsesScanLevelPredicateBeforeLegacySplitPredicate) {
109109
EXPECT_EQ(params["paimon_predicate"], "scan-predicate");
110110
}
111111

112+
TEST(PaimonJniReaderTest, UsesDefaultBlobMaxSizeWithoutRuntimeState) {
113+
auto range = make_paimon_jni_range();
114+
auto scan_params = make_scan_params();
115+
scan_params.__set_paimon_predicate("serialized-predicate");
116+
PaimonJniReader reader;
117+
ASSERT_TRUE(init_reader(&reader, &scan_params).ok());
118+
119+
std::map<std::string, std::string> params;
120+
ASSERT_TRUE(build_params(&reader, range, &params).ok());
121+
EXPECT_EQ(params.at("paimon_blob_max_size"), "10485760");
122+
}
123+
124+
TEST(PaimonJniReaderTest, ForwardsBlobMaxSizeFromRuntimeState) {
125+
TQueryOptions query_options;
126+
query_options.__set_paimon_blob_max_size(8);
127+
RuntimeState state {query_options, TQueryGlobals()};
128+
auto range = make_paimon_jni_range();
129+
auto scan_params = make_scan_params();
130+
scan_params.__set_paimon_predicate("serialized-predicate");
131+
PaimonJniReader reader;
132+
ASSERT_TRUE(init_reader(&reader, &scan_params, &state).ok());
133+
134+
std::map<std::string, std::string> params;
135+
ASSERT_TRUE(build_params(&reader, range, &params).ok());
136+
EXPECT_EQ(params.at("paimon_blob_max_size"), "8");
137+
}
138+
112139
TEST(PaimonJniReaderTest, ForwardsSerializedTableCacheKey) {
113140
auto range = make_paimon_jni_range();
114141
range.table_format_params.paimon_params.__set_paimon_predicate("serialized-predicate");

‎fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnType.java‎

Lines changed: 27 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,8 @@ public enum Type {
6666
VARIANT(-1),
6767
ARRAY(-1),
6868
MAP(-1),
69-
STRUCT(-1);
69+
STRUCT(-1),
70+
FILE(-1);
7071

7172
public final int size;
7273

@@ -87,27 +88,15 @@ public enum Type {
8788
private int scale;
8889

8990
public ColumnType(String name, Type type) {
90-
this.name = name;
91-
this.type = type;
92-
this.length = -1;
93-
this.precision = -1;
94-
this.scale = -1;
91+
this(name, type, -1, -1, -1);
9592
}
9693

9794
public ColumnType(String name, Type type, int length) {
98-
this.name = name;
99-
this.type = type;
100-
this.length = length;
101-
this.precision = -1;
102-
this.scale = -1;
95+
this(name, type, length, -1, -1);
10396
}
10497

10598
public ColumnType(String name, Type type, int precision, int scale) {
106-
this.name = name;
107-
this.type = type;
108-
this.length = -1;
109-
this.precision = precision;
110-
this.scale = scale;
99+
this(name, type, -1, precision, scale);
111100
}
112101

113102
public ColumnType(String name, Type type, int length, int precision, int scale) {
@@ -116,6 +105,17 @@ public ColumnType(String name, Type type, int length, int precision, int scale)
116105
this.length = length;
117106
this.precision = precision;
118107
this.scale = scale;
108+
if (type == Type.FILE) {
109+
// FILE has a fixed schema, even when nested inside ARRAY, MAP or STRUCT.
110+
childNames = Arrays.asList("uri", "offset", "size", "content_type", "checksum", "inline");
111+
childTypes = Arrays.asList(
112+
new ColumnType("uri", Type.STRING),
113+
new ColumnType("offset", Type.BIGINT),
114+
new ColumnType("size", Type.BIGINT),
115+
new ColumnType("content_type", Type.STRING),
116+
new ColumnType("checksum", Type.STRING),
117+
new ColumnType("inline", Type.VARBINARY));
118+
}
119119
}
120120

121121
public List<String> getChildNames() {
@@ -163,7 +163,7 @@ public boolean isVariantType() {
163163
}
164164

165165
public boolean isComplexType() {
166-
return type == Type.ARRAY || type == Type.MAP || type == Type.STRUCT;
166+
return type == Type.ARRAY || type == Type.MAP || type == Type.STRUCT || type == Type.FILE;
167167
}
168168

169169
public boolean isArray() {
@@ -186,6 +186,10 @@ public boolean isMap() {
186186
return type == Type.MAP;
187187
}
188188

189+
public boolean isFile() {
190+
return type == Type.FILE;
191+
}
192+
189193
public boolean isStruct() {
190194
return type == Type.STRUCT;
191195
}
@@ -240,10 +244,11 @@ public int metaSize() {
240244
case ARRAY:
241245
case MAP:
242246
case STRUCT:
247+
case FILE:
243248
// array & map : [const | nullMap | offsets | ... ]
244-
// struct : [const | nullMap | ... ]
249+
// struct & file : [const | nullMap | ... ]
245250
int size = 3;
246-
if (type == Type.STRUCT) {
251+
if (type == Type.STRUCT || type == Type.FILE) {
247252
size = 2;
248253
}
249254
for (ColumnType c : childTypes) {
@@ -353,6 +358,9 @@ private static ColumnType parseType(String columnName, String hiveType, boolean
353358
case "varbinary":
354359
type = Type.VARBINARY;
355360
break;
361+
case "file":
362+
type = Type.FILE;
363+
break;
356364
case "variant":
357365
type = Type.VARIANT;
358366
break;

‎fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnValue.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,10 @@ default byte[] getVariantValue() {
8585
throw new UnsupportedOperationException("Variant value is not available");
8686
}
8787

88+
default FileValue getFile() {
89+
throw new UnsupportedOperationException("FILE is not available");
90+
}
91+
8892
void unpackArray(List<ColumnValue> values);
8993

9094
void unpackMap(List<ColumnValue> keys, List<ColumnValue> values);

0 commit comments

Comments
 (0)