Skip to content

Commit 3b4e8f2

Browse files
committed
[fix](lance) Resolve branch-4.1 conflicts for storage options
### What problem does this PR solve? Issue Number: close apache#66772 Related PR: apache#66805 Problem Summary: Merge the latest branch-4.1 changes into the Lance REST storage-options work. Preserve the new SHOW INDEX metadata path while adapting it to the single provider-native storage option map shared by the FE Java SDK and BE lance-c readers. ### Release note None ### Check List (For Author) - Test: No need to test (merge conflict resolution; targeted git diff --check and static call-site checks passed. FE build and unit tests were not run because thirdparty/installed/bin/protoc is unavailable in the isolated worktree) - Behavior changed: No - Does this need documentation: No
2 parents 2bc5fc5 + 8479549 commit 3b4e8f2

406 files changed

Lines changed: 27237 additions & 2998 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎be/CMakeLists.txt‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -692,10 +692,17 @@ set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} clucene-contribs-lib)
692692

693693
if (ENABLE_PAIMON_CPP)
694694
if (PAIMON_FACTORY_REGISTRY_LIBS)
695-
set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES}
696-
-Wl,--whole-archive
697-
${PAIMON_FACTORY_REGISTRY_LIBS}
698-
-Wl,--no-whole-archive)
695+
if (APPLE)
696+
foreach(lib ${PAIMON_FACTORY_REGISTRY_LIBS})
697+
set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES}
698+
-Wl,-force_load,$<TARGET_FILE:${lib}>)
699+
endforeach()
700+
else()
701+
set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES}
702+
-Wl,--whole-archive
703+
${PAIMON_FACTORY_REGISTRY_LIBS}
704+
-Wl,--no-whole-archive)
705+
endif()
699706
endif()
700707

701708
# paimon-cpp internal dependencies (renamed with _paimon suffix)
@@ -779,6 +786,8 @@ else()
779786
"-framework Foundation"
780787
"-framework SystemConfiguration"
781788
"-framework Security"
789+
# liblance_c.a pulls in the Rust `sysinfo` crate, whose macOS backend calls IOKit.
790+
"-framework IOKit"
782791
)
783792
if (USE_JEMALLOC OR (NOT CMAKE_BUILD_TYPE STREQUAL "DEBUG" AND NOT CMAKE_BUILD_TYPE STREQUAL "RELEASE"))
784793
set(DORIS_LINK_LIBS
@@ -940,6 +949,7 @@ endif()
940949
option(BUILD_FILE_CACHE_MICROBENCH_TOOL "Build file cache mirobench Tool" OFF)
941950
if (BUILD_FILE_CACHE_MICROBENCH_TOOL)
942951
add_subdirectory(${SRC_DIR}/io/tools)
952+
add_subdirectory(${SRC_DIR}/io/cache/benchmark)
943953
install(FILES
944954
${BASE_DIR}/../bin/start_file_cache_microbench.sh
945955
PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE

‎be/src/common/config.cpp‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1208,6 +1208,11 @@ DEFINE_Validator(variant_storage_parse_mode,
12081208

12091209
// block file cache
12101210
DEFINE_Bool(enable_file_cache, "false");
1211+
// ATTENTION: For test only. Keep this enabled in production.
1212+
// Whether S3 storage write paths populate file cache while writing data to object storage.
1213+
// Disable this for tests that need load and compaction output to bypass file cache while keeping
1214+
// query-side file cache writes enabled.
1215+
DEFINE_mBool(enable_file_cache_write_from_s3_file_writer, "true");
12111216
// format: [{"path":"/path/to/file_cache","total_size":21474836480,"query_limit":10737418240}]
12121217
// format: [{"path":"/path/to/file_cache","total_size":21474836480,"query_limit":10737418240},{"path":"/path/to/file_cache2","total_size":21474836480,"query_limit":10737418240}]
12131218
// format: {"path": "/path/to/file_cache", "total_size":53687091200, "ttl_percent":50, "normal_percent":40, "disposable_percent":5, "index_percent":5}
@@ -1289,6 +1294,22 @@ DEFINE_mBool(file_cache_enable_only_warm_up_idx, "false");
12891294
DEFINE_Int32(file_cache_downloader_thread_num_min, "32");
12901295
DEFINE_Int32(file_cache_downloader_thread_num_max, "32");
12911296

1297+
// async file cache write
1298+
DEFINE_mBool(enable_async_file_cache_write, "false");
1299+
DEFINE_mInt32(async_file_cache_write_workers_per_disk, "16");
1300+
// A positive value is the BE-wide queued+active task ownership limit. The successfully initialized
1301+
// cache instances receive equal shares. -1 selects max(1 GiB, 1% of the BE memory limit) before
1302+
// that split.
1303+
DEFINE_mInt64(async_file_cache_write_max_pending_bytes, "-1");
1304+
DEFINE_mBool(enable_async_file_cache_write_inflight_write_buffer_index, "true");
1305+
DEFINE_Int32(async_file_cache_write_inflight_write_buffer_index_shard_count, "64");
1306+
DEFINE_Validator(async_file_cache_write_workers_per_disk,
1307+
[](int32_t value) { return value > 0 && value <= 128; });
1308+
DEFINE_Validator(async_file_cache_write_max_pending_bytes,
1309+
[](int64_t value) { return value == -1 || value > 0; });
1310+
DEFINE_Validator(async_file_cache_write_inflight_write_buffer_index_shard_count,
1311+
[](int32_t value) { return value > 0; });
1312+
12921313
DEFINE_mInt32(index_cache_entry_stay_time_after_lookup_s, "1800");
12931314
DEFINE_mInt32(inverted_index_cache_stale_sweep_time_sec, "600");
12941315
DEFINE_mBool(enable_write_index_searcher_cache, "false");
@@ -1341,6 +1362,8 @@ DEFINE_mInt64(s3_write_buffer_size, "5242880");
13411362
// Log interval when doing s3 upload task
13421363
DEFINE_mInt32(s3_file_writer_log_interval_second, "60");
13431364
DEFINE_mInt64(file_cache_max_file_reader_cache_size, "1000000");
1365+
// When file cache is enabled, the configured bytes must be divisible by
1366+
// file_cache_each_block_size so every non-EOF HDFS cache block is canonical.
13441367
DEFINE_mInt64(hdfs_write_batch_buffer_size_mb, "1"); // 1MB
13451368

13461369
//disable shrink memory by default
@@ -2210,6 +2233,7 @@ bool init(const char* conf_file, bool fill_conf_map, bool must_exist, bool set_t
22102233
} \
22112234
TYPE& ref_conf_value = *reinterpret_cast<TYPE*>((FIELD).storage); \
22122235
TYPE old_value = ref_conf_value; \
2236+
ref_conf_value = new_value; \
22132237
if (RegisterConfValidator::_s_field_validator != nullptr) { \
22142238
auto validator = RegisterConfValidator::_s_field_validator->find((FIELD).name); \
22152239
if (validator != RegisterConfValidator::_s_field_validator->end() && \
@@ -2219,7 +2243,6 @@ bool init(const char* conf_file, bool fill_conf_map, bool must_exist, bool set_t
22192243
(FIELD).name, new_value); \
22202244
} \
22212245
} \
2222-
ref_conf_value = new_value; \
22232246
if (full_conf_map != nullptr) { \
22242247
std::ostringstream oss; \
22252248
oss << new_value; \

‎be/src/common/config.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1251,6 +1251,7 @@ DECLARE_Int32(blocking_pipeline_executor_size);
12511251

12521252
// block file cache
12531253
DECLARE_Bool(enable_file_cache);
1254+
DECLARE_mBool(enable_file_cache_write_from_s3_file_writer);
12541255
// format: [{"path":"/path/to/file_cache","total_size":21474836480,"query_limit":10737418240}]
12551256
// format: [{"path":"/path/to/file_cache","total_size":21474836480,"query_limit":10737418240},{"path":"/path/to/file_cache2","total_size":21474836480,"query_limit":10737418240}]
12561257
// format: [{"path":"/path/to/file_cache","total_size":21474836480,"query_limit":10737418240, "ttl_percent":50, "normal_percent":40, "disposable_percent":5, "index_percent":5}]
@@ -1328,6 +1329,13 @@ DECLARE_mBool(enable_evaluate_shadow_queue_diff);
13281329

13291330
DECLARE_mBool(file_cache_enable_only_warm_up_idx);
13301331

1332+
// async file cache write
1333+
DECLARE_mBool(enable_async_file_cache_write);
1334+
DECLARE_mInt32(async_file_cache_write_workers_per_disk);
1335+
DECLARE_mInt64(async_file_cache_write_max_pending_bytes);
1336+
DECLARE_mBool(enable_async_file_cache_write_inflight_write_buffer_index);
1337+
DECLARE_Int32(async_file_cache_write_inflight_write_buffer_index_shard_count);
1338+
13311339
// inverted index searcher cache
13321340
// cache entry stay time after lookup
13331341
DECLARE_mInt32(index_cache_entry_stay_time_after_lookup_s);

‎be/src/common/phdr_cache.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,5 +75,7 @@ class ScopedPHDRCacheRead {
7575
ScopedPHDRCacheRead& operator=(const ScopedPHDRCacheRead&) = delete;
7676

7777
private:
78+
#if defined(__linux__) && !defined(THREAD_SANITIZER) && !defined(USE_MUSL)
7879
bool _previous = false;
80+
#endif
7981
};

‎be/src/core/column/variant_v2/column_variant_v2.cpp‎

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -500,31 +500,33 @@ class CompositeVariantShreddedState final : public VariantShreddedState {
500500
if (_segments.empty()) {
501501
return std::nullopt;
502502
}
503-
std::vector<VariantShreddedTypedValue> matches;
503+
std::vector<std::optional<VariantShreddedTypedValue>> matches;
504504
matches.reserve(_segments.size());
505505
bool all_direct = true;
506506
for (const auto& segment : _segments) {
507507
auto match = segment->find_typed_value(path);
508508
if (!match.has_value()) {
509509
all_direct = false;
510-
break;
511510
}
512-
matches.push_back(std::move(*match));
511+
// Visit every segment even after one requests normalized fallback. Format states use
512+
// find_typed_value() as their validation boundary, so stopping early would let a later
513+
// segment publish an unvalidated normalized value and make errors depend on row order.
514+
matches.push_back(std::move(match));
513515
}
514516

515-
const bool homogeneous = all_direct && matches.front().column && matches.front().type &&
517+
const auto& first = matches.front();
518+
const bool homogeneous = all_direct && first->column && first->type &&
516519
std::ranges::all_of(matches, [&](const auto& match) {
517-
return match.column && match.type && !match.normalized &&
518-
exact_typed_identity(matches.front().type, match.type);
520+
return match->column && match->type && !match->normalized &&
521+
exact_typed_identity(first->type, match->type);
519522
});
520523
if (homogeneous) {
521-
MutableColumnPtr combined = matches.front().column->clone_empty();
524+
MutableColumnPtr combined = first->column->clone_empty();
522525
for (const auto& match : matches) {
523-
combined->insert_range_from(*match.column, 0, match.column->size());
526+
combined->insert_range_from(*match->column, 0, match->column->size());
524527
}
525-
return VariantShreddedTypedValue {.column = std::move(combined),
526-
.type = matches.front().type,
527-
.normalized = nullptr};
528+
return VariantShreddedTypedValue {
529+
.column = std::move(combined), .type = first->type, .normalized = nullptr};
528530
}
529531

530532
auto normalized = find_normalized_value(path);

‎be/src/core/value/variant/variant_metadata.cpp‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,10 @@ uint32_t VariantMetadataRef::dict_size() const {
107107
}
108108

109109
StringRef VariantMetadataRef::key_at(uint32_t id) const {
110-
const Layout layout = _layout();
110+
return _key_at(_layout(), id);
111+
}
112+
113+
StringRef VariantMetadataRef::_key_at(const Layout& layout, uint32_t id) const {
111114
if (id >= layout.num_keys) {
112115
throw Exception(ErrorCode::INVALID_ARGUMENT,
113116
"Variant metadata dictionary id {} is out of range [0, {})", id,
@@ -132,7 +135,7 @@ int64_t VariantMetadataRef::find_key(StringRef key) const {
132135
const Layout layout = _layout();
133136
if (!sorted_strings()) {
134137
for (uint32_t id = 0; id < layout.num_keys; ++id) {
135-
if (key_at(id) == key) {
138+
if (_key_at(layout, id) == key) {
136139
return id;
137140
}
138141
}
@@ -143,25 +146,26 @@ int64_t VariantMetadataRef::find_key(StringRef key) const {
143146
uint32_t end = layout.num_keys;
144147
while (begin < end) {
145148
const uint32_t middle = begin + (end - begin) / 2;
146-
const int comparison = key_at(middle).compare(key);
149+
const int comparison = _key_at(layout, middle).compare(key);
147150
if (comparison < 0) {
148151
begin = middle + 1;
149152
} else {
150153
end = middle;
151154
}
152155
}
153-
if (begin < layout.num_keys && key_at(begin) == key) {
156+
if (begin < layout.num_keys && _key_at(layout, begin) == key) {
154157
return begin;
155158
}
156159
return -1;
157160
}
158161

159162
void VariantMetadataRef::validate() const {
160163
const Layout layout = _layout();
164+
const bool strings_are_sorted = sorted_strings();
161165
StringRef previous;
162166
for (uint32_t id = 0; id < layout.num_keys; ++id) {
163-
const StringRef current = key_at(id);
164-
if (sorted_strings() && id != 0 && previous.compare(current) >= 0) {
167+
const StringRef current = _key_at(layout, id);
168+
if (strings_are_sorted && id != 0 && previous.compare(current) >= 0) {
165169
throw Exception(ErrorCode::CORRUPTION,
166170
"Variant metadata dictionary is not sorted and unique at id {}", id);
167171
}

‎be/src/core/value/variant/variant_metadata.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ struct VariantMetadataRef {
4747
};
4848

4949
Layout _layout() const;
50+
StringRef _key_at(const Layout& layout, uint32_t id) const;
5051
};
5152

5253
} // namespace doris

‎be/src/core/value/variant/variant_value.cpp‎

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -469,24 +469,35 @@ bool VariantRef::object_find_by_id(uint32_t field_id, VariantRef* out) const {
469469

470470
bool VariantRef::_object_find_by_id(const ContainerLayout& layout, uint32_t field_id,
471471
VariantRef* out) const {
472+
const uint32_t dictionary_size = metadata.dict_size();
473+
if (field_id >= dictionary_size) {
474+
throw Exception(ErrorCode::INVALID_ARGUMENT,
475+
"Variant metadata dictionary id {} is out of range [0, {})", field_id,
476+
dictionary_size);
477+
}
478+
const bool strings_are_sorted = metadata.sorted_strings();
479+
// Besides supplying the key for unsorted dictionaries, this preserves the public lookup
480+
// contract's validation of the requested dictionary entry for sorted dictionaries.
472481
const StringRef target_key = metadata.key_at(field_id);
473482
uint32_t begin = 0;
474483
uint32_t end = layout.count;
475484
while (begin < end) {
476485
const uint32_t middle = begin + (end - begin) / 2;
477-
const uint32_t middle_id = _object_field_id(layout, middle);
478-
const int comparison = metadata.sorted_strings()
479-
? (middle_id > field_id) - (middle_id < field_id)
480-
: metadata.key_at(middle_id).compare(target_key);
486+
const uint32_t middle_id = _object_field_id(layout, middle, &dictionary_size);
487+
const int comparison = strings_are_sorted ? (middle_id > field_id) - (middle_id < field_id)
488+
: metadata.key_at(middle_id).compare(target_key);
481489
if (comparison < 0) {
482490
begin = middle + 1;
483491
} else {
484492
end = middle;
485493
}
486494
}
487-
if (begin == layout.count || _object_field_id(layout, begin) != field_id) {
488-
if (!metadata.sorted_strings() && begin < layout.count &&
489-
metadata.key_at(_object_field_id(layout, begin)) == target_key) {
495+
if (begin == layout.count) {
496+
return false;
497+
}
498+
const uint32_t found_id = _object_field_id(layout, begin, &dictionary_size);
499+
if (found_id != field_id) {
500+
if (!strings_are_sorted && metadata.key_at(found_id) == target_key) {
490501
*out = _container_value_at(layout, begin, false);
491502
return true;
492503
}

‎be/src/exec/connector/jni_connector.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -824,11 +824,11 @@ void JniConnector::_collect_profile_before_close() {
824824
COUNTER_UPDATE(_open_scanner_time, _jni_scanner_open_watcher);
825825
COUNTER_UPDATE(_fill_block_time, _fill_block_watcher);
826826

827-
int64_t append_data_time = 0;
827+
jlong append_data_time = 0;
828828
auto append_time_status =
829829
_jni_scanner_obj.call_long_method(env, _jni_scanner_get_append_data_time)
830830
.call(&append_data_time);
831-
int64_t create_vector_table_time = 0;
831+
jlong create_vector_table_time = 0;
832832
auto create_table_time_status =
833833
_jni_scanner_obj.call_long_method(env, _jni_scanner_get_create_vector_table_time)
834834
.call(&create_vector_table_time);

‎be/src/exec/operator/file_scan_operator.cpp‎

Lines changed: 60 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,35 @@ bool FileScanLocalState::_should_use_file_scanner_v2(const TQueryOptions& query_
152152
!is_transactional_hive;
153153
}
154154

155+
bool FileScanLocalState::_can_generate_physical_splits(const TQueryOptions& query_options,
156+
bool is_load,
157+
const TFileScanRangeParams& scan_params,
158+
const TFileRangeDesc& range) {
159+
if (!_should_use_file_scanner_v2(query_options, is_load, scan_params)) {
160+
return false;
161+
}
162+
const auto format = range.__isset.format_type ? range.format_type : scan_params.format_type;
163+
if (format == TFileFormatType::FORMAT_PARQUET || format == TFileFormatType::FORMAT_ORC) {
164+
// Keep scanner creation aligned with the downstream refinement guard. Otherwise an
165+
// Iceberg delete split creates idle scanners even though it can never publish children.
166+
return FileScannerV2::can_refine_source_split(range);
167+
}
168+
if (format != TFileFormatType::FORMAT_JNI || !range.__isset.table_format_params ||
169+
range.table_format_params.table_format_type != "paimon" ||
170+
!range.table_format_params.__isset.paimon_params) {
171+
return false;
172+
}
173+
const auto& paimon = range.table_format_params.paimon_params;
174+
return paimon.__isset.file_format &&
175+
(paimon.file_format == "parquet" || paimon.file_format == "orc") &&
176+
!paimon.__isset.paimon_split;
177+
}
178+
179+
int FileScanLocalState::_adjust_scanner_count(int requested, int initial_ranges,
180+
bool can_generate_physical_splits) {
181+
return can_generate_physical_splits ? requested : std::min(requested, initial_ranges);
182+
}
183+
155184
Status FileScanLocalState::_init_scanners(std::list<ScannerSPtr>* scanners) {
156185
if (_split_source->num_scan_ranges() == 0) {
157186
_eos = true;
@@ -258,9 +287,37 @@ void FileScanLocalState::set_scan_ranges(RuntimeState* state,
258287
if (_split_source == nullptr) {
259288
_split_source = std::make_shared<LocalSplitSourceConnector>(scan_ranges, _max_scanners);
260289
}
261-
// currently the total number of splits in the bach split mode cannot be accurately obtained,
262-
// so we don't do it in the batch split mode.
263-
_max_scanners = std::min(_max_scanners, _split_source->num_scan_ranges());
290+
// One FE columnar split can publish many format-local children after metadata planning.
291+
// Keep the requested scanner concurrency so those children do not run serially.
292+
bool can_generate_physical_splits = false;
293+
const TFileScanRangeParams* common_params = nullptr;
294+
if (state->get_query_ctx() != nullptr &&
295+
state->get_query_ctx()->file_scan_range_params_map.contains(parent_id())) {
296+
common_params = &state->get_query_ctx()->file_scan_range_params_map[parent_id()];
297+
}
298+
for (const auto& scan_range_params : scan_ranges) {
299+
const auto& file_scan_range =
300+
scan_range_params.scan_range.ext_scan_range.file_scan_range;
301+
const auto* params =
302+
file_scan_range.__isset.params ? &file_scan_range.params : common_params;
303+
if (params == nullptr) {
304+
continue;
305+
}
306+
const bool is_load =
307+
state->desc_tbl().get_tuple_descriptor(params->src_tuple_id) != nullptr;
308+
can_generate_physical_splits =
309+
std::ranges::any_of(file_scan_range.ranges, [&](const auto& range) {
310+
return _can_generate_physical_splits(state->query_options(), is_load,
311+
*params, range);
312+
});
313+
if (can_generate_physical_splits) {
314+
break;
315+
}
316+
}
317+
// Currently the total number of remote splits cannot be accurately obtained, so batch
318+
// mode already skips this cap.
319+
_max_scanners = _adjust_scanner_count(_max_scanners, _split_source->num_scan_ranges(),
320+
can_generate_physical_splits);
264321
}
265322

266323
if (!scan_ranges.empty() &&

0 commit comments

Comments
 (0)