|
| 1 | +// Licensed to the Apache Software Foundation (ASF) under one |
| 2 | +// or more contributor license agreements. See the NOTICE file |
| 3 | +// distributed with this work for additional information |
| 4 | +// regarding copyright ownership. The ASF licenses this file |
| 5 | +// to you under the Apache License, Version 2.0 (the |
| 6 | +// "License"); you may not use this file except in compliance |
| 7 | +// with the License. You may obtain a copy of the License at |
| 8 | +// |
| 9 | +// http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +// |
| 11 | +// Unless required by applicable law or agreed to in writing, |
| 12 | +// software distributed under the License is distributed on an |
| 13 | +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 14 | +// KIND, either express or implied. See the License for the |
| 15 | +// specific language governing permissions and limitations |
| 16 | +// under the License. |
| 17 | + |
| 18 | +#include "exec/operator/file_scan_operator.h" |
| 19 | + |
| 20 | +#include <gtest/gtest.h> |
| 21 | + |
| 22 | +#include <cstdint> |
| 23 | +#include <limits> |
| 24 | +#include <list> |
| 25 | +#include <map> |
| 26 | +#include <memory> |
| 27 | +#include <string> |
| 28 | +#include <utility> |
| 29 | +#include <vector> |
| 30 | + |
| 31 | +#include "common/object_pool.h" |
| 32 | +#include "exec/pipeline/dependency.h" |
| 33 | +#include "exec/scan/file_scanner_v2.h" |
| 34 | +#include "exec/scan/scanner.h" |
| 35 | +#include "exec/scan/scanner_scheduler.h" |
| 36 | +#include "gen_cpp/PlanNodes_types.h" |
| 37 | +#include "runtime/descriptors.h" |
| 38 | +#include "testutil/mock/mock_runtime_state.h" |
| 39 | + |
| 40 | +namespace doris::pipeline { |
| 41 | +namespace { |
| 42 | + |
| 43 | +// One tuple with no slots: all a file scan node needs to be initialized and prepared. |
| 44 | +Status create_one_tuple_descriptors(ObjectPool* pool, DescriptorTbl** descriptors) { |
| 45 | + TDescriptorTable thrift_descriptors; |
| 46 | + TTupleDescriptor tuple_descriptor; |
| 47 | + tuple_descriptor.id = 0; |
| 48 | + tuple_descriptor.byteSize = 0; |
| 49 | + tuple_descriptor.numNullBytes = 0; |
| 50 | + thrift_descriptors.tupleDescriptors.push_back(tuple_descriptor); |
| 51 | + return DescriptorTbl::create(pool, thrift_descriptors, descriptors); |
| 52 | +} |
| 53 | + |
| 54 | +// A file scan node over that tuple. |
| 55 | +TPlanNode file_scan_plan_node() { |
| 56 | + TPlanNode plan_node; |
| 57 | + plan_node.node_id = 0; |
| 58 | + plan_node.node_type = TPlanNodeType::FILE_SCAN_NODE; |
| 59 | + plan_node.num_children = 0; |
| 60 | + plan_node.limit = -1; |
| 61 | + plan_node.row_tuples.push_back(0); |
| 62 | + plan_node.file_scan_node.tuple_id = 0; |
| 63 | + plan_node.__isset.file_scan_node = true; |
| 64 | + return plan_node; |
| 65 | +} |
| 66 | + |
| 67 | +TFileScanRangeParams scan_level_format(const std::string& table_format) { |
| 68 | + TFileScanRangeParams params; |
| 69 | + if (!table_format.empty()) { |
| 70 | + TTableFormatFileDesc fmt; |
| 71 | + fmt.__set_table_format_type(table_format); |
| 72 | + params.__set_table_format_params(fmt); |
| 73 | + } |
| 74 | + return params; |
| 75 | +} |
| 76 | + |
| 77 | +TFileRangeDesc jni_range(const std::string& table_format) { |
| 78 | + TTableFormatFileDesc fmt; |
| 79 | + fmt.__set_table_format_type(table_format); |
| 80 | + TFileRangeDesc range; |
| 81 | + range.__set_format_type(TFileFormatType::FORMAT_JNI); |
| 82 | + range.__set_table_format_params(fmt); |
| 83 | + return range; |
| 84 | +} |
| 85 | + |
| 86 | +void expect_scanners_share_node_cache(FileScanLocalState* local_state, MockRuntimeState* state, |
| 87 | + FileScanOperatorX* node) { |
| 88 | + std::list<ScannerSPtr> scanners; |
| 89 | + ASSERT_TRUE(local_state->_init_scanners(&scanners).ok()); |
| 90 | + ASSERT_FALSE(scanners.empty()); |
| 91 | + for (const auto& scanner : scanners) { |
| 92 | + const auto* file_scanner = dynamic_cast<FileScannerV2*>(scanner.get()); |
| 93 | + ASSERT_NE(file_scanner, nullptr); |
| 94 | + EXPECT_EQ(file_scanner->_kv_cache, node->_kv_cache.get()); |
| 95 | + } |
| 96 | +} |
| 97 | + |
| 98 | +} // namespace |
| 99 | + |
| 100 | +// All instances of a node share cached log-tail and delete-file data. A per-instance cache |
| 101 | +// would reload the same entry when splits referencing it are assigned to different instances. |
| 102 | +TEST(FileScanOperatorCacheTest, EveryInstanceOfTheNodeReadsThroughTheNodesCache) { |
| 103 | + ObjectPool pool; |
| 104 | + DescriptorTbl* descriptors = nullptr; |
| 105 | + ASSERT_TRUE(create_one_tuple_descriptors(&pool, &descriptors).ok()); |
| 106 | + MockRuntimeState state; |
| 107 | + state.set_desc_tbl(descriptors); |
| 108 | + |
| 109 | + const TPlanNode plan_node = file_scan_plan_node(); |
| 110 | + FileScanOperatorX node(&pool, plan_node, 0, *descriptors, /*parallel_tasks=*/1); |
| 111 | + ASSERT_TRUE(node.init(plan_node, &state).ok()); |
| 112 | + ASSERT_TRUE(node.prepare(&state).ok()); |
| 113 | + ASSERT_NE(node._kv_cache, nullptr); |
| 114 | + |
| 115 | + TFileRangeDesc log_range = jni_range("fluss"); |
| 116 | + log_range.table_format_params.__set_fluss_params({{"fluss.range_type", "LOG"}}); |
| 117 | + TFileScanRange file_scan_range; |
| 118 | + file_scan_range.__set_params(scan_level_format("fluss")); |
| 119 | + // A query reads no source tuple; tuple 0 there would make this scan a load. |
| 120 | + file_scan_range.params.__set_src_tuple_id(1); |
| 121 | + file_scan_range.ranges.push_back(log_range); |
| 122 | + TScanRangeParams scan_range; |
| 123 | + scan_range.scan_range.ext_scan_range.__set_file_scan_range(file_scan_range); |
| 124 | + scan_range.scan_range.__isset.ext_scan_range = true; |
| 125 | + const std::vector<TScanRangeParams> scan_ranges {scan_range}; |
| 126 | + |
| 127 | + RuntimeProfile profile("EveryInstanceOfTheNodeReadsThroughTheNodesCache"); |
| 128 | + const std::map<int, std::pair<std::shared_ptr<BasicSharedState>, |
| 129 | + std::vector<std::shared_ptr<Dependency>>>> |
| 130 | + shared_state_map; |
| 131 | + for (int instance = 0; instance < 2; ++instance) { |
| 132 | + SCOPED_TRACE(instance); |
| 133 | + auto local_state = FileScanLocalState::create_unique(&state, &node); |
| 134 | + LocalStateInfo info {.parent_profile = &profile, |
| 135 | + .scan_ranges = scan_ranges, |
| 136 | + .shared_state = nullptr, |
| 137 | + .shared_state_map = shared_state_map, |
| 138 | + .task_idx = instance}; |
| 139 | + ASSERT_TRUE(local_state->init(&state, info).ok()); |
| 140 | + ASSERT_NO_FATAL_FAILURE(expect_scanners_share_node_cache(local_state.get(), &state, &node)); |
| 141 | + } |
| 142 | +} |
| 143 | + |
| 144 | +// An unlimited scanner setting can overflow an int when multiplied by the instance count. |
| 145 | +// Shard count must instead be capped by remote scan threads, preserving concurrency across instances. |
| 146 | +TEST(FileScanOperatorCacheTest, CacheShardsOfAnUnlimitedScannerSettingAreCappedNotOverflowed) { |
| 147 | + ObjectPool pool; |
| 148 | + DescriptorTbl* descriptors = nullptr; |
| 149 | + ASSERT_TRUE(create_one_tuple_descriptors(&pool, &descriptors).ok()); |
| 150 | + MockRuntimeState state; |
| 151 | + state.set_desc_tbl(descriptors); |
| 152 | + state._query_options.__set_parallel_instance(2); |
| 153 | + state._query_options.__set_max_file_scanners_concurrency(std::numeric_limits<int32_t>::max()); |
| 154 | + |
| 155 | + const TPlanNode plan_node = file_scan_plan_node(); |
| 156 | + FileScanOperatorX node(&pool, plan_node, 0, *descriptors, /*parallel_tasks=*/2); |
| 157 | + ASSERT_TRUE(node.init(plan_node, &state).ok()); |
| 158 | + ASSERT_TRUE(node.prepare(&state).ok()); |
| 159 | + EXPECT_EQ(node._kv_cache->_num_shards, |
| 160 | + static_cast<uint32_t>(ScannerScheduler::default_remote_scan_thread_num())); |
| 161 | +} |
| 162 | + |
| 163 | +} // namespace doris::pipeline |
0 commit comments