Skip to content

Commit 5b793d8

Browse files
committed
[improvement](lance) Expose query parallelism and complete row fetch profiles
1 parent 78aa9c9 commit 5b793d8

13 files changed

Lines changed: 247 additions & 4 deletions

File tree

‎be/src/exec/operator/materialization_opertor.cpp‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -481,6 +481,9 @@ void MaterializationSharedState::_update_profile_info(int64_t backend_id,
481481
update_profile_info_key(RowIdStorageReader::LanceRowIdTakeReadTimeProfile, false);
482482
update_profile_info_key(RowIdStorageReader::LanceArrowToDorisBlockTimeProfile, false);
483483
update_profile_info_key(RowIdStorageReader::LanceRowIdFetchTotalTimeProfile, false);
484+
for (const auto& [name, unit] : RowIdStorageReader::LanceFetchCountersProfile) {
485+
update_profile_info_key(name, false);
486+
}
484487
}
485488

486489
Status MaterializationSharedState::create_muiltget_result(const Columns& columns, bool child_eos,
@@ -674,7 +677,8 @@ Status MaterializationOperator::pull(RuntimeState* state, Block* output_block, b
674677

675678
Status MaterializationOperator::push(RuntimeState* state, Block* in_block, bool eos) const {
676679
auto& local_state = get_local_state(state);
677-
SCOPED_TIMER(local_state.exec_time_counter());
680+
// StatefulOperatorX::get_block_impl already times push() with this counter.
681+
// Nesting the same timer counts the synchronous row-fetch RPC wait twice.
678682
if (!local_state._materialization_state.rpc_struct_inited) {
679683
RETURN_IF_ERROR(local_state._materialization_state.init_multi_requests(
680684
_materialization_node, state));

‎be/src/exec/rowid_fetcher.cpp‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -755,6 +755,12 @@ const std::string RowIdStorageReader::LanceRowIdTakeReadTimeProfile = "LanceRowI
755755
const std::string RowIdStorageReader::LanceArrowToDorisBlockTimeProfile =
756756
"LanceArrowToDorisBlockTime";
757757
const std::string RowIdStorageReader::LanceRowIdFetchTotalTimeProfile = "LanceRowIdFetchTotalTime";
758+
const std::map<std::string, TUnit::type> RowIdStorageReader::LanceFetchCountersProfile = {
759+
{"LanceRowIdFetchRows", TUnit::UNIT},
760+
{"LanceRowIdFetchCalls", TUnit::UNIT},
761+
{"LanceDataCacheBytesReadFromCache", TUnit::BYTES},
762+
{"LanceDataCacheBytesReadFromRemote", TUnit::BYTES},
763+
};
758764
const std::string RowIdStorageReader::TopNLazyMaterializationSecondPhaseLocalIOCount =
759765
"TopNLazyMaterializationSecondPhaseLocalIOCount";
760766
const std::string RowIdStorageReader::TopNLazyMaterializationSecondPhaseLocalIOBytes =
@@ -889,6 +895,13 @@ Status RowIdStorageReader::read_lance_rows_by_row_ids(
889895
collect_lance_fetch_time(LanceRowIdTakeReadTimeProfile);
890896
collect_lance_fetch_time(LanceArrowToDorisBlockTimeProfile);
891897
collect_lance_fetch_time(LanceRowIdFetchTotalTimeProfile);
898+
// close() publishes dataset-handle cache statistics; collect after it and preserve the
899+
// units across the RPC instead of formatting byte/count values as nanoseconds.
900+
for (const auto& [name, unit] : LanceFetchCountersProfile) {
901+
if (const auto* counter = runtime_profile->get_counter(name); counter != nullptr) {
902+
fetch_statistics->lance_fetch_counters.emplace(name, counter->value());
903+
}
904+
}
892905
return Status::OK();
893906
}
894907

@@ -1203,6 +1216,7 @@ Status RowIdStorageReader::read_batch_external_row(
12031216
format_to(file_read_times_buffer, "[");
12041217

12051218
std::map<std::string, int64_t> lance_fetch_times_ns;
1219+
std::map<std::string, int64_t> lance_fetch_counters;
12061220
size_t idx = 0;
12071221
for (const auto& [_, scan_info] : scan_rows) {
12081222
format_to(file_read_lines_buffer, "{}, ", scan_info.first.size());
@@ -1213,6 +1227,9 @@ Status RowIdStorageReader::read_batch_external_row(
12131227
for (const auto& [time_name, time_value] : fetch_statistics[idx].lance_fetch_times_ns) {
12141228
lance_fetch_times_ns[time_name] += time_value;
12151229
}
1230+
for (const auto& [name, value] : fetch_statistics[idx].lance_fetch_counters) {
1231+
lance_fetch_counters[name] += value;
1232+
}
12161233
idx++;
12171234
}
12181235

@@ -1232,6 +1249,10 @@ Status RowIdStorageReader::read_batch_external_row(
12321249
fmt::to_string(file_read_bytes_buffer));
12331250
runtime_profile->add_info_string(FileScannerV2::FileReadTimeProfile,
12341251
fmt::to_string(file_read_times_buffer));
1252+
for (const auto& [name, value] : lance_fetch_counters) {
1253+
runtime_profile->add_info_string(
1254+
name, PrettyPrinter::print(value, LanceFetchCountersProfile.at(name)));
1255+
}
12351256
for (const auto& [time_name, time_value] : lance_fetch_times_ns) {
12361257
runtime_profile->add_info_string(time_name,
12371258
PrettyPrinter::print(time_value, TUnit::TIME_NS));

‎be/src/exec/rowid_fetcher.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,7 @@ class RowIdStorageReader {
108108
static const std::string LanceRowIdTakeReadTimeProfile;
109109
static const std::string LanceArrowToDorisBlockTimeProfile;
110110
static const std::string LanceRowIdFetchTotalTimeProfile;
111+
static const std::map<std::string, TUnit::type> LanceFetchCountersProfile;
111112
static const std::string TopNLazyMaterializationSecondPhaseLocalIOCount;
112113
static const std::string TopNLazyMaterializationSecondPhaseLocalIOBytes;
113114
static const std::string TopNLazyMaterializationSecondPhaseRemoteIOCount;
@@ -183,6 +184,7 @@ class RowIdStorageReader {
183184
int64_t init_reader_ms = 0;
184185
int64_t get_block_ms = 0;
185186
std::map<std::string, int64_t> lance_fetch_times_ns;
187+
std::map<std::string, int64_t> lance_fetch_counters;
186188
std::string file_read_bytes;
187189
std::string file_read_times;
188190
};

‎be/src/format_v2/table/lance_reader.cpp‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -255,6 +255,10 @@ Status LanceTableReader::read_by_row_ids(const TFileRangeDesc& range,
255255
_row_id_fetch_total_time = ADD_CHILD_TIMER_WITH_LEVEL(
256256
_scanner_profile, "LanceRowIdFetchTotalTime", LANCE_READER_PROFILE, 1);
257257
}
258+
auto* fetch_calls = ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, "LanceRowIdFetchCalls",
259+
TUnit::UNIT, LANCE_READER_PROFILE, 1);
260+
auto* fetch_rows = ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, "LanceRowIdFetchRows",
261+
TUnit::UNIT, LANCE_READER_PROFILE, 1);
258262
SCOPED_TIMER(_row_id_fetch_total_time);
259263

260264
// Phase-two row fetch does not execute FTS, so a reader created only for take_rows must not
@@ -271,6 +275,7 @@ Status LanceTableReader::read_by_row_ids(const TFileRangeDesc& range,
271275
int32_t take_rows_status = 0;
272276
{
273277
SCOPED_TIMER(_row_id_take_read_time);
278+
COUNTER_UPDATE(fetch_calls, 1);
274279
take_rows_status = lance_dataset_take_rows(_dataset, row_ids.data(), row_ids.size(),
275280
columns.data(), &stream);
276281
}
@@ -313,6 +318,7 @@ Status LanceTableReader::read_by_row_ids(const TFileRangeDesc& range,
313318
record_batch, block, _global_rowid_context, &rows));
314319
}
315320
fetched_rows += rows;
321+
COUNTER_UPDATE(fetch_rows, rows);
316322
}
317323
if (fetched_rows != row_ids.size()) {
318324
return Status::InternalError("Lance row-id fetch returned {} rows for {} requested row ids",
@@ -541,6 +547,9 @@ Status LanceTableReader::_validate_external_search_request() const {
541547

542548
if (_search_kind == SearchKind::VECTOR && request.__isset.vector_search_options) {
543549
const auto& options = request.vector_search_options;
550+
if (options.__isset.query_parallelism && options.query_parallelism < -1) {
551+
return Status::InvalidArgument("Lance query_parallelism must be -1, 0, or positive");
552+
}
544553
if (options.__isset.nprobes && options.nprobes <= 0) {
545554
return Status::InvalidArgument("Lance nprobes must be positive");
546555
}
@@ -1076,6 +1085,10 @@ Status LanceTableReader::_configure_vector_search(LanceScanner* scanner,
10761085
}
10771086
if (request.__isset.vector_search_options) {
10781087
const auto& options = request.vector_search_options;
1088+
if (options.__isset.query_parallelism &&
1089+
lance_scanner_set_query_parallelism(scanner, options.query_parallelism) != 0) {
1090+
return lance_error("set Lance vector query parallelism");
1091+
}
10791092
if (options.__isset.nprobes &&
10801093
lance_scanner_set_nprobes(scanner, static_cast<uint32_t>(options.nprobes)) != 0) {
10811094
return lance_error("set Lance vector nprobes");

‎be/test/exec/operator/materialization_shared_state_test.cpp‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
#include "exec/operator/materialization_opertor.h"
2828
#include "exec/pipeline/dependency.h"
2929
#include "runtime/runtime_profile.h"
30+
#include "testutil/mock/mock_runtime_state.h"
3031

3132
namespace doris {
3233

@@ -43,6 +44,10 @@ void set_lance_fetch_profile(PMultiGetBlockV2* response_block, int64_t scale) {
4344
profile.add_info_string("LanceRowIdTakeReadTime", std::to_string(2 * scale) + "ns");
4445
profile.add_info_string("LanceArrowToDorisBlockTime", std::to_string(3 * scale) + "ns");
4546
profile.add_info_string("LanceRowIdFetchTotalTime", std::to_string(4 * scale) + "ns");
47+
profile.add_info_string("LanceRowIdFetchRows", std::to_string(5 * scale));
48+
profile.add_info_string("LanceRowIdFetchCalls", "1");
49+
profile.add_info_string("LanceDataCacheBytesReadFromCache", "64.00 KB");
50+
profile.add_info_string("LanceDataCacheBytesReadFromRemote", "32.00 KB");
4651
profile.add_info_string("ScannersRunningTime", "0ms");
4752
profile.add_info_string("InitReaderAvgTime", "0ms");
4853
profile.add_info_string("GetBlockAvgTime", "0ms");
@@ -54,6 +59,34 @@ void set_lance_fetch_profile(PMultiGetBlockV2* response_block, int64_t scale) {
5459

5560
} // namespace
5661

62+
TEST(MaterializationOperatorTimingTest, PushDoesNotDuplicateFrameworkExecTimer) {
63+
ObjectPool pool;
64+
MockRuntimeState state;
65+
TPlanNode node;
66+
node.__set_node_id(0);
67+
node.__set_node_type(TPlanNodeType::MATERIALIZATION_NODE);
68+
MaterializationOperator op(&pool, node, 0, state.desc_tbl());
69+
RuntimeProfile profile("materialization_timing");
70+
auto local = MaterializationLocalState::create_unique(&state, &op);
71+
LocalStateInfo info {.parent_profile = &profile,
72+
.scan_ranges = {},
73+
.shared_state = nullptr,
74+
.shared_state_map = {},
75+
.task_idx = 0};
76+
ASSERT_TRUE(local->init(&state, info).ok());
77+
local->_materialization_state.rpc_struct_inited = true;
78+
auto* timer = local->exec_time_counter();
79+
state.resize_op_id_to_local_state(-1);
80+
state.emplace_local_state(0, std::move(local));
81+
Block block;
82+
const auto before = timer->value();
83+
// The framework owns ExecTime. Direct push calls must not contribute a second sample.
84+
for (int i = 0; i < 100; ++i) {
85+
ASSERT_TRUE(op.push(&state, &block, false).ok());
86+
}
87+
EXPECT_EQ(before, timer->value());
88+
}
89+
5790
class MaterializationSharedStateTest : public testing::Test {
5891
protected:
5992
void SetUp() override {
@@ -316,6 +349,10 @@ TEST_F(MaterializationSharedStateTest, TestMergeMultiResponse) {
316349
EXPECT_EQ("20ns, ", fmt::to_string(backend1_info.at("LanceRowIdTakeReadTime")));
317350
EXPECT_EQ("30ns, ", fmt::to_string(backend1_info.at("LanceArrowToDorisBlockTime")));
318351
EXPECT_EQ("40ns, ", fmt::to_string(backend1_info.at("LanceRowIdFetchTotalTime")));
352+
EXPECT_EQ("50, ", fmt::to_string(backend1_info.at("LanceRowIdFetchRows")));
353+
EXPECT_EQ("1, ", fmt::to_string(backend1_info.at("LanceRowIdFetchCalls")));
354+
EXPECT_EQ("64.00 KB, ", fmt::to_string(backend1_info.at("LanceDataCacheBytesReadFromCache")));
355+
EXPECT_EQ("32.00 KB, ", fmt::to_string(backend1_info.at("LanceDataCacheBytesReadFromRemote")));
319356
const auto& backend2_info = _shared_state->backend_profile_info_string.at(_backend_id2);
320357
EXPECT_EQ("1ns, ", fmt::to_string(backend2_info.at("LanceDatasetOpenTime")));
321358
EXPECT_EQ("2ns, ", fmt::to_string(backend2_info.at("LanceRowIdTakeReadTime")));

‎be/test/format_v2/table/lance_reader_test.cpp‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1226,6 +1226,47 @@ TEST(LanceTableReaderVectorSearchTest, MultiVectorRejectsActualNullAndNonFiniteE
12261226
}
12271227
}
12281228

1229+
TEST(LanceTableReaderVectorSearchTest, RejectsInvalidQueryParallelismBeforeOpeningDataset) {
1230+
TQueryGlobals globals;
1231+
RuntimeState state(globals);
1232+
RuntimeProfile profile("lance_invalid_query_parallelism");
1233+
auto params = make_float32_vector_search_params({0.0F, 0.0F, 0.0F}, 2, 0);
1234+
params.lance_scan_params.external_search_request.vector_search_options.__set_query_parallelism(
1235+
-2);
1236+
const Columns columns {projected_column("_distance", TYPE_FLOAT, true)};
1237+
LanceTableReader reader;
1238+
const auto status = init_reader(&reader, columns, &state, &profile, &params);
1239+
EXPECT_FALSE(status.ok());
1240+
EXPECT_NE(std::string::npos, status.to_string().find("query_parallelism"));
1241+
}
1242+
1243+
TEST(LanceTableReaderVectorSearchTest, QueryParallelismPreservesResults) {
1244+
const std::filesystem::path uri = "./be/test/format_v2/table/lance/data/all_types.lance";
1245+
LanceFixtureInfo fixture;
1246+
ASSERT_TRUE(get_fixture_info(uri, &fixture).ok());
1247+
const Columns columns {projected_column("row_id", TYPE_BIGINT, false),
1248+
projected_column("_distance", TYPE_FLOAT, true)};
1249+
TQueryGlobals globals;
1250+
RuntimeState state(globals);
1251+
for (const int parallelism : {-1, 0, 1, 4}) {
1252+
SCOPED_TRACE(parallelism);
1253+
auto params = make_float32_vector_search_params({0.0F, 0.0F, 0.0F}, 2, 1);
1254+
params.lance_scan_params.external_search_request.vector_search_options
1255+
.__set_query_parallelism(parallelism);
1256+
RuntimeProfile profile("lance_query_parallelism");
1257+
LanceTableReader reader;
1258+
ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &params).ok());
1259+
ASSERT_TRUE(prepare_fixture(&reader, uri, fixture, fixture.fragment_ids).ok());
1260+
Block block;
1261+
add_output_columns(&block, columns);
1262+
const auto rows = read_vector_search_rows(&reader, &block);
1263+
ASSERT_EQ(2U, rows.size());
1264+
EXPECT_EQ(2, rows[0].first);
1265+
EXPECT_EQ(4, rows[1].first);
1266+
ASSERT_TRUE(reader.close().ok());
1267+
}
1268+
}
1269+
12291270
TEST(LanceTableReaderVectorSearchTest, SearchesWholeSnapshotWithOffsetAndDistance) {
12301271
const std::filesystem::path dataset_uri =
12311272
"./be/test/format_v2/table/lance/data/all_types.lance";
@@ -1431,6 +1472,25 @@ TEST(LanceTableReaderVectorSearchTest, ReturnsStableGlobalRowIdsAndFetchesPayloa
14311472
EXPECT_EQ("extra", label_values.get_data_at(0).to_string());
14321473
EXPECT_EQ("unit-x", label_values.get_data_at(1).to_string());
14331474
EXPECT_EQ("extra", label_values.get_data_at(2).to_string());
1475+
ASSERT_NE(nullptr, fetch_profile.get_counter("LanceRowIdFetchRows"));
1476+
EXPECT_EQ(3, fetch_profile.get_counter("LanceRowIdFetchRows")->value());
1477+
ASSERT_NE(nullptr, fetch_profile.get_counter("LanceRowIdFetchCalls"));
1478+
EXPECT_EQ(1, fetch_profile.get_counter("LanceRowIdFetchCalls")->value());
1479+
Block second_block;
1480+
add_output_columns(&second_block, payload_columns);
1481+
ASSERT_TRUE(payload_reader
1482+
.read_by_row_ids(make_lance_range(dataset_uri, fixture.version,
1483+
fixture.fragment_ids),
1484+
fetch_row_ids, &second_block)
1485+
.ok());
1486+
EXPECT_EQ(6, fetch_profile.get_counter("LanceRowIdFetchRows")->value());
1487+
EXPECT_EQ(2, fetch_profile.get_counter("LanceRowIdFetchCalls")->value());
1488+
ASSERT_TRUE(payload_reader
1489+
.read_by_row_ids(make_lance_range(dataset_uri, fixture.version,
1490+
fixture.fragment_ids),
1491+
{}, &second_block)
1492+
.ok());
1493+
EXPECT_EQ(2, fetch_profile.get_counter("LanceRowIdFetchCalls")->value());
14341494
EXPECT_TRUE(payload_reader.close().ok());
14351495
}
14361496

‎docs/lance-ann-profile.md‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,3 +65,39 @@ queries, inspect partition load together with execution bytes, requests, and
6565
partition cache misses. Use repeated queries and the operator-level elapsed
6666
times to assess latency; cumulative parallel stage times alone are not a critical
6767
path trace.
68+
69+
## Query parallelism
70+
71+
`vector_search` accepts an optional `"query_parallelism"` integer:
72+
73+
- `0` (also the default when omitted): let Lance choose the parallelism.
74+
- `-1`: use the available Lance CPU parallelism.
75+
- Positive values: request that many concurrent partition searches, capped by
76+
Lance's compute pool and execution-plan limits.
77+
78+
For example, add `"query_parallelism" = "4"` alongside `"nprobes" = "64"`.
79+
This controls concurrency inside a Lance search, independently of Doris scan
80+
instances. Increasing it can increase intermediate candidates and memory usage;
81+
measure both single-query latency and concurrent throughput. EXPLAIN displays an
82+
explicit setting as `lanceQueryParallelism`.
83+
84+
## Second-phase row-ID fetch
85+
86+
Each `RowIDFetcher: BackendId:...` profile also reports:
87+
88+
| Counter | Scope |
89+
| --- | --- |
90+
| `LanceRowIdFetchCalls` | Non-empty dataset `take_rows` calls. |
91+
| `LanceRowIdFetchRows` | Rows converted from successful returned batches, including duplicate requested row IDs. |
92+
| `LanceDataCacheBytesReadFromCache` | Logical data-file bytes served by Foyer for the dataset handles used by this fetch. |
93+
| `LanceDataCacheBytesReadFromRemote` | Logical data-file bytes served through the Foyer origin path for those handles. |
94+
95+
Byte counts are collected after closing the reader and summed across dataset
96+
handles in the fetch RPC. They exclude block-alignment amplification, metadata,
97+
and index reads. With Foyer disabled or for paths outside its data-file wrapper,
98+
zero values do not imply zero physical IO. The current take API does not expose
99+
physical request counts or a separate decode timer; scanner-plan IO counters
100+
must not be substituted for them.
101+
102+
`MATERIALIZATION_OPERATOR.ExecTime` includes its synchronous fetch wait once.
103+
`MaxRpcTime` is nested within that execution time, not an additional duration.

‎fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanNode.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -354,6 +354,11 @@ public String getNodeExplainString(String prefix, TExplainLevel detailLevel) {
354354
.append(scanPlan.vectorIndexStatus).append("\n");
355355
result.append(prefix).append("lanceVectorColumn=")
356356
.append(vector.getColumn()).append("\n");
357+
if (externalSearchRequest.isSetVectorSearchOptions()
358+
&& externalSearchRequest.getVectorSearchOptions().isSetQueryParallelism()) {
359+
result.append(prefix).append("lanceQueryParallelism=")
360+
.append(externalSearchRequest.getVectorSearchOptions().getQueryParallelism()).append("\n");
361+
}
357362
result.append(prefix).append("lanceMetric=")
358363
.append(vector.isSetMetric()
359364
? VectorSearchTableValuedFunction.metricName(vector.getMetric()) : "default")

0 commit comments

Comments
 (0)