Skip to content

Commit 5dcf106

Browse files
authored
[opt](lance) Support distributed execution for vector_search (#66841)
### What problem does this PR solve? support two phase in distributed execution for lance vector_search. 1. lance_scan_node will query nearest with query_vector and prefilter in lance rust, will return _row_id, _distance and some postfilter column from lance scan result, and execute the where postfilter. 2. then local topn and global merge topn will get top_k rows. 3. MaterializeNode will use _row_id to fetch the payload project column from lance rust. ``` mysql> desc SELECT row_id, category, label, _distance FROM vector_search( "table"="test_lance_vector_search_two_phase.doris.vs_ivf_pq_f32", "column"="embedding", "query_vector"="[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]", "top_k"="3", "metric"="l2", "nprobes"="4", "refine_factor"="10", "use_index"="true") WHERE category = 'odd' ORDER BY _distance; +-----------------------------------------------------------------------------------------------------------------------------------------------------------+ | Explain String(Nereids Planner) | +-----------------------------------------------------------------------------------------------------------------------------------------------------------+ | PLAN FRAGMENT 0 | | OUTPUT EXPRS: | | row_id[#17] | | category[#18] | | label[#19] | | _distance[#20] | | PARTITION: UNPARTITIONED | | | | HAS_COLO_PLAN_NODE: false | | | | VRESULT SINK | | MYSQL_PROTOCOL | | | | 4:VSORT(193) | | | order by: _distance[#20] ASC | | | algorithm: full sort | | | offset: 0 | | | distribute expr lists: | | | | | 3:VMaterializeNode | | | materialize tuple id:3 | | | output tuple id:4 | | | projectList:[row_id, category, label, _distance] | | | column_descs_lists[[`row_id` bigint NOT NULL, `label` text NOT NULL]] | | | locations: [[2, 3]] | | | column_idxs_lists: [[0, 2]] | | | row_ids: [__DORIS_GLOBAL_ROWID_COL__vector_search] | | | isTopMaterializeNode: true | | | final projections: row_id[#11], category[#9], label[#12], _distance[#10] | | | final project output tuple id: 4 | | | | | 2:VMERGING-EXCHANGE | | offset: 0 | | limit: 3 | | distribute expr lists: | | | | PLAN FRAGMENT 1 | | | | PARTITION: RANDOM | | | | HAS_COLO_PLAN_NODE: false | | | | STREAM DATA SINK | | EXCHANGE ID: 02 | | UNPARTITIONED | | IS_MERGE: true | | | | 1:VTOP-N(182) | | | order by: _distance[#7] ASC | | | algorithm: heap sort | | | local merge sort | | | merge by exchange | | | offset: 0 | | | limit: 3 | | | distribute expr lists: | | | | | 0:VLANCE_SCAN_NODE(174) | | table: null.null.VectorSearchTableValuedFunction<test_lance_vector_search_two_phase.doris.vs_ivf_pq_f32> | | predicates: (_tvf_vector_search.category[#0] = 'odd') | | inputSplitNum=2, totalFileSize=0, scanRanges=2 | | partition=0/0 | | numNodes=1 | | pushdown agg=NONE | | externalSearchType=VECTOR | | lanceVectorColumn=embedding | | lanceTopK=3 | | lanceOffset=0 | | lanceMetric=l2 | | lanceVersion=3 | | lanceSearchFragments=2 | | final projections: _tvf_vector_search.category[#0], _tvf_vector_search._distance[#1], _tvf_vector_search.__DORIS_GLOBAL_ROWID_COL__vector_search[#2] | | final project output tuple id: 1 | | | | | | | | ========== STATISTICS ========== | +-----------------------------------------------------------------------------------------------------------------------------------------------------------+ 76 rows in set (0.04 sec) mysql> ``` ### Release note None ### Check List (For Author) - Test <!-- At least one of them must be included. --> - [x] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason <!-- Add your reason? --> - Behavior changed: - [ ] No. - [ ] Yes. <!-- Explain the behavior change --> - Does this need documentation? - [ ] No. - [ ] Yes. <!-- Add document PR link here. eg: apache/doris-website#1214 --> ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label <!-- Add branch pick label that this PR should merge into -->
1 parent f77661a commit 5dcf106

45 files changed

Lines changed: 3126 additions & 404 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎be/src/exec/operator/materialization_opertor.cpp‎

Lines changed: 68 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
#include <fmt/format.h>
2222
#include <gen_cpp/internal_service.pb.h>
2323

24+
#include <cstring>
2425
#include <set>
2526
#include <sstream>
2627
#include <utility>
@@ -66,6 +67,57 @@ constexpr const char* TOPN_LAZY_MAT_PHASE2_PER_BACKEND_REMOTE_IO_TIME =
6667
constexpr const char* TOPN_LAZY_MAT_PHASE2_PER_BACKEND_WRITE_CACHE_IO_TIME =
6768
"TopNLazyMaterializationSecondPhasePerBackendWriteCacheIOTime";
6869

70+
struct MaterializationRowLocation {
71+
ROW_VERSION version = ROW_VERSION::FILE_LOCAL_ROW_ID;
72+
int64_t backend_id = 0;
73+
uint32_t file_id = 0;
74+
uint64_t row_id = 0;
75+
};
76+
77+
Status decode_global_row_location_v2(const StringRef& encoded, ROW_VERSION version,
78+
MaterializationRowLocation* decoded) {
79+
if (encoded.size != sizeof(GlobalRowLoacationV2)) {
80+
return Status::InternalError(
81+
"invalid global row location size for version {}: actual={}, expected={}",
82+
static_cast<int>(version), encoded.size, sizeof(GlobalRowLoacationV2));
83+
}
84+
GlobalRowLoacationV2 location(GlobalRowLoacationV2::VERSION, 0, 0, 0);
85+
std::memcpy(&location, encoded.data, sizeof(location));
86+
decoded->version = version;
87+
decoded->backend_id = location.backend_id;
88+
switch (version) {
89+
case ROW_VERSION::FILE_LOCAL_ROW_ID:
90+
decoded->file_id = location.file_local.file_id;
91+
decoded->row_id = location.file_local.row_id;
92+
return Status::OK();
93+
case ROW_VERSION::LANCE_DATASET_ROW_ID:
94+
decoded->file_id = location.lance_file_id;
95+
decoded->row_id = location.lance_row_id;
96+
return Status::OK();
97+
}
98+
return Status::NotSupported("unsupported V2 global row location version: {}",
99+
static_cast<int>(version));
100+
}
101+
102+
Status decode_materialization_row_location(const StringRef& encoded,
103+
MaterializationRowLocation* decoded) {
104+
if (encoded.size < sizeof(uint8_t)) {
105+
return Status::InternalError("global row location is empty");
106+
}
107+
108+
const auto version_value = static_cast<uint8_t>(encoded.data[0]);
109+
const auto version = static_cast<ROW_VERSION>(version_value);
110+
// Keep size validation inside each version-family decoder. A future version can therefore
111+
// use a different encoded size without being rejected by the current 24-byte V2 contract.
112+
switch (version) {
113+
case ROW_VERSION::FILE_LOCAL_ROW_ID:
114+
case ROW_VERSION::LANCE_DATASET_ROW_ID:
115+
return decode_global_row_location_v2(encoded, version, decoded);
116+
}
117+
return Status::NotSupported("unsupported global row location version: {}, encoded_size={}",
118+
static_cast<int>(version_value), encoded.size);
119+
}
120+
69121
void update_counter(RuntimeProfile* profile, const std::string& name, TUnit::type unit,
70122
int64_t value) {
71123
COUNTER_UPDATE(ADD_COUNTER_WITH_LEVEL(profile, name, unit, 2), value);
@@ -446,19 +498,28 @@ Status MaterializationSharedState::create_muiltget_result(const Columns& columns
446498

447499
for (int j = 0; j < rows; ++j) {
448500
if (!null_map || !null_map[j]) {
449-
DCHECK(column_rowid->get_data_at(j).size == sizeof(GlobalRowLoacationV2));
450-
GlobalRowLoacationV2 row_location =
451-
*((GlobalRowLoacationV2*)column_rowid->get_data_at(j).data);
501+
MaterializationRowLocation row_location;
502+
RETURN_IF_ERROR(decode_materialization_row_location(column_rowid->get_data_at(j),
503+
&row_location));
452504
auto rpc_struct = rpc_struct_map.find(row_location.backend_id);
453505
if (UNLIKELY(rpc_struct == rpc_struct_map.end())) {
454506
return Status::InternalError(
455507
"MaterializationSinkOperatorX failed to find rpc_struct, backend_id={}",
456508
row_location.backend_id);
457509
}
458-
rpc_struct->second.request.mutable_request_block_descs(i)->add_row_id(
459-
row_location.row_id);
460-
rpc_struct->second.request.mutable_request_block_descs(i)->add_file_id(
461-
row_location.file_id);
510+
auto* request_block_desc =
511+
rpc_struct->second.request.mutable_request_block_descs(i);
512+
const auto row_location_version = static_cast<uint32_t>(row_location.version);
513+
if (request_block_desc->row_id_size() == 0) {
514+
request_block_desc->set_row_location_version(row_location_version);
515+
} else if (request_block_desc->row_location_version() != row_location_version) {
516+
return Status::InternalError(
517+
"mixed row location versions in one materialization request: "
518+
"actual={}, expected={}",
519+
row_location_version, request_block_desc->row_location_version());
520+
}
521+
request_block_desc->add_row_id(row_location.row_id);
522+
request_block_desc->add_file_id(row_location.file_id);
462523
block_order[j] = row_location.backend_id;
463524

464525
// Count rows per backend

0 commit comments

Comments
 (0)