Skip to content

Commit cd8255c

Browse files
committed
[Fix](lance) Correct multi-vector scoring before TopK
1 parent f1d9951 commit cd8255c

75 files changed

Lines changed: 1106 additions & 58 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎be/src/format_v2/table/lance_reader.cpp‎

Lines changed: 25 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,6 @@
3434
#include "common/logging.h"
3535
#include "core/column/column_nullable.h"
3636
#include "core/column/column_string.h"
37-
#include "core/column/column_vector.h"
3837
#include "exec/common/endian.h"
3938
#include "format_v2/lance/lance_reader_helper.h"
4039
#include "format_v2/lance/lance_runtime_filter_helper.h"
@@ -570,6 +569,22 @@ Status LanceTableReader::_validate_external_search_request() const {
570569
return Status::InvalidArgument(
571570
"Lance vector search top_k + offset exceeds uint32 range");
572571
}
572+
// Match FE/C API limits before constructing the per-subvector ANN plan branches.
573+
constexpr int MAX_QUERY_VECTORS = 128;
574+
constexpr int64_t MAX_QUERY_VECTOR_CANDIDATES = 100000;
575+
const auto refine_factor =
576+
request.__isset.vector_search_options &&
577+
request.vector_search_options.__isset.refine_factor
578+
? request.vector_search_options.refine_factor
579+
: 1;
580+
if (multi_vector &&
581+
(query_vector.num_vectors > MAX_QUERY_VECTORS || refine_factor <= 0 ||
582+
vector.top_k + vector.offset > MAX_QUERY_VECTOR_CANDIDATES / refine_factor ||
583+
query_vector.num_vectors >
584+
MAX_QUERY_VECTOR_CANDIDATES / (vector.top_k + vector.offset))) {
585+
return Status::InvalidArgument(
586+
"multi-vector query exceeds 128 subvectors or 100000 subvector-candidates");
587+
}
573588
} else {
574589
DORIS_CHECK(_search_kind == SearchKind::FULL_TEXT);
575590
if (request.schema_version != 1) {
@@ -956,9 +971,14 @@ Status LanceTableReader::_configure_vector_search(LanceScanner* scanner,
956971
static_cast<int>(query.element_type));
957972
}
958973

959-
if (vector.__isset.metric && vector.metric != TVectorMetric::DEFAULT) {
974+
{
975+
// FE plans an omitted metric as L2; never let indexed splits choose another default.
976+
const auto requested_metric =
977+
!vector.__isset.metric || vector.metric == TVectorMetric::DEFAULT
978+
? TVectorMetric::L2
979+
: vector.metric;
960980
LanceMetricType metric;
961-
switch (vector.metric) {
981+
switch (requested_metric) {
962982
case TVectorMetric::L2:
963983
metric = LANCE_METRIC_L2;
964984
break;
@@ -980,8 +1000,8 @@ Status LanceTableReader::_configure_vector_search(LanceScanner* scanner,
9801000
}
9811001
}
9821002

983-
// The pinned Lance ANN scorer and flat scorer differ by a query-count constant.
984-
// Exact refinement makes indexed and unindexed candidates use the same row-level score.
1003+
// Exact refinement validates stored elements and gives indexed and unindexed candidates
1004+
// the same row-level score before either path truncates its results.
9851005
if (query.__isset.num_vectors && lance_scanner_set_refine_factor(scanner, 1) != 0) {
9861006
return lance_error("enable Lance multi-vector refinement");
9871007
}
@@ -1265,26 +1285,6 @@ Status LanceTableReader::_fill_block_from_record_batch(
12651285
->read_column_from_arrow(*columns[output_idx],
12661286
normalized_column.get(), 0, row_count,
12671287
_ctz));
1268-
if (_search_kind == SearchKind::VECTOR && field->name() == LANCE_DISTANCE_COLUMN) {
1269-
const auto& query = _scan_params->lance_scan_params.external_search_request
1270-
.search_query.vector_search.query_vector;
1271-
if (query.__isset.num_vectors) {
1272-
// Lance's exact score is 1 - sum(1 - distance). Expose sum(min distance)
1273-
// so one logical row has the same scoring contract for every query size.
1274-
IColumn* output = columns[output_idx].get();
1275-
if (auto* nullable = check_and_get_column<ColumnNullable>(*output)) {
1276-
output = &nullable->get_nested_column();
1277-
}
1278-
auto* distances = check_and_get_column<ColumnFloat32>(*output);
1279-
if (distances == nullptr) {
1280-
return Status::InternalError("Lance multi-vector distance must be FLOAT");
1281-
}
1282-
auto& data = distances->get_data();
1283-
for (size_t i = data.size() - row_count; i < data.size(); ++i) {
1284-
data[i] += static_cast<float>(query.num_vectors - 1);
1285-
}
1286-
}
1287-
}
12881288
} catch (const Exception& e) {
12891289
return Status::InternalError("convert Lance Arrow column '{}' failed: {}",
12901290
field->name(), e.what());

docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/multivector.lance/_indices/a560507e-4082-48c6-9193-e33a7733cbbc/auxiliary.idx renamed to be/test/format_v2/table/lance/data/multivector.lance/_indices/28f6d79f-9c00-4135-ba4b-fc8de0b635f3/auxiliary.idx

833 Bytes
Binary file not shown.

docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/multivector.lance/_indices/a560507e-4082-48c6-9193-e33a7733cbbc/index.idx renamed to be/test/format_v2/table/lance/data/multivector.lance/_indices/28f6d79f-9c00-4135-ba4b-fc8de0b635f3/index.idx

333 Bytes
Binary file not shown.

docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/multivector.lance/_indices/edc3adc7-d8e3-4801-8342-d8cbfab4e18e/auxiliary.idx renamed to be/test/format_v2/table/lance/data/multivector.lance/_indices/3b5436e9-4e0b-4db4-abaa-6b34a4ec78ea/auxiliary.idx

829 Bytes
Binary file not shown.

be/test/format_v2/table/lance/data/multivector.lance/_indices/edc3adc7-d8e3-4801-8342-d8cbfab4e18e/index.idx renamed to be/test/format_v2/table/lance/data/multivector.lance/_indices/3b5436e9-4e0b-4db4-abaa-6b34a4ec78ea/index.idx

333 Bytes
Binary file not shown.

be/test/format_v2/table/lance/data/multivector.lance/_indices/3bccddf2-7f0c-4ffc-bc0e-646b8ca5f3d9/auxiliary.idx renamed to be/test/format_v2/table/lance/data/multivector.lance/_indices/54a5446e-5740-4741-9a84-2c3661ca9dac/auxiliary.idx

894 Bytes
Binary file not shown.

be/test/format_v2/table/lance/data/multivector.lance/_indices/3bccddf2-7f0c-4ffc-bc0e-646b8ca5f3d9/index.idx renamed to be/test/format_v2/table/lance/data/multivector.lance/_indices/54a5446e-5740-4741-9a84-2c3661ca9dac/index.idx

File renamed without changes.

0 commit comments

Comments
 (0)