Skip to content

Commit 97333dd

Browse files
committed
fts
add case [chore](lance) Upgrade Lance-C to v0.1.8 Issue Number: None Related PR: lance-format/lance-c#69 Problem Summary: Lance-C v0.1.8 now contains the upstream changes previously carried by the v0.1.7 PR apache#64 and runtime-filter patches. Upgrade the bundled source archive and checksum, remove both obsolete v0.1.7 patches, and retain only the current two-commit snapshot of upstream PR apache#69 for prepared FTS INDEX_ONLY plan scoping and explicit _rowid behavior. Upgrade the bundled Lance-C dependency to v0.1.8 and apply upstream PR apache#69. - Test: Manual test - Verified the official archive MD5, applied PR apache#69 to a clean v0.1.8 archive with zero fuzz, compared every tracked file byte-for-byte with PR head 6f0fae4, checked shell syntax, Rustfmt, stale references, and upstream CI status - Behavior changed: Yes. Prepared FTS INDEX_ONLY scans use the PR apache#69 plan rewrite without scanner-wide fast_search - Does this need documentation: No add fts more [chore](lance) Upgrade Lance-C to v0.1.9 Issue Number: None Related PR: lance-format/lance-c#73, lance-format/lance-c#74 Problem Summary: Pick the Lance-C third-party state from commit 417cd56 into the branch-4.1 Lance FTS branch. Upgrade the bundled release from v0.1.8 to v0.1.9, replace the obsolete PR apache#69 patch with PR apache#73, and rename the existing byte-identical PR apache#74 patch for the v0.1.9 base. Keep the branch-specific third-party changes outside the Lance-C section unchanged. None - Test: No need to test (verified shell syntax, patch blob equality, stale-reference removal, and scoped staged paths; compilation and tests were not requested) - Behavior changed: Yes. The bundled Lance-C version and patch baseline are updated. - Does this need documentation: No
1 parent a3f564c commit 97333dd

79 files changed

Lines changed: 2375 additions & 520 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎be/src/format_v2/lance/lance_reader_helper.cpp‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -207,6 +207,48 @@ size_t lance_vector_element_width(TVectorElementType::type type) {
207207
return 0;
208208
}
209209

210+
Status parse_fragment_ids(const TLanceFileDesc& lance_params, std::vector<uint64_t>* fragment_ids) {
211+
DORIS_CHECK(fragment_ids != nullptr);
212+
fragment_ids->clear();
213+
if (!lance_params.__isset.fragment_ids || lance_params.fragment_ids.empty()) {
214+
return Status::OK();
215+
}
216+
fragment_ids->reserve(lance_params.fragment_ids.size());
217+
for (const auto fragment_id : lance_params.fragment_ids) {
218+
if (fragment_id < 0) {
219+
return Status::InvalidArgument("Lance fragment id must be non-negative: {}",
220+
fragment_id);
221+
}
222+
fragment_ids->emplace_back(static_cast<uint64_t>(fragment_id));
223+
}
224+
return Status::OK();
225+
}
226+
227+
Status parse_index_segment_uuids(const TLanceFileDesc& lance_params,
228+
std::vector<uint8_t>* segment_uuids, size_t* segment_count) {
229+
DORIS_CHECK(segment_uuids != nullptr);
230+
DORIS_CHECK(segment_count != nullptr);
231+
segment_uuids->clear();
232+
*segment_count = 0;
233+
if (!lance_params.__isset.index_segment_uuids || lance_params.index_segment_uuids.empty()) {
234+
return Status::OK();
235+
}
236+
constexpr size_t UUID_SIZE = 16;
237+
if (lance_params.index_segment_uuids.size() > std::numeric_limits<size_t>::max() / UUID_SIZE) {
238+
return Status::InvalidArgument("too many Lance index segment UUIDs");
239+
}
240+
segment_uuids->reserve(lance_params.index_segment_uuids.size() * UUID_SIZE);
241+
for (const auto& uuid : lance_params.index_segment_uuids) {
242+
if (uuid.size() != UUID_SIZE) {
243+
return Status::InvalidArgument("Lance index segment UUID must contain 16 bytes, got {}",
244+
uuid.size());
245+
}
246+
segment_uuids->insert(segment_uuids->end(), uuid.begin(), uuid.end());
247+
}
248+
*segment_count = lance_params.index_segment_uuids.size();
249+
return Status::OK();
250+
}
251+
210252
Status convert_arrow_schema_to_doris(const std::shared_ptr<arrow::Schema>& arrow_schema,
211253
std::vector<std::string>* column_names,
212254
std::vector<DataTypePtr>* column_types) {

‎be/src/format_v2/lance/lance_reader_helper.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
#pragma once
1919

2020
#include <cstddef>
21+
#include <cstdint>
2122
#include <memory>
2223
#include <string>
2324
#include <string_view>
@@ -38,6 +39,7 @@ class Schema;
3839
namespace doris::format::lance {
3940

4041
inline constexpr std::string_view LANCE_DISTANCE_COLUMN = "_distance";
42+
inline constexpr std::string_view LANCE_SCORE_COLUMN = "_score";
4143
inline constexpr std::string_view LANCE_ROW_ID_COLUMN = "_rowid";
4244
inline constexpr const char* LANCE_READER_PROFILE = "LanceReader";
4345

@@ -55,6 +57,12 @@ struct LanceBatchDeleter {
5557

5658
size_t lance_vector_element_width(TVectorElementType::type type);
5759

60+
// Validate and convert the fragment and index-segment identifiers carried by the FE into the
61+
// unsigned and packed representations expected by lance-c.
62+
Status parse_fragment_ids(const TLanceFileDesc& lance_params, std::vector<uint64_t>* fragment_ids);
63+
Status parse_index_segment_uuids(const TLanceFileDesc& lance_params,
64+
std::vector<uint8_t>* segment_uuids, size_t* segment_count);
65+
5866
// Convert every top-level field without discarding unsupported columns. Malformed schemas still
5967
// return an error and leave both output vectors unchanged. DataTypeNothing is the local sentinel
6068
// for a valid Arrow field whose logical type Doris does not support.

‎be/src/format_v2/table/lance_reader.cpp‎

Lines changed: 338 additions & 143 deletions
Large diffs are not rendered by default.

‎be/src/format_v2/table/lance_reader.h‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434

3535
struct LanceBatch;
3636
struct LanceDataset;
37+
struct LanceFtsQueryContext;
3738
struct LanceScanner;
3839

3940
namespace doris {
@@ -80,11 +81,17 @@ class LanceTableReader final : public TableReader {
8081
bool operator==(const DatasetKey&) const = default;
8182
};
8283

84+
Status _resolve_search_kind();
8385
Status _validate_external_search_request() const;
84-
Status _ensure_dataset_open(const TFileRangeDesc& range);
86+
Status _ensure_dataset_open(const TFileRangeDesc& range, bool prepare_fts_context = true);
8587
Status _open_dataset(const DatasetKey& key);
88+
Status _prepare_fts_query_context();
8689
Status _open_scanner(const TFileRangeDesc& range);
87-
Status _configure_vector_search(LanceScanner* scanner) const;
90+
Status _configure_normal_scan(LanceScanner* scanner, const TLanceFileDesc& lance_params) const;
91+
Status _configure_vector_search(LanceScanner* scanner,
92+
const TLanceFileDesc& lance_params) const;
93+
Status _configure_full_text_search(LanceScanner* scanner,
94+
const TLanceFileDesc& lance_params) const;
8895
// Keep lance-c's anonymous statistics typedef out of this header. _open_scanner installs the
8996
// strongly typed C callback adapter before forwarding the borrowed value here.
9097
static void _collect_scan_statistics(void* callback_ctx, const void* opaque_statistics);
@@ -123,7 +130,9 @@ class LanceTableReader final : public TableReader {
123130
RuntimeProfile::Counter* _index_comparisons = nullptr;
124131
std::unordered_map<std::string_view, RuntimeProfile::Counter*> _lance_count_metrics;
125132
std::unordered_map<std::string_view, RuntimeProfile::Counter*> _lance_time_metrics;
126-
bool _vector_search = false;
133+
LanceFtsQueryContext* _fts_query_context = nullptr;
134+
enum class SearchKind { NORMAL, VECTOR, FULL_TEXT };
135+
SearchKind _search_kind = SearchKind::NORMAL;
127136
bool _eof = false;
128137
};
129138

0 commit comments

Comments
 (0)