Skip to content

Commit 60a84c5

Browse files
committed
[feature](lance) Add synchronous index prewarm SQL
1 parent 508f636 commit 60a84c5

18 files changed

Lines changed: 986 additions & 0 deletions

File tree

‎be/src/format_v2/lance/lance_session_manager.cpp‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -215,6 +215,12 @@ Status LanceSessionManager::_initialize() {
215215

216216
Status LanceSessionManager::open_dataset(const char* uri, const char* const* storage_options,
217217
uint64_t version, LanceDataset** dataset) {
218+
return _open_dataset(uri, storage_options, version, dataset, false);
219+
}
220+
221+
Status LanceSessionManager::_open_dataset(const char* uri, const char* const* storage_options,
222+
uint64_t version, LanceDataset** dataset,
223+
bool redact_errors) {
218224
if (uri == nullptr || dataset == nullptr) {
219225
return Status::InvalidArgument("Lance dataset URI and output must not be null");
220226
}
@@ -231,9 +237,46 @@ Status LanceSessionManager::open_dataset(const char* uri, const char* const* sto
231237

232238
*dataset = lance_dataset_open_with_session(uri, storage_options, version, _session);
233239
if (*dataset == nullptr) {
240+
// Constructing an SDK error Status can itself log the provider message. Redact before
241+
// that construction, not only when returning the prewarm RPC response.
242+
if (redact_errors) {
243+
return Status::InternalError(
244+
"Unable to open the pinned Lance snapshot for index prewarm");
245+
}
234246
return lance_error("open Lance dataset with shared session");
235247
}
236248
return Status::OK();
237249
}
238250

251+
Status LanceSessionManager::prewarm_index(const char* uri, const char* const* storage_options,
252+
uint64_t version, const char* index_name) {
253+
if (uri == nullptr || *uri == '\0' || version == 0 || index_name == nullptr ||
254+
*index_name == '\0') {
255+
return Status::InvalidArgument(
256+
"Lance index prewarm requires URI, index name and a fixed version");
257+
}
258+
// Bound concurrent SDK prewarm IO per BE without blocking all RPC workers behind a mutex.
259+
// A failed or timed-out caller may retry after the existing SDK operation finishes.
260+
std::unique_lock lock(_prewarm_mutex, std::try_to_lock);
261+
if (!lock.owns_lock()) {
262+
return Status::Cancelled("Another Lance index prewarm is running on this backend");
263+
}
264+
LanceDataset* raw_dataset = nullptr;
265+
auto status = _open_dataset(uri, storage_options, version, &raw_dataset, true);
266+
std::unique_ptr<LanceDataset, decltype(&lance_dataset_close)> dataset(raw_dataset,
267+
lance_dataset_close);
268+
if (!status.ok()) {
269+
// SDK storage errors can include credentials or signed URLs. Return operation context only.
270+
return Status::InternalError("Unable to open the pinned Lance snapshot for index prewarm");
271+
}
272+
if (lance_dataset_version(dataset.get()) != version) {
273+
return Status::InternalError("Lance index prewarm opened an unexpected snapshot");
274+
}
275+
if (lance_dataset_prewarm_index(dataset.get(), index_name) != 0) {
276+
return Status::InternalError(
277+
"Lance index prewarm failed: index unavailable, unsupported, or storage IO failed");
278+
}
279+
return Status::OK();
280+
}
281+
239282
} // namespace doris::format::lance

‎be/src/format_v2/lance/lance_session_manager.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,11 +58,17 @@ class LanceSessionManager final {
5858
Status open_dataset(const char* uri, const char* const* storage_options, uint64_t version,
5959
LanceDataset** dataset);
6060

61+
Status prewarm_index(const char* uri, const char* const* storage_options, uint64_t version,
62+
const char* index_name);
63+
6164
private:
6265
Status _initialize();
66+
Status _open_dataset(const char* uri, const char* const* storage_options, uint64_t version,
67+
LanceDataset** dataset, bool redact_errors);
6368

6469
Config _config;
6570
std::once_flag _initialize_once;
71+
std::mutex _prewarm_mutex;
6672
LanceSession* _session = nullptr;
6773
std::unique_ptr<LanceSessionMetrics> _metrics;
6874
Status _initialize_status = Status::OK();

‎be/src/service/internal_service.cpp‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
#include <sys/stat.h>
4343

4444
#include <algorithm>
45+
#include <chrono>
4546
#include <exception>
4647
#include <filesystem>
4748
#include <memory>
@@ -79,6 +80,7 @@
7980
#include "format/orc/vorc_reader.h"
8081
#include "format/parquet/vparquet_reader.h"
8182
#include "format/text/text_reader.h"
83+
#include "format_v2/lance/lance_session_manager.h"
8284
#include "format_v2/table/lance_reader.h"
8385
#include "io/fs/local_file_system.h"
8486
#include "io/fs/stream_load_pipe.h"
@@ -811,6 +813,60 @@ void PInternalService::outfile_write_success(google::protobuf::RpcController* co
811813
}
812814
}
813815

816+
void PInternalService::prewarm_lance_index(google::protobuf::RpcController* controller,
817+
const PLanceIndexPrewarmRequest* request,
818+
PLanceIndexPrewarmResponse* response,
819+
google::protobuf::Closure* done) {
820+
const auto received = std::chrono::steady_clock::now();
821+
bool offered = _heavy_work_pool.try_offer([request, response, done, received]() {
822+
brpc::ClosureGuard closure_guard(done);
823+
Status status;
824+
try {
825+
auto run = [&]() -> Status {
826+
DBUG_EXECUTE_IF("PInternalService.prewarm_lance_index.fail", {
827+
return Status::InternalError("Injected Lance index prewarm failure");
828+
});
829+
const auto queued_ms = std::chrono::duration_cast<std::chrono::milliseconds>(
830+
std::chrono::steady_clock::now() - received)
831+
.count();
832+
if (!request->has_timeout_ms() || request->timeout_ms() <= queued_ms) {
833+
return Status::TimedOut("Lance index prewarm expired before execution");
834+
}
835+
// C strings cannot represent embedded NUL bytes; reject rather than silently
836+
// opening a different path/index or truncating a vended credential.
837+
if (request->dataset_uri().find('\0') != std::string::npos ||
838+
request->index_name().find('\0') != std::string::npos) {
839+
return Status::InvalidArgument("Invalid Lance prewarm URI or index name");
840+
}
841+
std::vector<const char*> options;
842+
options.reserve(request->storage_options_size() * 2 + 1);
843+
for (const auto& [key, value] : request->storage_options()) {
844+
if (key.find('\0') != std::string::npos ||
845+
value.find('\0') != std::string::npos) {
846+
return Status::InvalidArgument("Invalid Lance prewarm storage option");
847+
}
848+
options.push_back(key.c_str());
849+
options.push_back(value.c_str());
850+
}
851+
options.push_back(nullptr);
852+
RETURN_IF_ERROR(format::lance::LanceSessionManager::instance().prewarm_index(
853+
request->dataset_uri().c_str(), options.data(), request->dataset_version(),
854+
request->index_name().c_str()));
855+
response->set_dataset_version(request->dataset_version());
856+
return Status::OK();
857+
};
858+
status = run();
859+
} catch (const std::exception&) {
860+
// Provider exception messages may contain the storage credentials in the request.
861+
status = Status::InternalError("Lance index prewarm failed on this backend");
862+
}
863+
status.to_protobuf(response->mutable_status());
864+
});
865+
if (!offered) {
866+
offer_failed(response, done, _heavy_work_pool);
867+
}
868+
}
869+
814870
void PInternalService::fetch_table_schema(google::protobuf::RpcController* controller,
815871
const PFetchTableSchemaRequest* request,
816872
PFetchTableSchemaResult* result,

‎be/src/service/internal_service.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,11 @@ class PInternalService : public PBackendService {
7575
POutfileWriteSuccessResult* result,
7676
google::protobuf::Closure* done) override;
7777

78+
void prewarm_lance_index(google::protobuf::RpcController* controller,
79+
const PLanceIndexPrewarmRequest* request,
80+
PLanceIndexPrewarmResponse* response,
81+
google::protobuf::Closure* done) override;
82+
7883
void fetch_table_schema(google::protobuf::RpcController* controller,
7984
const PFetchTableSchemaRequest* request,
8085
PFetchTableSchemaResult* result,

‎be/test/format_v2/lance/lance_session_manager_test.cpp‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,14 @@
2222

2323
#include <chrono>
2424
#include <filesystem>
25+
#include <lance/lance.hpp>
2526
#include <memory>
2627
#include <string>
2728
#include <utility>
2829

2930
#include "common/metrics/doris_metrics.h"
3031
#include "common/metrics/metrics.h"
32+
#include "util/defer_op.h"
3133

3234
namespace doris::format::lance {
3335
namespace {
@@ -99,6 +101,59 @@ TEST(LanceSessionManagerTest, PublishesSessionCacheMetrics) {
99101
EXPECT_NE(entity->get_metric("lance_session_metadata_cache_misses_total"), nullptr);
100102
}
101103

104+
TEST(LanceSessionManagerTest, PrewarmRequiresPinnedVersionAndIndex) {
105+
LanceSessionManager manager(
106+
{.lance_index_cache_size_bytes = 8 * 1024 * 1024, .lance_data_cache_path = ""});
107+
EXPECT_FALSE(manager.prewarm_index(nullptr, nullptr, 1, "idx").ok());
108+
EXPECT_FALSE(manager.prewarm_index("dataset.lance", nullptr, 0, "idx").ok());
109+
EXPECT_FALSE(manager.prewarm_index("dataset.lance", nullptr, 1, nullptr).ok());
110+
EXPECT_FALSE(manager.prewarm_index("dataset.lance", nullptr, 1, "").ok());
111+
}
112+
113+
TEST(LanceSessionManagerTest, PrewarmErrorsDoNotExposeStorageDetails) {
114+
LanceSessionManager manager(
115+
{.lance_index_cache_size_bytes = 8 * 1024 * 1024, .lance_data_cache_path = ""});
116+
const auto status =
117+
manager.prewarm_index("/nonexistent/private-dataset.lance", nullptr, 1, "idx");
118+
EXPECT_FALSE(status.ok());
119+
EXPECT_EQ(std::string::npos, status.to_string().find("private-dataset"));
120+
EXPECT_FALSE(
121+
manager.prewarm_index(lance_fixture_path().c_str(), nullptr, 1, "missing_idx").ok());
122+
}
123+
124+
TEST(LanceSessionManagerTest, PrewarmUsesTheQuerySessionAndFixedSnapshot) {
125+
const auto path = unique_cache_path();
126+
std::filesystem::copy(lance_fixture_path(), path, std::filesystem::copy_options::recursive);
127+
Defer cleanup {[&] {
128+
std::error_code error;
129+
std::filesystem::remove_all(path, error);
130+
}};
131+
auto writer = ::lance::Dataset::open(path.string());
132+
writer.create_scalar_index("row_id", LANCE_SCALAR_BTREE, "value_idx");
133+
const uint64_t version = writer.version();
134+
LanceSessionManager manager({.lance_index_cache_size_bytes = 8 * 1024 * 1024,
135+
.lance_metadata_cache_size_bytes = 4 * 1024 * 1024,
136+
.lance_data_cache_path = ""});
137+
ASSERT_TRUE(manager.prewarm_index(path.c_str(), nullptr, version, "value_idx").ok());
138+
auto* entity = DorisMetrics::instance()->server_entity();
139+
entity->trigger_hook_unlocked(true);
140+
auto* hits =
141+
static_cast<IntCounter*>(entity->get_metric("lance_session_index_cache_hits_total"));
142+
const auto before = hits->value();
143+
144+
// A separate query handle must see the cache populated by the completed prewarm call.
145+
LanceDataset* raw_dataset = nullptr;
146+
ASSERT_TRUE(manager.open_dataset(path.c_str(), nullptr, version, &raw_dataset).ok());
147+
LanceDatasetPtr query(raw_dataset, lance_dataset_close);
148+
ASSERT_EQ(0, lance_dataset_prewarm_index(query.get(), "value_idx"));
149+
entity->trigger_hook_unlocked(true);
150+
EXPECT_GT(hits->value(), before);
151+
152+
writer.drop_index("value_idx");
153+
EXPECT_FALSE(manager.prewarm_index(path.c_str(), nullptr, writer.version(), "value_idx").ok());
154+
EXPECT_TRUE(manager.prewarm_index(path.c_str(), nullptr, version, "value_idx").ok());
155+
}
156+
102157
TEST(LanceSessionManagerTest, CreatesFoyerBackedSession) {
103158
const auto cache_path = unique_cache_path();
104159
std::filesystem::create_directories(cache_path);

‎fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -538,6 +538,8 @@ supportedOtherStatement
538538
| RESTORE SNAPSHOT label=multipartIdentifier FROM repo=identifier
539539
((ON | EXCLUDE) LEFT_PAREN baseTableRef (COMMA baseTableRef)* RIGHT_PAREN)?
540540
properties=propertyClause? #restore
541+
| WARM UP INDEX indexName=identifier ON tableName=multipartIdentifier
542+
(WITH COMPUTE GROUP computeGroup=identifier)? #warmUpIndex
541543
| WARM UP (CLUSTER | COMPUTE GROUP) destination=identifier WITH
542544
((CLUSTER | COMPUTE GROUP) source=identifier |
543545
(warmUpItem (AND warmUpItem)*)) FORCE?

0 commit comments

Comments
 (0)