Skip to content

Commit db0cbc2

Browse files
committed
[enhancement](lance) Add cache diagnostics and scalar index auto selection
1 parent 08f19f6 commit db0cbc2

18 files changed

Lines changed: 544 additions & 18 deletions

File tree

‎be/src/common/config.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1232,6 +1232,8 @@ DEFINE_Bool(enable_lance_data_cache, "true");
12321232
DEFINE_String(lance_data_cache_path, "${DORIS_HOME}/lance_data_cache");
12331233
DEFINE_Int64(lance_data_cache_disk_capacity_bytes, "107374182400"); // 100GB
12341234
DEFINE_Int64(lance_data_cache_read_block_size_bytes, "1048576"); // 1MB
1235+
// Zero preserves the existing one-block Foyer memory tier, including custom block sizes.
1236+
DEFINE_Int64(lance_data_cache_memory_capacity_bytes, "0");
12351237

12361238
// I/O buffering budget per Lance scanner, not a cap on its total memory usage.
12371239
// Runtime changes apply to newly created scanners.

‎be/src/common/config.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1268,6 +1268,7 @@ DECLARE_Bool(enable_lance_data_cache);
12681268
DECLARE_String(lance_data_cache_path);
12691269
DECLARE_Int64(lance_data_cache_disk_capacity_bytes);
12701270
DECLARE_Int64(lance_data_cache_read_block_size_bytes);
1271+
DECLARE_Int64(lance_data_cache_memory_capacity_bytes);
12711272

12721273
// I/O buffering budget per Lance scanner, applied when a new scanner is created.
12731274
DECLARE_mInt64(lance_io_buffer_size_bytes);

‎be/src/format_v2/lance/lance_session_manager.cpp‎

Lines changed: 34 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,8 @@ LanceSessionManager::Config load_lance_session_config() {
6464
.lance_data_cache_disk_capacity_bytes = config::lance_data_cache_disk_capacity_bytes,
6565
.lance_data_cache_read_block_size_bytes =
6666
config::lance_data_cache_read_block_size_bytes,
67+
.lance_data_cache_memory_capacity_bytes =
68+
config::lance_data_cache_memory_capacity_bytes,
6769
};
6870
}
6971

@@ -159,8 +161,8 @@ LanceSessionManager::LanceSessionManager(Config config) : _config(std::move(conf
159161
<< ", lance_data_cache_disk_capacity_bytes="
160162
<< _config.lance_data_cache_disk_capacity_bytes
161163
<< ", lance_data_cache_read_block_size_bytes="
162-
<< _config.lance_data_cache_read_block_size_bytes
163-
<< ", foyer_memory_capacity_bytes=" << _config.lance_data_cache_read_block_size_bytes;
164+
<< _config.lance_data_cache_read_block_size_bytes << ", foyer_memory_capacity_bytes="
165+
<< _config.resolved_data_cache_memory_capacity_bytes();
164166
}
165167

166168
LanceSessionManager::~LanceSessionManager() {
@@ -170,17 +172,20 @@ LanceSessionManager::~LanceSessionManager() {
170172

171173
Status LanceSessionManager::_initialize() {
172174
if (_config.enable_lance_data_cache) {
175+
if (_config.lance_data_cache_memory_capacity_bytes < 0) {
176+
return Status::InvalidArgument(
177+
"lance_data_cache_memory_capacity_bytes must be non-negative");
178+
}
173179
// Treat the configured cache mode as a process-level requirement. If an enabled
174180
// data cache cannot initialize, report the failure instead of silently creating a
175181
// session without it. This keeps directory/configuration/device failures visible
176182
// to the operator. Disabling the cache is an explicit configuration change.
177183
const LanceDataCacheOptions data_cache_options {
178184
.directory = _config.lance_data_cache_path.c_str(),
179-
// Foyer's HybridCache requires a memory tier. Keep it at the minimum useful
180-
// capacity of exactly one range-cache block; WriteOnInsertion enqueues disk
181-
// writes on insertion rather than waiting for memory-tier eviction.
185+
// Keep the historical one-block tier when the memory capacity is unset.
186+
// Its capacity is independent of the range-cache block granularity otherwise.
182187
.memory_capacity_bytes =
183-
static_cast<uint64_t>(_config.lance_data_cache_read_block_size_bytes),
188+
static_cast<uint64_t>(_config.resolved_data_cache_memory_capacity_bytes()),
184189
.disk_capacity_bytes =
185190
static_cast<uint64_t>(_config.lance_data_cache_disk_capacity_bytes),
186191
.read_block_size_bytes =
@@ -220,19 +225,37 @@ Status LanceSessionManager::open_dataset(const char* uri, const char* const* sto
220225
}
221226
*dataset = nullptr;
222227

223-
// Initialize lazily on the first dataset open, not at BE startup. A failed Status is a
228+
RETURN_IF_ERROR(_ensure_initialized());
229+
230+
*dataset = lance_dataset_open_with_session(uri, storage_options, version, _session);
231+
if (*dataset == nullptr) {
232+
return lance_error("open Lance dataset with shared session");
233+
}
234+
return Status::OK();
235+
}
236+
237+
Status LanceSessionManager::_ensure_initialized() {
238+
// Initialize lazily on the first use, not at BE startup. A failed Status is a
224239
// normal return from this lambda, so call_once completes and retains that failure just
225240
// like a successful initialization. All subsequent readers receive the same copied
226241
// error; fixing the cache directory alone does not trigger another attempt. This is
227242
// intentional: repair the cache configuration/storage (or disable the data cache), then
228243
// restart the BE to recreate the process-wide manager and session.
229244
std::call_once(_initialize_once, [this] { _initialize_status = _initialize(); });
230-
RETURN_IF_ERROR(_initialize_status);
245+
return _initialize_status;
246+
}
231247

232-
*dataset = lance_dataset_open_with_session(uri, storage_options, version, _session);
233-
if (*dataset == nullptr) {
234-
return lance_error("open Lance dataset with shared session");
248+
Status LanceSessionManager::get_cache_statistics(CacheStatistics* statistics) {
249+
if (statistics == nullptr) {
250+
return Status::InvalidArgument("Lance cache statistics output must not be null");
251+
}
252+
RETURN_IF_ERROR(_ensure_initialized());
253+
LanceSessionCacheStats stats {};
254+
if (lance_session_get_cache_stats(_session, &stats) != 0) {
255+
return lance_error("collect Lance session cache statistics");
235256
}
257+
statistics->index_cache_misses = stats.index_cache_misses;
258+
statistics->metadata_cache_misses = stats.metadata_cache_misses;
236259
return Status::OK();
237260
}
238261

‎be/src/format_v2/lance/lance_session_manager.h‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,18 @@ class LanceSessionManager final {
4343
std::string lance_data_cache_path;
4444
int64_t lance_data_cache_disk_capacity_bytes = 0;
4545
int64_t lance_data_cache_read_block_size_bytes = 0;
46+
int64_t lance_data_cache_memory_capacity_bytes = 0;
47+
48+
int64_t resolved_data_cache_memory_capacity_bytes() const {
49+
return lance_data_cache_memory_capacity_bytes == 0
50+
? lance_data_cache_read_block_size_bytes
51+
: lance_data_cache_memory_capacity_bytes;
52+
}
53+
};
54+
55+
struct CacheStatistics {
56+
uint64_t index_cache_misses = 0;
57+
uint64_t metadata_cache_misses = 0;
4658
};
4759

4860
static LanceSessionManager& instance();
@@ -58,8 +70,12 @@ class LanceSessionManager final {
5870
Status open_dataset(const char* uri, const char* const* storage_options, uint64_t version,
5971
LanceDataset** dataset);
6072

73+
// A process-wide snapshot, including other readers using the shared session.
74+
Status get_cache_statistics(CacheStatistics* statistics);
75+
6176
private:
6277
Status _initialize();
78+
Status _ensure_initialized();
6379

6480
Config _config;
6581
std::once_flag _initialize_once;

‎be/src/format_v2/table/lance_reader.cpp‎

Lines changed: 48 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,15 @@ Status LanceTableReader::init(TableReadOptions&& options) {
117117
file_scan_profile::TABLE_READER, 1);
118118
_dataset_open_time = add_lance_counter(_scanner_profile, "LanceDatasetOpenTime", TUnit::TIME_NS,
119119
LANCE_TIMING_PROFILE);
120+
_session_index_cache_misses = add_lance_counter(
121+
_scanner_profile, "LanceSessionIndexCacheMisses", TUnit::UNIT, "LanceSessionCache");
122+
_session_metadata_cache_misses = add_lance_counter(
123+
_scanner_profile, "LanceSessionMetadataCacheMisses", TUnit::UNIT, "LanceSessionCache");
124+
_scanner_profile->add_info_string(
125+
"LanceSessionCacheStatsScope",
126+
"BE-wide changes during dataset-open windows; includes concurrent readers; "
127+
"overlapping windows must not be summed as query-exclusive misses");
128+
_scanner_profile->add_info_string("LanceSessionCacheStatsAvailable", "false");
120129
_arrow_to_doris_block_time = add_lance_counter(_scanner_profile, "LanceArrowToDorisBlockTime",
121130
TUnit::TIME_NS, LANCE_TIMING_PROFILE);
122131
_data_cache_bytes_read_from_cache =
@@ -647,8 +656,15 @@ Status LanceTableReader::_open_dataset(const DatasetKey& key) {
647656
std::unique_ptr<LanceDataset, LanceDatasetDeleter> dataset;
648657
{
649658
SCOPED_TIMER(_dataset_open_time);
659+
auto& session = LanceSessionManager::instance();
660+
_session_cache_statistics_valid =
661+
session.get_cache_statistics(&_session_cache_statistics).ok();
662+
if (!_session_cache_statistics_valid) {
663+
_session_cache_statistics_complete = false;
664+
_scanner_profile->add_info_string("LanceSessionCacheStatsAvailable", "false");
665+
}
650666
LanceDataset* raw_dataset = nullptr;
651-
Status status = LanceSessionManager::instance().open_dataset(
667+
Status status = session.open_dataset(
652668
key.uri.c_str(), key.storage_options.empty() ? nullptr : storage_option_ptrs.data(),
653669
static_cast<uint64_t>(key.version), &raw_dataset);
654670
if (!status.ok()) {
@@ -1261,6 +1277,7 @@ void LanceTableReader::_close_dataset() {
12611277
_fts_query_context = nullptr;
12621278
}
12631279
if (_dataset != nullptr) {
1280+
_collect_session_cache_statistics();
12641281
_collect_data_cache_statistics();
12651282
lance_dataset_close(_dataset);
12661283
_dataset = nullptr;
@@ -1271,6 +1288,36 @@ void LanceTableReader::_close_dataset() {
12711288
_record_batch_converter.reset_schema();
12721289
}
12731290

1291+
void LanceTableReader::_collect_session_cache_statistics() {
1292+
if (!_session_cache_statistics_valid) {
1293+
return;
1294+
}
1295+
LanceSessionManager::CacheStatistics current;
1296+
const auto status = LanceSessionManager::instance().get_cache_statistics(&current);
1297+
if (!status.ok()) {
1298+
_session_cache_statistics_complete = false;
1299+
_scanner_profile->add_info_string("LanceSessionCacheStatsAvailable", "false");
1300+
LOG(WARNING) << status;
1301+
return;
1302+
}
1303+
// Session counters are monotonic and shared. Subtract the last snapshot exactly once
1304+
// per window; sampling them never resets counters used by other readers or metrics.
1305+
const auto update = [](RuntimeProfile::Counter* counter, uint64_t before, uint64_t after) {
1306+
if (counter != nullptr && after >= before) {
1307+
const auto delta =
1308+
std::min<uint64_t>(after - before, std::numeric_limits<int64_t>::max());
1309+
COUNTER_UPDATE(counter, static_cast<int64_t>(delta));
1310+
}
1311+
};
1312+
update(_session_index_cache_misses, _session_cache_statistics.index_cache_misses,
1313+
current.index_cache_misses);
1314+
update(_session_metadata_cache_misses, _session_cache_statistics.metadata_cache_misses,
1315+
current.metadata_cache_misses);
1316+
_session_cache_statistics = current;
1317+
_scanner_profile->add_info_string("LanceSessionCacheStatsAvailable",
1318+
_session_cache_statistics_complete ? "true" : "false");
1319+
}
1320+
12741321
void LanceTableReader::_collect_data_cache_statistics() {
12751322
if (_dataset == nullptr) {
12761323
return;

‎be/src/format_v2/table/lance_reader.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626

2727
#include "common/status.h"
2828
#include "format_v2/lance/lance_record_batch_converter.h"
29+
#include "format_v2/lance/lance_session_manager.h"
2930
#include "format_v2/table_reader.h"
3031
#include "runtime/runtime_profile.h"
3132

@@ -92,6 +93,7 @@ class LanceTableReader final : public TableReader {
9293
// Collect the cumulative statistics owned by this dataset handle. The Lance-C API returns an
9394
// absolute snapshot, so this method replaces rather than increments the profile counters.
9495
void _collect_data_cache_statistics();
96+
void _collect_session_cache_statistics();
9597
// Keep lance-c's anonymous statistics typedef out of this header. _open_scanner installs the
9698
// strongly typed C callback adapter before forwarding the borrowed value here.
9799
static void _collect_scan_statistics(void* callback_ctx, const void* opaque_statistics);
@@ -101,6 +103,11 @@ class LanceTableReader final : public TableReader {
101103
Status _dataset_key(const TFileRangeDesc& range, DatasetKey* key) const;
102104

103105
LanceDataset* _dataset = nullptr;
106+
LanceSessionManager::CacheStatistics _session_cache_statistics;
107+
bool _session_cache_statistics_valid = false;
108+
bool _session_cache_statistics_complete = true;
109+
RuntimeProfile::Counter* _session_index_cache_misses = nullptr;
110+
RuntimeProfile::Counter* _session_metadata_cache_misses = nullptr;
104111
std::shared_ptr<arrow::Schema> _dataset_schema;
105112
LanceScanner* _scanner = nullptr;
106113
ShardedKVCache* _runtime_filter_cache = nullptr;

‎be/test/format_v2/lance/lance_session_manager_test.cpp‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ TEST(LanceSessionManagerTest, SessionAndDataCacheConfigurationsAreIndependent) {
5454
.lance_data_cache_path = "",
5555
.lance_data_cache_disk_capacity_bytes = -1,
5656
.lance_data_cache_read_block_size_bytes = -1,
57+
.lance_data_cache_memory_capacity_bytes = -1,
5758
};
5859
LanceSessionManager manager(std::move(config));
5960
LanceDataset* raw_dataset = nullptr;
@@ -62,6 +63,27 @@ TEST(LanceSessionManagerTest, SessionAndDataCacheConfigurationsAreIndependent) {
6263
ASSERT_NE(dataset, nullptr);
6364
}
6465

66+
TEST(LanceSessionManagerTest, MemoryCapacityDefaultsToOneConfiguredBlock) {
67+
LanceSessionManager::Config config;
68+
config.lance_data_cache_read_block_size_bytes = 64 * 1024;
69+
EXPECT_EQ(config.resolved_data_cache_memory_capacity_bytes(), 64 * 1024);
70+
config.lance_data_cache_memory_capacity_bytes = 8 * 1024 * 1024;
71+
EXPECT_EQ(config.resolved_data_cache_memory_capacity_bytes(), 8 * 1024 * 1024);
72+
EXPECT_EQ(config.lance_data_cache_read_block_size_bytes, 64 * 1024);
73+
}
74+
75+
TEST(LanceSessionManagerTest, RejectsNegativeMemoryCapacityBeforeUnsignedConversion) {
76+
LanceSessionManager::Config config;
77+
config.enable_lance_data_cache = true;
78+
config.lance_data_cache_memory_capacity_bytes = -1;
79+
LanceSessionManager manager(std::move(config));
80+
LanceDataset* dataset = nullptr;
81+
const auto status = manager.open_dataset(lance_fixture_path().c_str(), nullptr, 0, &dataset);
82+
EXPECT_FALSE(status.ok());
83+
EXPECT_NE(status.to_string().find("lance_data_cache_memory_capacity_bytes"), std::string::npos);
84+
EXPECT_EQ(dataset, nullptr);
85+
}
86+
6587
TEST(LanceSessionManagerTest, PublishesSessionCacheMetrics) {
6688
constexpr int64_t INDEX_CACHE_CAPACITY = 8 * 1024 * 1024;
6789
constexpr int64_t METADATA_CACHE_CAPACITY = 4 * 1024 * 1024;
@@ -114,6 +136,7 @@ TEST(LanceSessionManagerTest, CreatesFoyerBackedSession) {
114136
.lance_data_cache_path = cache_path.string(),
115137
.lance_data_cache_disk_capacity_bytes = 32 * 1024 * 1024,
116138
.lance_data_cache_read_block_size_bytes = 64 * 1024,
139+
.lance_data_cache_memory_capacity_bytes = 8 * 1024 * 1024,
117140
};
118141
{
119142
LanceSessionManager manager(std::move(config));

‎be/test/format_v2/table/lance_reader_test.cpp‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1349,6 +1349,43 @@ TEST(LanceTableReaderSnapshotTest, ProfileShowsTheSnapshotRead) {
13491349
EXPECT_TRUE(latest_reader.close().ok());
13501350
}
13511351

1352+
TEST(LanceTableReaderSnapshotTest, ProfileReportsObservedSessionCacheMisses) {
1353+
LanceSessionManager::CacheStatistics before;
1354+
ASSERT_TRUE(LanceSessionManager::instance().get_cache_statistics(&before).ok());
1355+
const std::filesystem::path dataset_uri =
1356+
"./be/test/format_v2/table/lance/data/all_types.lance";
1357+
TQueryGlobals query_globals;
1358+
RuntimeState state(query_globals);
1359+
RuntimeProfile profile("lance_session_cache_profile");
1360+
TFileScanRangeParams scan_params;
1361+
const Columns columns {projected_column("row_id", TYPE_BIGINT, false)};
1362+
LanceTableReader reader;
1363+
ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok());
1364+
ASSERT_TRUE(prepare_range(&reader, make_latest_lance_range(dataset_uri)).ok());
1365+
ASSERT_TRUE(reader.close().ok());
1366+
LanceSessionManager::CacheStatistics after;
1367+
ASSERT_TRUE(LanceSessionManager::instance().get_cache_statistics(&after).ok());
1368+
for (const char* name : {"LanceSessionIndexCacheMisses", "LanceSessionMetadataCacheMisses"}) {
1369+
auto* counter = profile.get_counter(name);
1370+
ASSERT_NE(counter, nullptr) << name;
1371+
EXPECT_GE(counter->value(), 0) << name;
1372+
}
1373+
EXPECT_EQ(profile.get_counter("LanceSessionIndexCacheMisses")->value(),
1374+
after.index_cache_misses - before.index_cache_misses);
1375+
EXPECT_EQ(profile.get_counter("LanceSessionMetadataCacheMisses")->value(),
1376+
after.metadata_cache_misses - before.metadata_cache_misses);
1377+
const auto* scope = profile.get_info_string("LanceSessionCacheStatsScope");
1378+
ASSERT_NE(scope, nullptr);
1379+
EXPECT_NE(scope->find("concurrent"), std::string::npos);
1380+
const auto* available = profile.get_info_string("LanceSessionCacheStatsAvailable");
1381+
ASSERT_NE(available, nullptr);
1382+
EXPECT_EQ(*available, "true");
1383+
auto* misses = profile.get_counter("LanceSessionMetadataCacheMisses");
1384+
const auto before_close = misses->value();
1385+
ASSERT_TRUE(reader.close().ok());
1386+
EXPECT_EQ(misses->value(), before_close);
1387+
}
1388+
13521389
TEST(LanceTableReaderSnapshotTest, OpenErrorsNameTheVersionWithoutTheQueryString) {
13531390
const std::filesystem::path dataset_uri =
13541391
"./be/test/format_v2/table/lance/data/all_types.lance";

0 commit comments

Comments
 (0)