Skip to content

Commit a425660

Browse files
committed
cache and scan options
1 parent 27d0147 commit a425660

12 files changed

Lines changed: 2530 additions & 10 deletions

File tree

‎be/src/common/config.cpp‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1208,13 +1208,24 @@ DEFINE_Validator(variant_storage_parse_mode,
12081208

12091209
// Lance uses one BE-wide session so metadata/index caches and the optional Foyer data-file cache
12101210
// can be shared by all Lance dataset readers.
1211-
DEFINE_Int64(lance_index_cache_size_bytes, "6442450944"); // 6GB
1211+
DEFINE_Int64(lance_index_cache_size_bytes, "10737418240"); // 10 GiB
12121212
DEFINE_Int64(lance_metadata_cache_size_bytes, "1073741824"); // 1GB
12131213
DEFINE_Bool(enable_lance_data_cache, "true");
12141214
DEFINE_String(lance_data_cache_path, "${DORIS_HOME}/lance_data_cache");
12151215
DEFINE_Int64(lance_data_cache_disk_capacity_bytes, "107374182400"); // 100GB
12161216
DEFINE_Int64(lance_data_cache_read_block_size_bytes, "1048576"); // 1MB
12171217

1218+
// I/O buffering budget per Lance scanner, not a cap on its total memory usage.
1219+
// Runtime changes apply to newly created scanners.
1220+
DEFINE_mInt64(lance_io_buffer_size_bytes, "2147483648"); // 2 GiB
1221+
DEFINE_Validator(lance_io_buffer_size_bytes, [](int64_t value) { return value > 0; });
1222+
1223+
// Read-ahead limits per Lance scanner. Runtime changes apply to newly created scanners.
1224+
DEFINE_mInt32(lance_batch_readahead, "5");
1225+
DEFINE_Validator(lance_batch_readahead, [](int32_t value) { return value > 0; });
1226+
DEFINE_mInt32(lance_fragment_readahead, "5");
1227+
DEFINE_Validator(lance_fragment_readahead, [](int32_t value) { return value > 0; });
1228+
12181229
// block file cache
12191230
DEFINE_Bool(enable_file_cache, "false");
12201231
// ATTENTION: For test only. Keep this enabled in production.

‎be/src/common/config.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1257,6 +1257,13 @@ DECLARE_String(lance_data_cache_path);
12571257
DECLARE_Int64(lance_data_cache_disk_capacity_bytes);
12581258
DECLARE_Int64(lance_data_cache_read_block_size_bytes);
12591259

1260+
// I/O buffering budget per Lance scanner, applied when a new scanner is created.
1261+
DECLARE_mInt64(lance_io_buffer_size_bytes);
1262+
1263+
// Read-ahead limits per Lance scanner, applied when a new scanner is created.
1264+
DECLARE_mInt32(lance_batch_readahead);
1265+
DECLARE_mInt32(lance_fragment_readahead);
1266+
12601267
// block file cache
12611268
DECLARE_Bool(enable_file_cache);
12621269
DECLARE_mBool(enable_file_cache_write_from_s3_file_writer);

‎be/src/format_v2/table/lance_reader.cpp‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
#include <memory>
3131
#include <unordered_set>
3232

33+
#include "common/config.h"
3334
#include "common/consts.h"
3435
#include "common/logging.h"
3536
#include "core/column/column_nullable.h"
@@ -802,6 +803,7 @@ Status LanceTableReader::_open_scanner(const TFileRangeDesc& range) {
802803
if (lance_scanner_set_batch_size(scanner, static_cast<int64_t>(batch_size)) != 0) {
803804
return lance_error("set Lance scanner batch size");
804805
}
806+
RETURN_IF_ERROR(_configure_scan_options(scanner));
805807

806808
const auto& lance_params = range.table_format_params.lance_params;
807809
switch (_search_kind) {
@@ -820,6 +822,32 @@ Status LanceTableReader::_open_scanner(const TFileRangeDesc& range) {
820822
return Status::OK();
821823
}
822824

825+
Status LanceTableReader::_configure_scan_options(LanceScanner* scanner) const {
826+
DORIS_CHECK(scanner != nullptr);
827+
// Doris runs multiple scanners concurrently. Limit each scanner's read-ahead;
828+
// the I/O budget does not cap its total memory usage.
829+
const auto io_buffer_size = static_cast<uint64_t>(config::lance_io_buffer_size_bytes);
830+
const auto batch_readahead = static_cast<size_t>(config::lance_batch_readahead);
831+
const auto fragment_readahead = static_cast<size_t>(config::lance_fragment_readahead);
832+
constexpr bool scan_in_order = false;
833+
834+
if (lance_scanner_set_io_buffer_size(scanner, io_buffer_size) != 0) {
835+
return lance_error("set Lance scanner I/O buffer size");
836+
}
837+
if (lance_scanner_set_batch_readahead(scanner, batch_readahead) != 0) {
838+
return lance_error("set Lance scanner batch readahead");
839+
}
840+
if (lance_scanner_set_fragment_readahead(scanner, fragment_readahead) != 0) {
841+
return lance_error("set Lance scanner fragment readahead");
842+
}
843+
// Storage order is not required; query ordering is enforced by Sort/TopN operators.
844+
if (lance_scanner_set_scan_in_order(scanner, scan_in_order) != 0) {
845+
return lance_error("set Lance scanner scan order");
846+
}
847+
848+
return Status::OK();
849+
}
850+
823851
Status LanceTableReader::_configure_normal_scan(LanceScanner* scanner,
824852
const TLanceFileDesc& lance_params) const {
825853
DORIS_CHECK(scanner != nullptr);

‎be/src/format_v2/table/lance_reader.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@ class LanceTableReader final : public TableReader {
8787
Status _open_dataset(const DatasetKey& key);
8888
Status _prepare_fts_query_context();
8989
Status _open_scanner(const TFileRangeDesc& range);
90+
Status _configure_scan_options(LanceScanner* scanner) const;
9091
Status _configure_normal_scan(LanceScanner* scanner, const TLanceFileDesc& lance_params) const;
9192
Status _configure_vector_search(LanceScanner* scanner,
9293
const TLanceFileDesc& lance_params) const;

‎fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PlanPostProcessors.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan;
2424
import org.apache.doris.nereids.util.MoreFieldsThread;
2525
import org.apache.doris.qe.ConnectContext;
26+
import org.apache.doris.qe.SessionVariable;
2627
import org.apache.doris.thrift.TRuntimeFilterMode;
2728

2829
import com.google.common.collect.ImmutableList;
@@ -73,7 +74,8 @@ public List<PlanPostProcessor> getProcessors() {
7374
2. LazyMaterializeTopN should be applied after RecomputeLogicalPropertiesProcessor
7475
PhysicalLazyMaterialize.materializedSlots should be subsequence of topN.getOutput().
7576
*/
76-
if (cascadesContext.getConnectContext().getSessionVariable().enableTopnLazyMaterialization()) {
77+
SessionVariable sessionVariable = cascadesContext.getConnectContext().getSessionVariable();
78+
if (sessionVariable.enableTopnLazyMaterialization() || sessionVariable.enableLanceLazyMaterialization) {
7779
builder.add(new LazyMaterializeTopN());
7880
}
7981
builder.add(new MergeProjectPostProcessor());

‎fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazyMaterializeTopN.java‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,10 @@ public Plan visitPhysicalTopN(PhysicalTopN topN, CascadesContext ctx) {
7777
if (hasMaterialized) {
7878
return topN;
7979
}
80-
if (SessionVariable.getTopNLazyMaterializationThreshold() < topN.getLimit()) {
80+
SessionVariable sessionVariable = ctx.getConnectContext().getSessionVariable();
81+
boolean enableOtherTables = sessionVariable.topNLazyMaterializationThreshold > 0
82+
&& topN.getLimit() <= sessionVariable.topNLazyMaterializationThreshold;
83+
if (!sessionVariable.enableLanceLazyMaterialization && !enableOtherTables) {
8184
return topN;
8285
}
8386
/*
@@ -91,7 +94,10 @@ public Plan visitPhysicalTopN(PhysicalTopN topN, CascadesContext ctx) {
9194
List<Slot> materializedSlots = new ArrayList<>();
9295
// find the slots which can be lazy materialized
9396
for (Slot slot : topN.getOutput()) {
94-
Optional<MaterializeSource> source = computeMaterializeSource(topN, (SlotReference) slot);
97+
// Decide per source so a Lance relation does not bypass the threshold for other tables.
98+
Optional<MaterializeSource> source = computeMaterializeSource(topN, (SlotReference) slot)
99+
.filter(candidate -> MaterializeProbeVisitor.isLanceExternalSearch(candidate.relation)
100+
? sessionVariable.enableLanceLazyMaterialization : enableOtherTables);
95101
if (source.isPresent()) {
96102
SlotReference baseSlot = source.get().baseSlot;
97103
if (source.get().baseSlot.hasSubColPath()) {

‎fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitor.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -147,8 +147,11 @@ boolean checkTVFRelationTableSupportedType(PhysicalTVFRelation tvfRelation) {
147147
return false;
148148
}
149149

150-
private boolean isLanceExternalSearch(PhysicalTVFRelation tvfRelation) {
151-
String functionName = tvfRelation.getFunction().getName();
150+
static boolean isLanceExternalSearch(Relation relation) {
151+
if (!(relation instanceof PhysicalTVFRelation)) {
152+
return false;
153+
}
154+
String functionName = ((PhysicalTVFRelation) relation).getFunction().getName();
152155
return VectorSearchTableValuedFunction.NAME.equals(functionName)
153156
|| FullTextSearchTableValuedFunction.NAME.equals(functionName);
154157
}

‎fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -580,6 +580,8 @@ public String toString() {
580580

581581
public static final String LANCE_FRAGMENTS_PER_SPLIT = "lance_fragments_per_split";
582582

583+
public static final String ENABLE_LANCE_LAZY_MATERIALIZATION = "enable_lance_lazy_materialization";
584+
583585
public static final String FILE_SPLIT_SIZE_ON_FE = "file_split_size_on_fe";
584586

585587
public static final String FILE_SPLIT_SIZE_ON_BE = "file_split_size_on_be";
@@ -2585,6 +2587,14 @@ public Map<String, Action> getForceEagerAggHintMap() {
25852587
+ "and minimum BE parallelism. Does not affect vector/FTS queries."})
25862588
public int lanceFragmentsPerSplit = 0;
25872589

2590+
@VariableMgr.VarAttr(name = ENABLE_LANCE_LAZY_MATERIALIZATION, needForward = true,
2591+
description = {
2592+
"是否启用 Lance 两阶段延迟读取,默认开启,不受 topn_lazy_materialization_threshold 控制。"
2593+
+ "当前支持 vector_search 和 full_text_search 中可安全延迟读取的列。",
2594+
"Enable Lance two-phase lazy materialization, independently of topn_lazy_materialization_threshold. "
2595+
+ "Enabled by default for eligible columns in vector_search and full_text_search."})
2596+
public boolean enableLanceLazyMaterialization = true;
2597+
25882598
@VariableMgr.VarAttr(name = FILE_SPLIT_SIZE_ON_FE, needForward = true, description = {
25892599
"支持 BE 细粒度切分时,FE 粗粒度文件分片的目标大小,单位为字节,默认为 512MB",
25902600
"Target size in bytes for FE coarse-grained file splits when BE refinement is supported. "

‎regression-test/suites/external_table_p0/lance/test_lance_runtime_filter_pushdown.groovy‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,7 @@ suite("test_lance_runtime_filter_pushdown", "p0,external") {
118118
"""
119119

120120
sql "SET topn_lazy_materialization_threshold = 1024"
121+
sql "SET enable_lance_lazy_materialization = true"
121122
explain {
122123
sql "verbose ${twoPhaseQuery}"
123124
contains "VMaterializeNode"
@@ -139,6 +140,7 @@ suite("test_lance_runtime_filter_pushdown", "p0,external") {
139140
"""
140141

141142
sql "SET topn_lazy_materialization_threshold = -1"
143+
sql "SET enable_lance_lazy_materialization = false"
142144
sql "SET disable_join_reorder = true"
143145
explain {
144146
sql "verbose ${explicitJoinQuery}"

‎regression-test/suites/external_table_p0/lance/test_lance_vector_search_two_phase.groovy‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,9 @@ suite("test_lance_vector_search_two_phase", "p0,external") {
7171
// phase 1: Lance reads embedding for ANN, but outputs only _distance and the hidden row ID
7272
// global TopN: OFFSET 1 / LIMIT 5
7373
// phase 2: fetch row_id, category and label by the hidden row ID
74-
sql "SET topn_lazy_materialization_threshold = 1024"
74+
// Lance remains enabled even when the generic TopN optimization is disabled.
75+
sql "SET topn_lazy_materialization_threshold = -1"
76+
sql "SET enable_lance_lazy_materialization = true"
7577
explain {
7678
sql "verbose ${resultQuery}"
7779
check { explainString ->
@@ -100,9 +102,10 @@ suite("test_lance_vector_search_two_phase", "p0,external") {
100102

101103
qt_two_phase_execution "${resultQuery}"
102104

103-
// Turning the threshold off removes both the Materialization node and the hidden row ID.
105+
// The Lance switch disables materialization even when the generic threshold allows it.
104106
// The user-visible result must remain identical to the two-phase result.
105-
sql "SET topn_lazy_materialization_threshold = -1"
107+
sql "SET topn_lazy_materialization_threshold = 1024"
108+
sql "SET enable_lance_lazy_materialization = false"
106109
explain {
107110
sql "verbose ${resultQuery}"
108111
notContains "VMaterializeNode"
@@ -115,7 +118,7 @@ suite("test_lance_vector_search_two_phase", "p0,external") {
115118
}
116119
qt_one_phase_execution "${resultQuery}"
117120

118-
sql "SET topn_lazy_materialization_threshold = 1024"
121+
sql "SET enable_lance_lazy_materialization = true"
119122

120123
// TVF filter is a Lance prefilter. Lance reads category while selecting ANN candidates
121124
// and removes ineligible rows before nearest(), so the three nearest category='odd' rows

0 commit comments

Comments
 (0)