Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
7654cf7
[feature](be) Add linear file range coalescing
bobhan1 Aug 28, 2026
90cf5b6
[feature](be) Add cache-aware file range planning
bobhan1 Aug 28, 2026
85894d0
[feature](be) Add file range read scheduler
bobhan1 Aug 28, 2026
c64e719
[feature](be) Reuse remote-only reads for no-write ranges
bobhan1 Aug 28, 2026
563b5ba
[feature](be) Add query read-ahead runtime ownership
bobhan1 Aug 28, 2026
e2a4012
[improvement](be) Add opt-in file range IO tracing
bobhan1 Sep 22, 2026
8a08ae1
[fix](be) Scope filesystem error-code imports to their functions
bobhan1 Oct 15, 2026
4271da9
[feature](be) Add page-driven column read-ahead windows
bobhan1 Aug 28, 2026
e7dbfa1
[feature](be) Add segment range read-ahead coordination
bobhan1 Oct 29, 2026
00fa56b
[feature](be) Connect scalar column readers to range read-ahead
bobhan1 Sep 2, 2026
8190e0f
[feature](be) Initialize scan read-ahead once after pruning
bobhan1 Oct 29, 2026
d03f274
[feature](be) Add complete-block cache write handoff and admission
bobhan1 Aug 28, 2026
a547700
[feature](be) Split consumed ranges for cache writeback
bobhan1 Aug 28, 2026
7cfe76c
[feature](be) Add cache block hole-fill planning
bobhan1 Aug 28, 2026
a7e3899
[feature](be) Add background partial block writeback
bobhan1 Aug 28, 2026
999f5c6
[feature](be) Initialize background hole fill
bobhan1 Aug 28, 2026
6fd7bb6
[feature](be) Route consumed ranges to cache writeback
bobhan1 Aug 28, 2026
1ae1272
[feature](be) Connect read-ahead range writeback
bobhan1 Aug 28, 2026
fd6df2c
[feature](be) Extend range read-ahead to point queries
bobhan1 Sep 2, 2026
a881473
[fix](be) Validate S3 upload buffer alignment on config updates
bobhan1 Oct 29, 2026
b237ebc
[fix](be) Handle irregular cache blocks in writeback coverage checks
bobhan1 Oct 29, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
[feature](be) Add cache block hole-fill planning
### What problem does this PR solve?

Issue Number: None

Related PR: #67292

Problem Summary: Partial query ranges can cover unordered and overlapping fragments inside one File Cache block. Add a block-local planner that unions existing coverage, takes its complement to produce sorted holes, and applies the dedicated three-parameter coalescing policy without crossing the block boundary. The coalescing size limit controls combining holes; one mandatory hole may be larger and remains one remote read.

### Release note

None

### Check List (For Author)

- Test: Unit Test
    - ./run-be-ut.sh -j100 --run '--filter=HoleFillPlannerTest.*'
- Behavior changed: No; this commit adds an internal hole-fill planner.
- Does this need documentation: No
  • Loading branch information
bobhan1 committed Oct 29, 2026
commit 7cfe76cbbd336c57ad3894bb2aecd0e5e2bfdab0
91 changes: 91 additions & 0 deletions be/src/io/cache/hole_fill_planner.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

#include "io/cache/hole_fill_planner.h"

#include <algorithm>
#include <limits>
#include <utility>

#include "common/logging.h"

namespace doris::io {

Status HoleFillPlanner::plan(const FileRange& block,
const std::vector<FileRange>& covered_intervals,
const FileRangeCoalesceOptions& options,
std::vector<FileRange>* output_ranges) {
DORIS_CHECK(output_ranges != nullptr);
RETURN_IF_ERROR(options.validate());
if (block.size == 0) {
return Status::InvalidArgument("hole-fill block at offset {} is empty", block.offset);
}
if (block.offset > std::numeric_limits<size_t>::max() - block.size) {
return Status::InvalidArgument("hole-fill block at offset {} with size {} overflows",
block.offset, block.size);
}
for (const auto& covered : covered_intervals) {
if (covered.size == 0) {
return Status::InvalidArgument("covered interval at offset {} is empty",
covered.offset);
}
if (covered.offset < block.offset || covered.offset > block.end() ||
covered.size > block.end() - covered.offset) {
return Status::InvalidArgument(
"covered interval at offset {} with size {} exceeds block [{}, {})",
covered.offset, covered.size, block.offset, block.end());
}
}

auto sorted_coverage = covered_intervals;
std::ranges::sort(sorted_coverage, [](const FileRange& left, const FileRange& right) {
return left.offset < right.offset;
});
std::vector<FileRange> covered_union;
covered_union.reserve(sorted_coverage.size());
for (const auto& covered : sorted_coverage) {
if (covered_union.empty() || covered_union.back().end() < covered.offset) {
covered_union.push_back(covered);
} else {
covered_union.back().size = std::max(covered_union.back().end(), covered.end()) -
covered_union.back().offset;
}
}

std::vector<FileRange> holes;
size_t cursor = block.offset;
for (const auto& covered : covered_union) {
DORIS_CHECK(cursor <= covered.offset);
if (cursor < covered.offset) {
holes.push_back({.offset = cursor, .size = covered.offset - cursor});
}
cursor = covered.end();
}
if (cursor < block.end()) {
holes.push_back({.offset = cursor, .size = block.end() - cursor});
}

auto coalesced_holes = FileRangeCoalescer::coalesce(holes, options);
for (const auto& hole : coalesced_holes) {
DORIS_CHECK(block.offset <= hole.offset);
DORIS_CHECK(hole.end() <= block.end());
}
*output_ranges = std::move(coalesced_holes);
return Status::OK();
}

} // namespace doris::io
39 changes: 39 additions & 0 deletions be/src/io/cache/hole_fill_planner.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

#pragma once

#include <vector>

#include "common/status.h"
#include "io/fs/file_range_coalescer.h"

namespace doris::io {

/// Computes the remote reads needed to complete one File Cache block. Covered intervals are first
/// unioned, their complement inside `block` becomes the hole list, and the supplied coalescing
/// policy may combine nearby holes without ever crossing the block boundary.
class HoleFillPlanner {
public:
/// `covered_intervals` may overlap and arrive in any order, but each interval must be non-empty
/// and contained by `block`. The output is sorted, disjoint, and contained by `block`.
static Status plan(const FileRange& block, const std::vector<FileRange>& covered_intervals,
const FileRangeCoalesceOptions& options,
std::vector<FileRange>* output_ranges);
};

} // namespace doris::io
132 changes: 132 additions & 0 deletions be/test/io/cache/hole_fill_planner_test.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

#include "io/cache/hole_fill_planner.h"

#include <gtest/gtest.h>

#include <vector>

namespace doris::io {
namespace {

constexpr size_t KiB(size_t value) {
return value * 1024;
}

const FileRangeCoalesceOptions kOptions {
.max_gap_bytes = KiB(32),
.max_range_bytes = KiB(1024),
.max_read_amplification_ratio = 2.0,
};

TEST(HoleFillPlannerTest, MergesNearbyHolesWithinAmplificationLimit) {
const FileRange block {.offset = 0, .size = KiB(1024)};
const std::vector<FileRange> covered {{.offset = KiB(544), .size = KiB(480)},
{.offset = 0, .size = KiB(256)},
{.offset = KiB(384), .size = KiB(32)}};
std::vector<FileRange> ranges;

ASSERT_TRUE(HoleFillPlanner::plan(block, covered, kOptions, &ranges).ok());

EXPECT_EQ(ranges, (std::vector<FileRange> {{.offset = KiB(256), .size = KiB(288)}}));
}

TEST(HoleFillPlannerTest, KeepsHolesSeparateWhenGapExceedsLimit) {
const FileRange block {.offset = 0, .size = KiB(1024)};
const std::vector<FileRange> covered {{.offset = KiB(480), .size = KiB(64)}};
std::vector<FileRange> ranges;

ASSERT_TRUE(HoleFillPlanner::plan(block, covered, kOptions, &ranges).ok());

EXPECT_EQ(ranges, (std::vector<FileRange> {{.offset = 0, .size = KiB(480)},
{.offset = KiB(544), .size = KiB(480)}}));
}

TEST(HoleFillPlannerTest, AppliesAmplificationLimitAfterGapCheck) {
const FileRange block {.offset = 0, .size = KiB(1024)};
std::vector<FileRange> ranges;

ASSERT_TRUE(HoleFillPlanner::plan(block,
{{.offset = KiB(8), .size = KiB(16)},
{.offset = KiB(32), .size = KiB(992)}},
kOptions, &ranges)
.ok());
EXPECT_EQ(ranges, (std::vector<FileRange> {{.offset = 0, .size = KiB(32)}}));

ASSERT_TRUE(HoleFillPlanner::plan(block,
{{.offset = KiB(4), .size = KiB(16)},
{.offset = KiB(24), .size = KiB(1000)}},
kOptions, &ranges)
.ok());
EXPECT_EQ(ranges, (std::vector<FileRange> {{.offset = 0, .size = KiB(4)},
{.offset = KiB(20), .size = KiB(4)}}));
}

TEST(HoleFillPlannerTest, UsesOnlyTheCurrentPhysicalEofBlock) {
const FileRange block {.offset = KiB(1024), .size = KiB(300)};
std::vector<FileRange> ranges;

ASSERT_TRUE(HoleFillPlanner::plan(block, {{.offset = KiB(1124), .size = KiB(100)}}, kOptions,
&ranges)
.ok());

EXPECT_EQ(ranges, (std::vector<FileRange> {{.offset = KiB(1024), .size = KiB(100)},
{.offset = KiB(1224), .size = KiB(100)}}));
for (const auto& range : ranges) {
EXPECT_GE(range.offset, block.offset);
EXPECT_LE(range.end(), block.end());
}
}

TEST(HoleFillPlannerTest, HandlesEmptyAndCompleteCoverage) {
const FileRange block {.offset = 0, .size = KiB(1024)};
std::vector<FileRange> ranges;

ASSERT_TRUE(HoleFillPlanner::plan(block, {}, kOptions, &ranges).ok());
EXPECT_EQ(ranges, (std::vector<FileRange> {block}));

ASSERT_TRUE(HoleFillPlanner::plan(block, {block}, kOptions, &ranges).ok());
EXPECT_TRUE(ranges.empty());
}

TEST(HoleFillPlannerTest, KeepsSingleHoleLargerThanCoalesceLimit) {
const FileRange block {.offset = 0, .size = KiB(1024)};
const FileRangeCoalesceOptions options {
.max_gap_bytes = KiB(32),
.max_range_bytes = KiB(128),
.max_read_amplification_ratio = 2.0,
};
std::vector<FileRange> ranges;

ASSERT_TRUE(HoleFillPlanner::plan(block, {}, options, &ranges).ok());
EXPECT_EQ(ranges, (std::vector<FileRange> {block}));
}

TEST(HoleFillPlannerTest, RejectsCoverageOutsideBlockWithoutChangingOutput) {
const FileRange block {.offset = KiB(1024), .size = KiB(1024)};
std::vector<FileRange> ranges {{.offset = 7, .size = 11}};

EXPECT_FALSE(HoleFillPlanner::plan(block, {{.offset = KiB(512), .size = KiB(1024)}}, kOptions,
&ranges)
.ok());

EXPECT_EQ(ranges, (std::vector<FileRange> {{.offset = 7, .size = 11}}));
}

} // namespace
} // namespace doris::io