Skip to content

Commit 75ab5ba

Browse files
authored
[fix](be) Fix ORC condition-cache seeks past filtered stripe suffixes (apache#70079)
### What problem does this PR solve? Related PR: apache/doris-thirdparty#431 (merge the ORC dependency fix first) A condition-cache granule can start in a stripe's SARG-filtered suffix while containing a matching row in the next stripe. On an initialized stripe, ORC's `seekToRow()` uses the stripe-end sentinel as a row-group index. This can access past the index when the stripe ends on an index boundary, or prematurely return EOF when the last row group is partial. The same-stripe fast path originates in [Apache ORC ORC-1170](apache/orc#1114); the same missing stripe-end handling remains in the current upstream C++ reader. Pin the ORC fix that initializes the next selected stripe before positioning its column readers. Preserve SARG pruning, lazy reads and split boundaries. Add a Doris reader regression and a 1.5 KB synthetic ORC fixture with its generator. The fixture has two 20,000-row stripes, a 10,000-row index stride, and one matching row at the start of each stripe. A cache hit seeks to row 18,432 in the first stripe's filtered suffix. The SQL regression compares cache-disabled and cached results with lazy reads and min/max pruning enabled or disabled, and checks the profile for actual cache hits and skipped rows. ### Release note Fix potential crashes and missing rows when ORC condition-cache seeks reach a SARG-filtered stripe suffix. ### Check List (For Author) - Test: - [x] Unit Test: 247 ORC-related BE tests passed under AddressSanitizer, including the new Doris condition-cache case and 32 dependency boundary cases. - [x] Unit Test: 56 focused standalone ORC tests passed under AddressSanitizer. Reverting the fix makes all 16 initialized-stripe boundary cases fail. - [ ] Regression test: SQL suite added; Groovy compilation checked. The external SQL suite has not been executed locally. - [x] clang-format 16 and BE build-hygiene checks passed. - [ ] Local full-file clang-tidy analysis did not complete within ten minutes; no completed result is available. - Behavior changed: - [x] Yes. Safely advance to the next matching stripe or EOF. - Does this need documentation? - [x] No.
1 parent ec7f806 commit 75ab5ba

5 files changed

Lines changed: 228 additions & 1 deletion

File tree

‎be/test/format_v2/orc/orc_reader_test.cpp‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6343,6 +6343,72 @@ TEST_F(NewOrcReaderTest, ConditionCacheHitSkipsFalseGranulesBeforeColumnRead) {
63436343
EXPECT_EQ(rows, 0);
63446344
}
63456345

6346+
TEST_F(NewOrcReaderTest, ConditionCacheHitSeeksPastSargFilteredStripeSuffix) {
6347+
const auto file_path = find_repo_file(
6348+
"regression-test/data/external_table_p0/tvf/test_data/condition_cache_seek.orc");
6349+
ASSERT_FALSE(file_path.empty());
6350+
for (const bool lazy : {false, true}) {
6351+
for (const bool min_max : {false, true}) {
6352+
SCOPED_TRACE(fmt::format("lazy={}, min_max={}", lazy, min_max));
6353+
auto io_ctx = std::make_shared<io::IOContext>();
6354+
auto reader = create_reader_for_path(file_path.string(), nullptr, io_ctx);
6355+
TQueryOptions options;
6356+
options.__set_enable_orc_lazy_mat(lazy);
6357+
options.__set_enable_orc_filter_by_min_max(min_max);
6358+
RuntimeState state {options, TQueryGlobals()};
6359+
ASSERT_TRUE(reader->init(&state).ok());
6360+
reader->set_batch_size(ConditionCacheContext::GRANULE_SIZE);
6361+
6362+
std::vector<format::ColumnDefinition> schema;
6363+
ASSERT_TRUE(reader->get_schema(&schema).ok());
6364+
auto request = std::make_shared<format::FileScanRequest>();
6365+
request->predicate_columns = {field_projection(0)};
6366+
request->non_predicate_columns = {field_projection(1)};
6367+
request->local_positions = {{format::LocalColumnId(0), format::LocalIndex(0)},
6368+
{format::LocalColumnId(1), format::LocalIndex(1)}};
6369+
request->conjuncts.push_back(VExprContext::create_shared(
6370+
std::make_shared<NullableInt32GreaterThanExpr>(0, 5000)));
6371+
ASSERT_TRUE(reader->open(request).ok());
6372+
6373+
auto ctx = std::make_shared<ConditionCacheContext>();
6374+
ctx->is_hit = true;
6375+
ctx->filter_result = std::make_shared<std::vector<bool>>(20, false);
6376+
(*ctx->filter_result)[0] = true;
6377+
(*ctx->filter_result)[9] = true;
6378+
reader->set_condition_cache_context(ctx);
6379+
int seeks = 0;
6380+
ScopedDebugPoint debug_point(
6381+
"OrcReader._skip_condition_cache_false_granules.before_seek_to_row",
6382+
[&]() { ++seeks; });
6383+
6384+
// Granule 9 contains the next stripe's matching row, but starts in the
6385+
// current stripe's filtered suffix. Both columns must resume in stripe 2.
6386+
std::vector<int32_t> ids;
6387+
std::vector<int64_t> payloads;
6388+
bool eof = false;
6389+
while (!eof) {
6390+
Block block = build_file_block(schema);
6391+
size_t rows = 0;
6392+
ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok());
6393+
const auto& id_column = assert_cast<const ColumnInt32&>(
6394+
assert_cast<const ColumnNullable&>(*block.get_by_position(0).column)
6395+
.get_nested_column());
6396+
const auto& payload_column = assert_cast<const ColumnInt64&>(
6397+
assert_cast<const ColumnNullable&>(*block.get_by_position(1).column)
6398+
.get_nested_column());
6399+
for (size_t row = 0; row < rows; ++row) {
6400+
ids.push_back(id_column.get_element(row));
6401+
payloads.push_back(payload_column.get_element(row));
6402+
}
6403+
}
6404+
EXPECT_EQ(ids, (std::vector<int32_t> {10000, 10001}));
6405+
EXPECT_EQ(payloads, (std::vector<int64_t> {0, 20000}));
6406+
EXPECT_GT(seeks, 0);
6407+
EXPECT_GT(io_ctx->condition_cache_filtered_rows, 0);
6408+
}
6409+
}
6410+
}
6411+
63466412
TEST_F(NewOrcReaderTest, ConditionCacheSeekReturnsCleanEofWhenCancelled) {
63476413
constexpr int64_t row_count = ConditionCacheContext::GRANULE_SIZE * 2;
63486414
const auto file_path = (_test_dir / "condition_cache_cancelled_seek.orc").string();
Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#include <memory>
19+
#include <orc/OrcFile.hh>
20+
#include <orc/Writer.hh>
21+
22+
// Link against the ORC library used by Doris and pass the output fixture path.
23+
// Each stripe has 20,000 rows and two 10,000-row index groups. Only its first row
24+
// matches id > 5000. The second match makes cache granule [18432, 20480) true,
25+
// so a cache hit seeks into the first stripe's SARG-filtered suffix at row 18432.
26+
int main(int argc, char** argv) {
27+
if (argc != 2) {
28+
return 1;
29+
}
30+
auto stream = orc::writeLocalFile(argv[1]);
31+
auto type = std::unique_ptr<orc::Type>(
32+
orc::Type::buildTypeFromString("struct<id:int,payload:bigint>"));
33+
orc::WriterOptions options;
34+
options.setStripeSize(1).setRowIndexStride(10000).setCompression(orc::CompressionKind_NONE);
35+
auto writer = orc::createWriter(*type, stream.get(), options);
36+
auto batch = writer->createRowBatch(20000);
37+
auto& root = dynamic_cast<orc::StructVectorBatch&>(*batch);
38+
auto& ids = dynamic_cast<orc::LongVectorBatch&>(*root.fields[0]);
39+
auto& payloads = dynamic_cast<orc::LongVectorBatch&>(*root.fields[1]);
40+
for (int64_t stripe = 0; stripe < 2; ++stripe) {
41+
for (int64_t row = 0; row < 20000; ++row) {
42+
ids.data[row] = row == 0 ? 10000 + stripe : 0;
43+
payloads.data[row] = stripe * 20000 + row;
44+
}
45+
root.numElements = ids.numElements = payloads.numElements = 20000;
46+
writer->add(*batch);
47+
}
48+
writer->close();
49+
stream->close();
50+
return 0;
51+
}
Binary file not shown.
Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
import com.azure.storage.blob.BlobClientBuilder
19+
import com.azure.storage.common.StorageSharedKeyCredential
20+
import org.apache.doris.regression.action.ProfileAction
21+
22+
suite("test_orc_condition_cache_seek", "p0,external") {
23+
String provider = getS3Provider()
24+
String endpoint = getS3Endpoint()
25+
if (provider.equalsIgnoreCase("AZURE")) {
26+
if (endpoint == "azure-endpoint") {
27+
endpoint = "https://${getS3AK()}.blob.core.windows.net"
28+
} else if (!endpoint.contains("://")) {
29+
endpoint = "https://${endpoint}"
30+
}
31+
}
32+
String bucket = context.config.otherConfigs.get("s3BucketName")
33+
String key = "orc_condition_cache_seek/${UUID.randomUUID()}/values.orc"
34+
File fixture = new File(context.dataPath, "test_data/condition_cache_seek.orc")
35+
def blobClient = null
36+
def s3Client = null
37+
boolean uploaded = false
38+
Map<String, String> originalSettings = [:]
39+
for (String variable : ["enable_condition_cache", "enable_orc_lazy_materialization",
40+
"enable_orc_filter_by_min_max", "batch_size", "enable_sql_cache",
41+
"enable_query_cache", "enable_profile", "profile_level", "parallel_pipeline_task_num",
42+
"max_file_scanners_concurrency"]) {
43+
originalSettings[variable] = sql("SHOW VARIABLES LIKE '${variable}'")[0][1].toString()
44+
}
45+
try {
46+
if (provider.equalsIgnoreCase("AZURE")) {
47+
blobClient = new BlobClientBuilder().endpoint(endpoint).containerName(bucket)
48+
.blobName(key).credential(new StorageSharedKeyCredential(getS3AK(), getS3SK()))
49+
.buildClient()
50+
blobClient.uploadFromFile(fixture.absolutePath)
51+
} else {
52+
s3Client = getS3Client()
53+
s3Client.putObject(bucket, key, fixture)
54+
}
55+
uploaded = true
56+
String source = """S3('uri'='s3://${bucket}/${key}', 'format'='orc',
57+
'provider'='${provider}', 's3.endpoint'='${endpoint}', 's3.region'='${getS3Region()}',
58+
's3.access_key'='${getS3AK()}', 's3.secret_key'='${getS3SK()}')"""
59+
sql "SET enable_sql_cache = false"
60+
sql "SET enable_query_cache = false"
61+
sql "SET enable_profile = true"
62+
sql "SET profile_level = 2"
63+
sql "SET parallel_pipeline_task_num = 1"
64+
sql "SET max_file_scanners_concurrency = 1"
65+
sql "SET batch_size = 2048"
66+
// A full LRU-K shard needs one scan for ghost history and another for admission.
67+
// Allow a third visit to one backend even when scans are spread across all backends.
68+
int attempts = 2 * sql("SHOW BACKENDS").size() + 1
69+
for (boolean minMax : [false, true]) {
70+
sql "SET enable_orc_filter_by_min_max = ${minMax}"
71+
for (boolean lazy : [false, true]) {
72+
sql "SET enable_orc_lazy_materialization = ${lazy}"
73+
sql "SET enable_condition_cache = false"
74+
quickTest("uncached", "SELECT id, payload FROM ${source} WHERE id > 5000 ORDER BY id",
75+
"SELECT 10000, CAST(0 AS BIGINT) UNION ALL SELECT 10001, CAST(20000 AS BIGINT) ORDER BY 1")
76+
sql "SET enable_condition_cache = true"
77+
boolean cacheHit = false
78+
for (int attempt = 0; attempt < attempts && !cacheHit; ++attempt) {
79+
String token = "orc_seek_${UUID.randomUUID().toString().replace('-', '')}"
80+
// The first match populates granule 0. The next stripe's match populates
81+
// granule 9, whose start (18432) lies in the first stripe's pruned suffix.
82+
quickTest("cached",
83+
"SELECT id, payload FROM (SELECT id, payload, '${token}' AS marker " +
84+
"FROM ${source} WHERE id > 5000) t ORDER BY id",
85+
"SELECT 10000, CAST(0 AS BIGINT) UNION ALL " +
86+
"SELECT 10001, CAST(20000 AS BIGINT) ORDER BY 1")
87+
String profile = new ProfileAction(context).getProfileBySql(token,
88+
["ConditionCacheHit", "ConditionCacheFilteredRows"])
89+
cacheHit = (profile =~ /(?m)ConditionCacheHit:\s*[1-9]/).find()
90+
if (cacheHit) {
91+
assertTrue((profile =~ /(?m)ConditionCacheFilteredRows:\s*[1-9]/).find(), profile)
92+
}
93+
}
94+
assertTrue(cacheHit, "No condition cache hit after ${attempts} scans")
95+
}
96+
}
97+
} finally {
98+
try {
99+
originalSettings.each { variable, value -> sql "SET ${variable} = ${value}" }
100+
} finally {
101+
if (uploaded) {
102+
if (blobClient != null) {
103+
blobClient.delete()
104+
} else {
105+
s3Client.deleteObject(bucket, key)
106+
}
107+
}
108+
}
109+
}
110+
}

0 commit comments

Comments
 (0)