Skip to content

Commit 7881be5

Browse files
committed
Merge remote-tracking branch 'origin/master' into fix-keyword-match-score
# Conflicts: # be/test/storage/segment/index_reader_helper_test.cpp
2 parents db7d9a8 + 6991e76 commit 7881be5

460 files changed

Lines changed: 22338 additions & 1817 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.
Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
#!/usr/bin/env python3
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
19+
import ast
20+
import re
21+
import unittest
22+
from pathlib import Path
23+
24+
from resolve_code_review_status import resolve_status
25+
26+
27+
WORKFLOWS = Path(__file__).resolve().parents[1] / "workflows"
28+
29+
30+
def expression_value(expression, github):
31+
"""Evaluate the string/boolean subset used by these checkout expressions."""
32+
expression = expression.strip().removeprefix("${{").removesuffix("}}").strip()
33+
tree = ast.parse(expression.replace("&&", " and ").replace("||", " or "), mode="eval")
34+
35+
def visit(node):
36+
if isinstance(node, ast.Constant) and isinstance(node.value, str):
37+
return node.value
38+
if isinstance(node, ast.Name) and node.id == "github":
39+
return github
40+
if isinstance(node, ast.Attribute):
41+
return visit(node.value).get(node.attr, "")
42+
if isinstance(node, ast.BoolOp):
43+
for item in node.values:
44+
value = visit(item)
45+
if isinstance(node.op, ast.And) and not value:
46+
return value
47+
if isinstance(node.op, ast.Or) and value:
48+
return value
49+
return value
50+
if isinstance(node, ast.Compare) and len(node.ops) == 1:
51+
left, right = visit(node.left), visit(node.comparators[0])
52+
if isinstance(node.ops[0], ast.Eq):
53+
return left == right
54+
if isinstance(node.ops[0], ast.NotEq):
55+
return left != right
56+
raise AssertionError(f"Unsupported workflow expression: {ast.dump(node)}")
57+
58+
return visit(tree.body)
59+
60+
61+
class ReviewCheckoutWorkflowTest(unittest.TestCase):
62+
def setUp(self):
63+
workflow = (WORKFLOWS / "code-review-aggregate-status.yml").read_text()
64+
self.step = workflow.split(" - name: Checkout trusted GitHub API diagnostics\n", 1)[1]
65+
self.step = self.step.split("\n - name:", 1)[0]
66+
self.inputs = dict(re.findall(r"^ ([\w-]+): (.*)$", self.step, re.MULTILINE))
67+
68+
def ref(self, github):
69+
return expression_value(self.inputs["ref"], github)
70+
71+
def pr_event(self, *, action="closed", merged=True, same_repo=False, sha="b" * 40):
72+
return {
73+
"event_name": "pull_request_target",
74+
"repository": "apache/doris",
75+
"ref": "refs/heads/master",
76+
"sha": sha,
77+
"workflow_sha": sha,
78+
"event": {
79+
"action": action,
80+
"repository": {"id": 1, "default_branch": "master"},
81+
"pull_request": {
82+
"state": "closed" if action == "closed" else "open",
83+
"merged": merged,
84+
"head": {"sha": "a" * 40, "repo": {
85+
"id": 1 if same_repo else 2,
86+
"full_name": "apache/doris" if same_repo else "contributor/doris",
87+
}},
88+
"merge_commit_sha": sha if merged else "c" * 40,
89+
},
90+
},
91+
}
92+
93+
def assert_trusted_default_checkout(self, github):
94+
# An empty ref on the workflow repository makes checkout use the
95+
# event's trusted ref/SHA. An explicit merge SHA enters its fork guard.
96+
self.assertEqual("", self.ref(github))
97+
self.assertNotIn("repository", self.inputs)
98+
self.assertEqual("refs/heads/master", github["ref"])
99+
self.assertNotEqual(github["event"]["pull_request"]["head"]["sha"], github["sha"])
100+
101+
def test_merged_fork_pr_uses_default_checkout_even_when_workflow_sha_is_merge_sha(self):
102+
# The two production failures: PRs #69223 and #67138.
103+
for sha in ("707ed45ee8cfde440a1c14ea85b78bc8be9339ce",
104+
"e779124b5f60533088730175957944819d22adcb"):
105+
with self.subTest(sha=sha):
106+
github = self.pr_event(sha=sha)
107+
self.assertEqual(github["workflow_sha"],
108+
github["event"]["pull_request"]["merge_commit_sha"])
109+
self.assert_trusted_default_checkout(github)
110+
111+
def test_other_pr_target_events_keep_trusted_event_checkout(self):
112+
for action in ("opened", "synchronize", "reopened", "ready_for_review", "edited", "closed"):
113+
for same_repo in (False, True):
114+
with self.subTest(action=action, same_repo=same_repo):
115+
self.assert_trusted_default_checkout(
116+
self.pr_event(action=action, merged=False, same_repo=same_repo))
117+
118+
def test_other_callers_keep_workflow_sha_pinning_and_sha_fallback(self):
119+
for event in ("issue_comment", "workflow_dispatch"):
120+
for workflow_sha in ("b" * 40, ""):
121+
with self.subTest(event=event, workflow_sha=workflow_sha):
122+
self.assertEqual(workflow_sha or "c" * 40, self.ref({
123+
"event_name": event, "workflow_sha": workflow_sha, "sha": "c" * 40,
124+
}))
125+
126+
def test_checkout_does_not_opt_out_of_fork_guard_or_persist_credentials(self):
127+
self.assertEqual("false", self.inputs["persist-credentials"])
128+
self.assertEqual("false", self.inputs.get("allow-unsafe-pr-checkout", "false"))
129+
self.assertEqual(".review-api-source", self.inputs["path"])
130+
self.assertEqual(".github/scripts/review_github_api.py", self.inputs["sparse-checkout"])
131+
132+
def test_closed_pr_still_reaches_aggregate_and_releases_shared_head_context(self):
133+
sync = (WORKFLOWS / "code-review-sync-result.yml").read_text()
134+
triggers = sync.split(" pull_request_target:\n", 1)[1].split(" issue_comment:", 1)[0]
135+
self.assertIn("closed", triggers)
136+
caller = sync.split("\n sync-status:\n", 1)[1].split("\n sync-previous-head-status:", 1)[0]
137+
self.assertIn("uses: ./.github/workflows/code-review-aggregate-status.yml", caller)
138+
condition = re.search(r"^ if: (.*)$", caller, re.MULTILINE).group(1)
139+
github = self.pr_event()
140+
self.assertTrue(expression_value(condition, github))
141+
self.assert_trusted_default_checkout(github)
142+
head = github["event"]["pull_request"]["head"]["sha"]
143+
# A merged PR must no longer hold another open PR at the same head pending.
144+
pulls = [{"number": n, "state": state, "head": {"sha": head}, "base": {"sha": "d" * 40}}
145+
for n, state in ((1, "closed"), (2, "open"))]
146+
sources = [{"id": 1, "state": "success",
147+
"context": "code-review/source/automated/pr-2/base-" + "d" * 40}]
148+
self.assertEqual("pending", resolve_status(
149+
[dict(p, state="open") for p in pulls], sources, head_sha=head).state)
150+
self.assertEqual("success", resolve_status(pulls, sources, head_sha=head).state)
151+
152+
153+
if __name__ == "__main__":
154+
unittest.main()

‎.github/workflows/code-review-aggregate-status.yml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,10 @@ jobs:
2626
timeout-minutes: 2
2727
uses: actions/checkout@v4
2828
with:
29-
ref: ${{ github.workflow_sha || github.sha }}
29+
# PR target events already provide a trusted default-branch ref/SHA.
30+
# Passing that SHA explicitly after merge trips checkout's fork guard
31+
# when it equals the PR merge commit. Other callers stay SHA-pinned.
32+
ref: ${{ github.event_name != 'pull_request_target' && (github.workflow_sha || github.sha) || '' }}
3033
persist-credentials: false
3134
path: .review-api-source
3235
sparse-checkout: .github/scripts/review_github_api.py

‎.licenserc.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,13 @@ header:
9696
- "fe/fe-core/src/test/resources/upgrade/413/labels.db.txt"
9797
- "fe/fe-core/src/test/resources/upgrade/413/labels.ds.txt"
9898
- "fe/fe-core/src/test/resources/upgrade/413/labels.tbl.txt"
99+
# Same for the golden 4.1.5 (Lance catalog) fixtures emitted by Gen415Fixtures;
100+
# upgrade/415/generator/Gen415Fixtures.java stays checked.
101+
- "fe/fe-core/src/test/resources/upgrade/415/PROVENANCE.txt"
102+
- "fe/fe-core/src/test/resources/upgrade/415/editlog.index"
103+
- "fe/fe-core/src/test/resources/upgrade/415/labels.db.txt"
104+
- "fe/fe-core/src/test/resources/upgrade/415/labels.ds.txt"
105+
- "fe/fe-core/src/test/resources/upgrade/415/labels.tbl.txt"
99106
- "fe/fe-core/src/main/java/software/amazon/awssdk/core/client/builder/SdkDefaultClientBuilder.java"
100107
- "fe/fe-core/src/main/antlr4/org/apache/doris/nereids/JavaLexer.g4"
101108
- "fe/fe-core/src/main/antlr4/org/apache/doris/nereids/JavaParser.g4"

‎be/src/agent/task_worker_pool.cpp‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -868,16 +868,33 @@ void update_tablet_meta_callback(StorageEngine& engine, const TAgentTaskRequest&
868868
LOG(INFO) << "get update tablet meta task. signature=" << req.signature;
869869

870870
Status status;
871+
int64_t row_ttl_config_version = -1;
871872
const auto& update_tablet_meta_req = req.update_tablet_meta_info_req;
872873
for (const auto& tablet_meta_info : update_tablet_meta_req.tabletMetaInfos) {
873874
auto tablet = engine.tablet_manager()->get_tablet(tablet_meta_info.tablet_id);
874875
if (tablet == nullptr) {
875876
status = Status::NotFound("tablet not found");
876877
LOG(WARNING) << "could not find tablet when update tablet meta. tablet_id="
877878
<< tablet_meta_info.tablet_id;
879+
// A later success in this batch must not acknowledge a failed TTL update.
880+
if (tablet_meta_info.__isset.row_ttl_config) {
881+
break;
882+
}
878883
continue;
879884
}
880885
bool need_to_save = false;
886+
if (tablet_meta_info.__isset.row_ttl_config) {
887+
RowTtlConfigPB config;
888+
config.set_duration_us(tablet_meta_info.row_ttl_config.duration_us);
889+
config.set_config_version(tablet_meta_info.row_ttl_config.config_version);
890+
std::unique_lock lock(tablet->get_header_lock());
891+
status = tablet->tablet_meta()->update_row_ttl_config(config);
892+
if (!status.ok()) {
893+
break;
894+
}
895+
tablet->save_meta();
896+
row_ttl_config_version = config.config_version();
897+
}
881898
if (tablet_meta_info.__isset.partition_id) {
882899
// for fix partition_id = 0
883900
LOG(WARNING) << "change be tablet id: " << tablet->tablet_meta()->tablet_id()
@@ -1042,6 +1059,9 @@ void update_tablet_meta_callback(StorageEngine& engine, const TAgentTaskRequest&
10421059
LOG(INFO) << "finish update tablet meta task. signature=" << req.signature;
10431060
if (req.signature != -1) {
10441061
TFinishTaskRequest finish_task_request;
1062+
if (row_ttl_config_version >= 0) {
1063+
finish_task_request.__set_row_ttl_config_version(row_ttl_config_version);
1064+
}
10451065
finish_task_request.__set_task_status(status.to_thrift());
10461066
finish_task_request.__set_backend(BackendOptions::get_local_backend());
10471067
finish_task_request.__set_task_type(req.task_type);

‎be/src/cloud/cloud_full_compaction.cpp‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
#include <gen_cpp/cloud.pb.h>
2121

22+
#include <algorithm>
2223
#include <boost/container_hash/hash.hpp>
2324

2425
#include "cloud/cloud_meta_mgr.h"
@@ -134,12 +135,18 @@ Status CloudFullCompaction::pick_rowsets_to_compact() {
134135
DCHECK(false) << st;
135136
return st;
136137
}
137-
if (_input_rowsets.size() <= 1) {
138+
// An already compacted rowset can expire later without another write. Full compaction
139+
// is also the explicit TTL GC operation, including the initial [0-1], [2-y] layout.
140+
const bool ttl_gc = _tablet->tablet_schema()->has_ttl_col() &&
141+
std::ranges::any_of(_input_rowsets, [](const auto& rowset) {
142+
return rowset->num_rows() > 0;
143+
});
144+
if (_input_rowsets.size() <= 1 && !ttl_gc) {
138145
return Status::Error<BE_NO_SUITABLE_VERSION>(
139146
"insufficent compaction input rowset, #rowsets={}", _input_rowsets.size());
140147
}
141148

142-
if (_input_rowsets.size() == 2 && _input_rowsets[0]->end_version() == 1) {
149+
if (_input_rowsets.size() == 2 && _input_rowsets[0]->end_version() == 1 && !ttl_gc) {
143150
// the tablet is with rowset: [0-1], [2-y]
144151
// and [0-1] has no data. in this situation, no need to do full compaction.
145152
return Status::Error<BE_NO_SUITABLE_VERSION>("no suitable versions for compaction");

0 commit comments

Comments
 (0)