Skip to content

Commit 3c060af

Browse files
authored
[fix](ci) Deliver non-converged reviews at the round limit (apache#68996)
Problem Summary: The review runner simultaneously required unanimous `NO_NEW_VALUABLE_FINDINGS` before submission and required submission after at most three rounds. When round 3 still produced candidates, the agent could mark its goal blocked and withhold verified findings. This happened in the reviews of [apache#67549](https://github.com/apache/doris/actions/runs/38222336278) and [apache#68499](https://github.com/apache/doris/actions/runs/38271916203). Separate verified delivery from convergence: - Submit after convergence or the third round. At the cap, publish verified findings and disclose remaining questions or coverage gaps; unverified candidates must not become inline findings. - Require `converged`, `rounds`, and `incomplete_reason` in the submission. Non-convergence requires round 3 and a nonempty reason. The trusted helper renders this declaration into the review body and binds it to the existing digest and exact API readback. - A verified delivery completes execution, but non-convergence produces a failing review status even with zero P0/P1. Both the source and aggregate status explain non-convergence. Reviews without P0/P1 remain GitHub comments; no artificial blocking finding is created. - Preserve head/base validation, submission deduplication, capacity recovery, stale-head cancellation, and existing authorized local/skip overrides. The prompt and helpers are loaded from the same workflow revision. Existing invocations retain their old schema; new invocations reject missing convergence fields rather than defaulting to a pass. This change does not update the agent binary or model.
1 parent 317ca22 commit 3c060af

8 files changed

Lines changed: 303 additions & 31 deletions

‎.github/scripts/resolve_code_review_status.py‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -73,19 +73,30 @@ def resolve_status(
7373

7474
approved_contexts: set[tuple[int, str]] = set()
7575
blocked_contexts: set[tuple[int, str]] = set()
76+
incomplete_contexts: set[tuple[int, str]] = set()
7677
for context, item in latest_by_context.items():
7778
match = SOURCE_CONTEXT_RE.fullmatch(context)
7879
if match is not None and item["state"] == "success":
7980
approved_contexts.add((int(match.group(1)), match.group(2).casefold()))
8081
elif (match is not None and item["state"] == "failure"
8182
and context.startswith("code-review/source/automated/")):
82-
blocked_contexts.add((int(match.group(1)), match.group(2).casefold()))
83+
key = (int(match.group(1)), match.group(2).casefold())
84+
blocked_contexts.add(key)
85+
# State remains the sole gating signal. Use the runner's description
86+
# only to explain an already-blocking result, never to grant a pass.
87+
description = item.get("description")
88+
if isinstance(description, str) and description.startswith("Review did not converge after "):
89+
incomplete_contexts.add(key)
8390

8491
# Preserve the authorized local PASS / skip override. Otherwise a completed
85-
# automated review with P0/P1 blocks, rather than appearing to still be running.
92+
# automated review with P0/P1 or incomplete coverage blocks instead of pending.
8693
blocked = sorted((open_contexts - approved_contexts) & blocked_contexts)
8794
if blocked:
8895
number, base_sha = blocked[0]
96+
if (number, base_sha) in incomplete_contexts:
97+
return Resolution(
98+
"failure", f"Review did not converge for PR #{number} at {base_sha[:12]}.",
99+
)
89100
return Resolution(
90101
"failure", f"Review completed with P0/P1 for PR #{number} at {base_sha[:12]}.",
91102
)

‎.github/scripts/run_review_with_resume.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -568,6 +568,7 @@ def fail(message):
568568
print(
569569
f"Verified final review {result['review_id']}: "
570570
f"P0={result['p0']}, P1={result['p1']}, "
571+
f"converged={result['converged']}, rounds={result['rounds']}, "
571572
f"recovered_after_capacity={result['recovered_after_capacity']}",
572573
file=sys.stderr, flush=True,
573574
)

‎.github/scripts/submit_review.py‎

Lines changed: 30 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -49,9 +49,25 @@ def priority(body):
4949

5050
def validate_submission(value):
5151
if not isinstance(value, dict) or set(value) != {
52-
"body", "comments", "existing_blocking_comment_ids"
52+
"body", "comments", "existing_blocking_comment_ids",
53+
"converged", "rounds", "incomplete_reason"
5354
}:
54-
raise ValueError("Final review requires body, comments and existing_blocking_comment_ids")
55+
raise ValueError(
56+
"Final review requires body, comments, existing_blocking_comment_ids, "
57+
"converged, rounds and incomplete_reason"
58+
)
59+
if type(value["converged"]) is not bool:
60+
raise ValueError("converged must be a boolean")
61+
if type(value["rounds"]) is not int or not 1 <= value["rounds"] <= 3:
62+
raise ValueError("rounds must be an integer from 1 to 3")
63+
reason = value["incomplete_reason"]
64+
if not isinstance(reason, str):
65+
raise ValueError("incomplete_reason must be text")
66+
if value["converged"]:
67+
if reason.strip():
68+
raise ValueError("A converged review cannot have an incomplete_reason")
69+
elif value["rounds"] != 3 or not reason.strip():
70+
raise ValueError("A non-converged review requires three rounds and an incomplete_reason")
5571
if not isinstance(value["body"], str) or not value["body"].strip():
5672
raise ValueError("Final review summary is empty")
5773
if not isinstance(value["comments"], list):
@@ -119,6 +135,15 @@ def review_payload(submission, run, root, remaining):
119135
digest = hashlib.sha256(json.dumps(submission, sort_keys=True).encode()).hexdigest()
120136
marker = f"<!-- code-review-completion:{run['token']}:{digest} -->"
121137
body = submission["body"]
138+
# Render the declared convergence state ourselves and bind it to the same
139+
# digest/readback as the findings. A delivered review can still be incomplete.
140+
if submission["converged"]:
141+
body += f"\n\nReview converged after {submission['rounds']} round(s)."
142+
else:
143+
body += (
144+
"\n\n**Review did not converge after 3 rounds. This automated review cannot pass the review check.**"
145+
f"\n\n{submission['incomplete_reason']}"
146+
)
122147
if ids:
123148
body += "\n\nExisting P0/P1 findings confirmed for this head: " + ", ".join(
124149
f"https://github.com/{run['repository']}/pull/{run['pr_number']}#discussion_r{i}"
@@ -187,7 +212,9 @@ def verify_completion(context, run, remaining, *, submit=False, submission=None)
187212
# Recheck after the API reads so a moving PR cannot pass an obsolete review.
188213
check_target(run, remaining)
189214
return {
190-
"state": "failure" if counts[0] + counts[1] else "success",
215+
"state": "failure" if counts[0] + counts[1] or not submission["converged"] else "success",
216+
"converged": submission["converged"], "rounds": submission["rounds"],
217+
"incomplete_reason": submission["incomplete_reason"],
191218
"p0": counts[0], "p1": counts[1], "review_id": review["id"],
192219
"review_url": review["html_url"],
193220
}

‎.github/scripts/test_resolve_code_review_status.py‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,36 @@ def test_completed_blocking_review_is_failure(self) -> None:
115115
self.assertEqual("failure", result.state)
116116
self.assertIn("P0/P1", result.description)
117117

118+
def test_nonconverged_result_blocks_with_an_explicit_description(self) -> None:
119+
incomplete = dict(status(1, source="automated", state="failure"),
120+
description="Review did not converge after 3 rounds for PR #123: P0=0, P1=0.")
121+
result = resolve_status([pull(123)], [incomplete], head_sha=HEAD_SHA)
122+
self.assertEqual("failure", result.state)
123+
self.assertIn("did not converge", result.description)
124+
self.assertNotIn("P0/P1", result.description)
125+
126+
def test_description_does_not_decide_whether_review_passes(self) -> None:
127+
for state in ("success", "pending", "failure"):
128+
with self.subTest(state=state):
129+
source = dict(status(1, source="automated", state=state),
130+
description="Review did not converge after 3 rounds")
131+
self.assertEqual(state, resolve_status([pull(123)], [source], head_sha=HEAD_SHA).state)
132+
133+
def test_incomplete_result_can_be_replaced_or_explicitly_overridden(self) -> None:
134+
incomplete = dict(status(1, source="automated", state="failure"),
135+
description="Review did not converge after 3 rounds")
136+
for source in ("automated", "local", "skip"):
137+
with self.subTest(source=source):
138+
result = resolve_status([pull(123)], [incomplete, status(2, source=source)], head_sha=HEAD_SHA)
139+
self.assertEqual("success", result.state)
140+
141+
def test_incomplete_result_does_not_follow_other_heads_or_bases(self) -> None:
142+
incomplete = dict(status(1, source="automated", state="failure"),
143+
description="Review did not converge after 3 rounds")
144+
for pulls in ([pull(124)], [pull(123, base=OTHER_BASE_SHA)], [pull(123, head=OTHER_HEAD_SHA)]):
145+
with self.subTest(pulls=pulls):
146+
self.assertEqual("pending", resolve_status(pulls, [incomplete], head_sha=HEAD_SHA).state)
147+
118148
def test_local_or_skip_success_can_override_blocking_review(self) -> None:
119149
for source in ("local", "skip"):
120150
with self.subTest(source=source):
Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
#!/usr/bin/env python3
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
19+
import json
20+
import os
21+
import re
22+
import shutil
23+
import subprocess
24+
import tempfile
25+
import textwrap
26+
import unittest
27+
from pathlib import Path
28+
29+
from resolve_code_review_status import resolve_status
30+
from submit_review import validate_submission
31+
32+
33+
WORKFLOW = Path(__file__).resolve().parents[1] / "workflows/code-review-runner.yml"
34+
35+
36+
def step_script(name):
37+
step = WORKFLOW.read_text().split(f" - name: {name}\n", 1)[1].split("\n - name:", 1)[0]
38+
script = textwrap.dedent(step.split(" run: |\n", 1)[1])
39+
return re.sub(r"\$\{\{.*?\}\}", "test", script)
40+
41+
42+
class ReviewPromptContractTest(unittest.TestCase):
43+
def test_prompt_schema_supports_both_delivery_outcomes(self):
44+
prompt = WORKFLOW.read_text().split("<<'PROMPT'\n", 1)[1].split(" PROMPT\n", 1)[0]
45+
example = next(line.strip() for line in prompt.splitlines() if line.strip().startswith('{"body":'))
46+
submission = json.loads(example.replace("<diff_position>", "1"))
47+
validate_submission(submission)
48+
validate_submission(dict(submission, converged=False, rounds=3,
49+
incomplete_reason="Round 3 still found candidates."))
50+
# Regression guard for the clauses that blocked #67549 and #68499.
51+
for obsolete in (
52+
"can only be submitted after all subagents return",
53+
"MUST remain incomplete until every suspicious point",
54+
"Only finish when there are no unresolved suspicious points",
55+
"declare the COMPLETE final review",
56+
"Submit only after all review completion criteria are satisfied",
57+
):
58+
self.assertNotIn(obsolete, prompt)
59+
self.assertIn("do not start a fourth round", prompt)
60+
self.assertIn("MUST NOT prevent delivery of verified findings", prompt)
61+
62+
63+
@unittest.skipUnless(shutil.which("jq"), "workflow shell test requires jq")
64+
class ReviewConvergenceWorkflowTest(unittest.TestCase):
65+
def test_verified_result_flows_through_source_and_aggregate_status(self):
66+
output_script = step_script("Run automated code review").split("status=$?\n", 1)[1]
67+
source_script = step_script("Sync Code Review check for current head")
68+
# Replace the only external mutation with an argument capture. Run the
69+
# actual Bash/jq blocks, then feed their source into the real resolver.
70+
capture = """gh() {
71+
python3 -c 'import json, sys; print(json.dumps(sys.argv[1:]))' "$@"
72+
}
73+
"""
74+
for converged, p0, p1, expected in (
75+
(True, 0, 0, "success"),
76+
(True, 1, 0, "failure"),
77+
(True, 0, 1, "failure"),
78+
(False, 0, 0, "failure"),
79+
(False, 0, 1, "failure"),
80+
):
81+
with self.subTest(converged=converged, p0=p0, p1=p1), tempfile.TemporaryDirectory() as tmp:
82+
root = Path(tmp)
83+
(root / "review-result.json").write_text(json.dumps({
84+
"state": expected, "p0": p0, "p1": p1, "converged": converged,
85+
"rounds": 3, "incomplete_reason": "" if converged else "Coverage incomplete",
86+
}))
87+
env = {**os.environ, "status": "0", "REVIEW_CONTEXT_DIR": tmp,
88+
"GITHUB_OUTPUT": str(root / "output"), "GITHUB_STEP_SUMMARY": str(root / "summary")}
89+
subprocess.run(["bash", "-e", "-c", output_script], env=env,
90+
capture_output=True, text=True, check=True)
91+
outputs = dict(line.split("=", 1) for line in (root / "output").read_text().splitlines())
92+
self.assertEqual(str(converged).lower(), outputs["converged"])
93+
result = subprocess.run(["bash", "-e", "-c", capture + source_script], env={
94+
**env, "BASE_SHA": "b" * 40, "HEAD_SHA": "a" * 40,
95+
"PR_NUMBER": "123", "REPO": "apache/doris", "JOB_STATUS": "success",
96+
"REVIEW_CONTEXT_OUTCOME": "success", "REVIEW_OUTCOME": "success",
97+
"REVIEW_STATE": outputs["review_state"], "REVIEW_P0": outputs["p0"],
98+
"REVIEW_P1": outputs["p1"], "REVIEW_CONVERGED": outputs["converged"],
99+
"REVIEW_ROUNDS": outputs["rounds"],
100+
}, capture_output=True, text=True, check=True)
101+
args = json.loads(result.stdout)
102+
posted = dict(args[i + 1].split("=", 1) for i in range(len(args)) if args[i] == "-f")
103+
self.assertEqual(expected, posted["state"])
104+
self.assertIn("repos/apache/doris/statuses/" + "a" * 40, args)
105+
resolution = resolve_status([{
106+
"state": "open", "number": 123, "head": {"sha": "a" * 40}, "base": {"sha": "b" * 40},
107+
}], [{"id": 1, **posted}], head_sha="a" * 40)
108+
self.assertEqual(expected, resolution.state)
109+
if not converged:
110+
self.assertIn("did not converge", posted["description"])
111+
self.assertIn("did not converge", resolution.description)
112+
else:
113+
self.assertNotIn("did not converge", resolution.description)
114+
115+
116+
if __name__ == "__main__":
117+
unittest.main()

‎.github/scripts/test_run_review_with_resume.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,7 @@ def setUp(self):
118118
self.verification = self.enterContext(mock.patch.object(
119119
runner, "verify_completion", return_value={
120120
"state": "success", "p0": 0, "p1": 0, "review_id": 123,
121+
"converged": True, "rounds": 1, "incomplete_reason": "",
121122
}
122123
))
123124

@@ -1362,6 +1363,7 @@ def test_real_process_capacity_then_resume_preserves_state(self):
13621363
mock.patch.object(runner, "check_resume_target"),
13631364
mock.patch.object(runner, "verify_completion", return_value={
13641365
"state": "success", "p0": 0, "p1": 0, "review_id": 123,
1366+
"converged": True, "rounds": 1, "incomplete_reason": "",
13651367
}),
13661368
):
13671369
self.assertEqual(0, runner.run_review(args))

0 commit comments

Comments
 (0)