|
| 1 | +#!/usr/bin/env python3 |
| 2 | +# Licensed to the Apache Software Foundation (ASF) under one |
| 3 | +# or more contributor license agreements. See the NOTICE file |
| 4 | +# distributed with this work for additional information |
| 5 | +# regarding copyright ownership. The ASF licenses this file |
| 6 | +# to you under the Apache License, Version 2.0 (the |
| 7 | +# "License"); you may not use this file except in compliance |
| 8 | +# with the License. You may obtain a copy of the License at |
| 9 | +# |
| 10 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 11 | +# |
| 12 | +# Unless required by applicable law or agreed to in writing, |
| 13 | +# software distributed under the License is distributed on an |
| 14 | +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 15 | +# KIND, either express or implied. See the License for the |
| 16 | +# specific language governing permissions and limitations |
| 17 | +# under the License. |
| 18 | + |
| 19 | +import json |
| 20 | +import os |
| 21 | +import re |
| 22 | +import shutil |
| 23 | +import subprocess |
| 24 | +import tempfile |
| 25 | +import textwrap |
| 26 | +import unittest |
| 27 | +from pathlib import Path |
| 28 | + |
| 29 | +from resolve_code_review_status import resolve_status |
| 30 | +from submit_review import validate_submission |
| 31 | + |
| 32 | + |
| 33 | +WORKFLOW = Path(__file__).resolve().parents[1] / "workflows/code-review-runner.yml" |
| 34 | + |
| 35 | + |
| 36 | +def step_script(name): |
| 37 | + step = WORKFLOW.read_text().split(f" - name: {name}\n", 1)[1].split("\n - name:", 1)[0] |
| 38 | + script = textwrap.dedent(step.split(" run: |\n", 1)[1]) |
| 39 | + return re.sub(r"\$\{\{.*?\}\}", "test", script) |
| 40 | + |
| 41 | + |
| 42 | +class ReviewPromptContractTest(unittest.TestCase): |
| 43 | + def test_prompt_schema_supports_both_delivery_outcomes(self): |
| 44 | + prompt = WORKFLOW.read_text().split("<<'PROMPT'\n", 1)[1].split(" PROMPT\n", 1)[0] |
| 45 | + example = next(line.strip() for line in prompt.splitlines() if line.strip().startswith('{"body":')) |
| 46 | + submission = json.loads(example.replace("<diff_position>", "1")) |
| 47 | + validate_submission(submission) |
| 48 | + validate_submission(dict(submission, converged=False, rounds=3, |
| 49 | + incomplete_reason="Round 3 still found candidates.")) |
| 50 | + # Regression guard for the clauses that blocked #67549 and #68499. |
| 51 | + for obsolete in ( |
| 52 | + "can only be submitted after all subagents return", |
| 53 | + "MUST remain incomplete until every suspicious point", |
| 54 | + "Only finish when there are no unresolved suspicious points", |
| 55 | + "declare the COMPLETE final review", |
| 56 | + "Submit only after all review completion criteria are satisfied", |
| 57 | + ): |
| 58 | + self.assertNotIn(obsolete, prompt) |
| 59 | + self.assertIn("do not start a fourth round", prompt) |
| 60 | + self.assertIn("MUST NOT prevent delivery of verified findings", prompt) |
| 61 | + |
| 62 | + |
| 63 | +@unittest.skipUnless(shutil.which("jq"), "workflow shell test requires jq") |
| 64 | +class ReviewConvergenceWorkflowTest(unittest.TestCase): |
| 65 | + def test_verified_result_flows_through_source_and_aggregate_status(self): |
| 66 | + output_script = step_script("Run automated code review").split("status=$?\n", 1)[1] |
| 67 | + source_script = step_script("Sync Code Review check for current head") |
| 68 | + # Replace the only external mutation with an argument capture. Run the |
| 69 | + # actual Bash/jq blocks, then feed their source into the real resolver. |
| 70 | + capture = """gh() { |
| 71 | + python3 -c 'import json, sys; print(json.dumps(sys.argv[1:]))' "$@" |
| 72 | +} |
| 73 | +""" |
| 74 | + for converged, p0, p1, expected in ( |
| 75 | + (True, 0, 0, "success"), |
| 76 | + (True, 1, 0, "failure"), |
| 77 | + (True, 0, 1, "failure"), |
| 78 | + (False, 0, 0, "failure"), |
| 79 | + (False, 0, 1, "failure"), |
| 80 | + ): |
| 81 | + with self.subTest(converged=converged, p0=p0, p1=p1), tempfile.TemporaryDirectory() as tmp: |
| 82 | + root = Path(tmp) |
| 83 | + (root / "review-result.json").write_text(json.dumps({ |
| 84 | + "state": expected, "p0": p0, "p1": p1, "converged": converged, |
| 85 | + "rounds": 3, "incomplete_reason": "" if converged else "Coverage incomplete", |
| 86 | + })) |
| 87 | + env = {**os.environ, "status": "0", "REVIEW_CONTEXT_DIR": tmp, |
| 88 | + "GITHUB_OUTPUT": str(root / "output"), "GITHUB_STEP_SUMMARY": str(root / "summary")} |
| 89 | + subprocess.run(["bash", "-e", "-c", output_script], env=env, |
| 90 | + capture_output=True, text=True, check=True) |
| 91 | + outputs = dict(line.split("=", 1) for line in (root / "output").read_text().splitlines()) |
| 92 | + self.assertEqual(str(converged).lower(), outputs["converged"]) |
| 93 | + result = subprocess.run(["bash", "-e", "-c", capture + source_script], env={ |
| 94 | + **env, "BASE_SHA": "b" * 40, "HEAD_SHA": "a" * 40, |
| 95 | + "PR_NUMBER": "123", "REPO": "apache/doris", "JOB_STATUS": "success", |
| 96 | + "REVIEW_CONTEXT_OUTCOME": "success", "REVIEW_OUTCOME": "success", |
| 97 | + "REVIEW_STATE": outputs["review_state"], "REVIEW_P0": outputs["p0"], |
| 98 | + "REVIEW_P1": outputs["p1"], "REVIEW_CONVERGED": outputs["converged"], |
| 99 | + "REVIEW_ROUNDS": outputs["rounds"], |
| 100 | + }, capture_output=True, text=True, check=True) |
| 101 | + args = json.loads(result.stdout) |
| 102 | + posted = dict(args[i + 1].split("=", 1) for i in range(len(args)) if args[i] == "-f") |
| 103 | + self.assertEqual(expected, posted["state"]) |
| 104 | + self.assertIn("repos/apache/doris/statuses/" + "a" * 40, args) |
| 105 | + resolution = resolve_status([{ |
| 106 | + "state": "open", "number": 123, "head": {"sha": "a" * 40}, "base": {"sha": "b" * 40}, |
| 107 | + }], [{"id": 1, **posted}], head_sha="a" * 40) |
| 108 | + self.assertEqual(expected, resolution.state) |
| 109 | + if not converged: |
| 110 | + self.assertIn("did not converge", posted["description"]) |
| 111 | + self.assertIn("did not converge", resolution.description) |
| 112 | + else: |
| 113 | + self.assertNotIn("did not converge", resolution.description) |
| 114 | + |
| 115 | + |
| 116 | +if __name__ == "__main__": |
| 117 | + unittest.main() |
0 commit comments