Skip to content

Commit d4f052b

Browse files
authored
[fix](ci) Check terminal events before accepting review recovery (apache#68372)
### What problem does this PR solve? After a capacity error, the review helper can accept `codex exec --goal resume` exiting with status 0 even when the attempt reports `turn.failed` or never produces a terminal event. The helper then stops retrying and the workflow replaces the underlying error with “no new pull request review was submitted.” Require both a zero process exit and a completed latest turn before accepting an attempt. Explicit capacity errors remain eligible for the existing bounded retry sequence even when the process exits with 0. Authentication, usage-limit and other errors fail without retry; a missing terminal event fails explicitly instead of borrowing a previous attempt’s capacity error or completion. Existing PR-state checks, duplicate-submission protection and the shared deadline are retained. Each attempt now logs its exit status, last turn event and event count. The installation step logs the actual goal binary’s SHA-256 to make runtime failures reproducible. Related PR: apache#67624 ### Validation - Linux Python 3.12: all 147 pipeline tests passed, including real process cleanup and workflow/auth-quarantine integration tests. Tests ran without network access or host mounts. - Added 12 regression tests for zero-exit capacity retries and exhaustion, zero-exit authentication failures, missing terminal events, event ordering and workflow error propagation. - The new helper tests fail against the previous implementation, including false success and premature retry termination; they pass after the fix. - macOS helper suite: 55 tests, passed with 3 Linux-only skips. - Ruff checks and helper/test formatting, actionlint, workflow YAML parsing, all 22 shell blocks with `bash -n`, and `git diff --check` passed. **Rollout limit:** this validates helper behavior with controlled Codex output and real Linux subprocesses. The actual OSS goal binary was not available for validation (anonymous read is denied); no claim is made that the underlying binary’s goal/resume issue is fixed. Before production rollout, validate that exact binary’s resumed turn execution and terminal-event behavior. The change fails incomplete resumes explicitly; it does not blindly retry empty successful exits.
1 parent 7dc2985 commit d4f052b

4 files changed

Lines changed: 203 additions & 10 deletions

File tree

‎.github/scripts/run_review_with_resume.py‎

Lines changed: 43 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -105,14 +105,41 @@ def read_events(path):
105105
return events
106106

107107

108-
def failure(events, status, stderr_path):
109-
for event_type in ("turn.failed", "error"):
110-
for event in reversed(events):
111-
if event.get("type") == event_type:
112-
error = event.get("error") or event
113-
return error.get("message") or f"Codex exited with status {status}"
108+
def attempt_result(events, status, stderr_path):
109+
"""Use the latest turn in this attempt, never a previous attempt's failure."""
110+
terminal = next(
111+
(
112+
event
113+
for event in reversed(events)
114+
if event.get("type")
115+
in ("turn.started", "turn.completed", "turn.failed", "error")
116+
),
117+
{},
118+
)
119+
event_type = terminal.get("type", "missing")
120+
if event_type == "turn.completed":
121+
message = (
122+
None
123+
if status == 0
124+
else f"Codex exited with status {status} after turn.completed"
125+
)
126+
return event_type, message
127+
if event_type in ("turn.failed", "error"):
128+
error = terminal.get("error") or terminal
129+
message = (
130+
error.get("message")
131+
or f"Codex reported {event_type} (exit status {status})"
132+
)
133+
return event_type, message
134+
# A zero exit after only thread.started (or an unfinished new turn) does
135+
# not prove recovery. Do not reuse an older capacity event to retry it.
136+
if status == 0:
137+
return (
138+
event_type,
139+
"Codex exited with status 0 without a terminal turn event; review is incomplete",
140+
)
114141
lines = stderr_path.read_text(errors="replace").splitlines()
115-
return next(
142+
return event_type, next(
116143
(line for line in reversed(lines) if line.strip()),
117144
f"Codex exited with status {status}",
118145
)
@@ -388,8 +415,14 @@ def fail(message):
388415
return fail(
389416
f"Codex was interrupted or timed out (status {status}); not resuming"
390417
)
391-
message = failure(events, status, stderr_path)
392-
if status != 0 and (
418+
terminal, message = attempt_result(events, status, stderr_path)
419+
print(
420+
f"Finished Codex review attempt {attempt + 1}/4 "
421+
f"(exit_status={status}, terminal_event={terminal}, events={len(events)})",
422+
file=sys.stderr,
423+
flush=True,
424+
)
425+
if message is not None and (
393426
message != CAPACITY_MESSAGE or attempt == len(RETRY_DELAYS)
394427
):
395428
return fail(message)
@@ -399,7 +432,7 @@ def fail(message):
399432
"Codex resumed a different session; refusing further attempts"
400433
)
401434
thread_id = current_id
402-
if status == 0:
435+
if message is None:
403436
return 0
404437
require_rollout(Path(os.environ["CODEX_HOME"]), thread_id, args.cwd)
405438
# Check parser support without authenticating or starting a model request.

‎.github/scripts/test_review_auth_quarantine.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -373,9 +373,23 @@ def test_tool_output_quoting_reused_error_is_not_quarantined(self):
373373
{"type": "turn.failed", "error": {"message": "Request timed out"}},
374374
], expected_invalid=False)
375375

376+
def test_zero_exit_auth_failure_still_reaches_quarantine(self):
377+
_, outputs = self.fail_review(FAKE_CODEX_STATUS="0")
378+
self.assertIn(REUSED_MESSAGE, outputs)
379+
self.assertNotIn("no new pull request review", outputs)
380+
381+
def test_zero_exit_without_terminal_event_does_not_pass_with_a_review(self):
382+
_, outputs = self.fail_review(
383+
events=[], expected_invalid=False, FAKE_CODEX_STATUS="0"
384+
)
385+
# The fake GitHub API reports a review, but an incomplete attempt must
386+
# still fail rather than borrowing that review as proof of completion.
387+
self.assertIn("without a terminal turn event", outputs)
388+
376389
def test_success_is_not_quarantined_even_with_earlier_stderr_error(self):
377390
_, outputs = self.run_step(
378391
"Run automated code review", FAKE_CODEX_STATUS="0",
392+
FAKE_CODEX_EVENTS=json.dumps({"type": "turn.completed", "usage": {}}),
379393
FAKE_CODEX_STDERR='{"code":"refresh_token_reused"}',
380394
)
381395
self.assertNotIn("auth_invalid_reason", outputs)

‎.github/scripts/test_run_review_with_resume.py‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,149 @@ def test_capacity_resumes_exact_session_with_same_settings_and_ledger(self):
188188
self.assertEqual("done", (self.context / "codex-final-message.txt").read_text())
189189
self.target_check.assert_called_once()
190190

191+
def test_zero_exit_capacity_on_resume_still_retries(self):
192+
self.assertEqual(
193+
0,
194+
self.execute(
195+
[
196+
{"events": [thread_event(), failed()]},
197+
{"events": [thread_event(), failed()], "status": 0},
198+
{"events": [thread_event(), completed()], "status": 0},
199+
]
200+
),
201+
)
202+
self.assertEqual(3, len(self.commands))
203+
self.assertEqual([30, 60], self.sleeps)
204+
self.assertEqual(2, self.target_check.call_count)
205+
self.assertEqual("completed", exporter.latest_turn_result(self.events())[0])
206+
207+
def test_zero_exit_capacity_stops_at_the_retry_limit(self):
208+
self.assertEqual(
209+
1, self.execute([{"events": [thread_event(), failed()], "status": 0}] * 4)
210+
)
211+
self.assertEqual([30, 60, 120], self.sleeps)
212+
self.assertEqual(4, len(self.commands))
213+
self.assertEqual(runner.CAPACITY_MESSAGE, self.last_error())
214+
215+
def test_zero_exit_error_event_can_identify_capacity(self):
216+
self.assertEqual(
217+
0,
218+
self.execute(
219+
[
220+
{
221+
"events": [
222+
thread_event(),
223+
{"type": "error", "message": runner.CAPACITY_MESSAGE},
224+
],
225+
"status": 0,
226+
},
227+
{"events": [thread_event(), completed()], "status": 0},
228+
]
229+
),
230+
)
231+
self.assertEqual([30], self.sleeps)
232+
233+
def test_zero_exit_auth_usage_and_generic_failures_do_not_retry(self):
234+
for message in (
235+
"refresh_token_reused",
236+
"You've hit your usage limit.",
237+
"HTTP 500",
238+
):
239+
with self.subTest(message=message), tempfile.TemporaryDirectory() as tmp:
240+
self.args.context_dir = Path(tmp)
241+
(self.args.context_dir / "codex_goal_prompt.txt").write_text("review")
242+
self.assertEqual(
243+
1,
244+
self.execute(
245+
[{"events": [thread_event(), failed(message)], "status": 0}]
246+
),
247+
)
248+
events = runner.read_events(
249+
self.args.context_dir / "codex-events.jsonl"
250+
)
251+
self.assertEqual(message, events[-1]["error"]["message"])
252+
self.assertEqual([], self.sleeps)
253+
self.target_check.assert_not_called()
254+
255+
def test_zero_exit_without_terminal_event_fails_closed(self):
256+
self.assertEqual(1, self.execute([{"events": [thread_event()], "status": 0}]))
257+
self.assertIn("without a terminal turn event", self.last_error())
258+
self.assertEqual([], self.sleeps)
259+
260+
def test_empty_resume_does_not_reuse_the_previous_capacity_error(self):
261+
self.assertEqual(
262+
1,
263+
self.execute(
264+
[
265+
{"events": [thread_event(), failed()]},
266+
{"events": [thread_event()], "status": 0},
267+
]
268+
),
269+
)
270+
self.assertEqual(2, len(self.commands))
271+
self.assertEqual([30], self.sleeps)
272+
self.assertIn("without a terminal turn event", self.last_error())
273+
self.assertNotEqual(runner.CAPACITY_MESSAGE, self.last_error())
274+
275+
def test_later_completion_supersedes_earlier_error_in_the_same_attempt(self):
276+
self.assertEqual(
277+
0,
278+
self.execute(
279+
[{"events": [thread_event(), failed(), completed()], "status": 0}]
280+
),
281+
)
282+
self.assertEqual([], self.sleeps)
283+
284+
def test_later_failure_supersedes_earlier_completion(self):
285+
self.assertEqual(
286+
1,
287+
self.execute(
288+
[
289+
{
290+
"events": [thread_event(), completed(), failed("auth failed")],
291+
"status": 0,
292+
}
293+
]
294+
),
295+
)
296+
self.assertEqual("auth failed", self.last_error())
297+
self.assertEqual([], self.sleeps)
298+
299+
def test_unfinished_new_turn_does_not_reuse_an_earlier_completion(self):
300+
self.assertEqual(
301+
1,
302+
self.execute(
303+
[
304+
{
305+
"events": [
306+
thread_event(),
307+
completed(),
308+
{"type": "turn.started"},
309+
],
310+
"status": 0,
311+
}
312+
]
313+
),
314+
)
315+
self.assertIn("without a terminal turn event", self.last_error())
316+
self.assertEqual([], self.sleeps)
317+
318+
def test_completion_with_nonzero_exit_does_not_retry_old_stderr_capacity(self):
319+
self.assertEqual(
320+
1,
321+
self.execute(
322+
[
323+
{
324+
"events": [thread_event(), completed()],
325+
"status": 1,
326+
"stderr": runner.CAPACITY_MESSAGE,
327+
}
328+
]
329+
),
330+
)
331+
self.assertIn("status 1 after turn.completed", self.last_error())
332+
self.assertEqual([], self.sleeps)
333+
191334
def test_retry_count_is_bounded(self):
192335
self.assertEqual(1, self.execute([{"events": [thread_event(), failed()]}] * 4))
193336
self.assertEqual([30, 60, 120], self.sleeps)

‎.github/workflows/code-review-runner.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,9 @@ jobs:
200200
sudo install -m 0755 "$tmp_dir/codex-goal" "$codex_target"
201201
"$codex_cmd" exec --help | grep -q -- '--goal'
202202
"$codex_cmd" --version
203+
# The deployed goal binary currently reports only 0.0.0. Record its
204+
# identity so a resume failure can be reproduced with the same build.
205+
sha256sum "$codex_target"
203206
env:
204207
OSS_AK: ${{ secrets.OSS_AK }}
205208
OSS_SK: ${{ secrets.OSS_SK }}

0 commit comments

Comments
 (0)