Skip to content

Commit 69c2b46

Browse files
committed
ci: retry runner-loss failures for same-repository PR runs
1 parent 49bfbc1 commit 69c2b46

3 files changed

Lines changed: 245 additions & 17 deletions

File tree

‎.github/workflows/rerun-preempted.yml‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ on:
66
workflow_run:
77
workflows:
88
- CI post-merge
9-
branches: [main]
9+
- GKE PR
1010
types: [completed]
1111

1212
permissions:
@@ -22,8 +22,10 @@ jobs:
2222
rerun:
2323
if: >-
2424
github.event.workflow_run.conclusion == 'failure' &&
25-
github.event.workflow_run.run_attempt < 3 &&
26-
(github.event.workflow_run.event == 'push' || github.event.workflow_run.event == 'schedule')
25+
(((github.event.workflow_run.event == 'push' ||
26+
github.event.workflow_run.event == 'schedule') &&
27+
github.event.workflow_run.head_branch == 'main') ||
28+
github.event.workflow_run.event == 'pull_request_target')
2729
runs-on: ubuntu-24.04
2830
steps:
2931
- name: Require controller tools

‎scripts/ci/rerun_preempted.py‎

Lines changed: 182 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111

1212
import argparse
1313
import base64
14+
from datetime import datetime
1415
from graphlib import TopologicalSorter
1516
import itertools
1617
import json
@@ -20,15 +21,19 @@
2021
import sys
2122
import textwrap
2223
import time
24+
from urllib.parse import quote
2325

2426
PRIMARY_BRANCH = 'main'
2527
MAX_ATTEMPTS = 3
28+
PR_WORKFLOW = '.github/workflows/gke-pr.yml'
29+
PR_EVENT = 'pull_request_target'
2630
# Let the jobs API converge after the workflow_run completion event.
2731
JOB_POLL_INTERVAL = 30
2832
JOB_POLL_BOUND = 10 * 60
2933
# Exact entrypoints/events, independently checked in addition to workflow_run names.
3034
ALLOWED_WORKFLOWS = {
3135
'.github/workflows/ci-post.yml': frozenset({'push', 'schedule'}),
36+
PR_WORKFLOW: frozenset({PR_EVENT}),
3237
}
3338
# Bare amd64 GKE scale sets, optionally with the fleet's disk-size suffix.
3439
# No arm64, gkp pools or operator-dispatched entrypoints are admitted.
@@ -145,19 +150,21 @@ def _sleep(seconds):
145150
time.sleep(seconds)
146151

147152

148-
def _gate(run, repo, expected_attempt):
153+
def _gate(run, repo, expected_attempt, max_attempts=MAX_ATTEMPTS):
154+
if type(max_attempts) is not int or max_attempts < 1:
155+
return 'invalid_max_attempts'
149156
if run['repository']['full_name'] != repo:
150157
return 'repository_mismatch'
151158
if run['status'] != 'completed' or run['conclusion'] != 'failure':
152159
return 'not_a_completed_failed_run'
153160
attempt = int(run['run_attempt'])
154-
if not 1 <= attempt < MAX_ATTEMPTS:
161+
if not 1 <= attempt < max_attempts:
155162
return 'attempt_limit'
156163
if expected_attempt is not None and attempt != expected_attempt:
157164
return 'stale_completion_event'
158165
if run['event'] not in ALLOWED_WORKFLOWS.get(run['path'], ()):
159166
return 'workflow_or_event_not_allowlisted'
160-
if run['head_branch'] != PRIMARY_BRANCH:
167+
if run['path'] == '.github/workflows/ci-post.yml' and run['head_branch'] != PRIMARY_BRANCH:
161168
return 'not_the_primary_branch'
162169
return None
163170

@@ -597,12 +604,156 @@ def _jobs(api, endpoint, attempt):
597604
for job in page['jobs']]
598605

599606

600-
def _run_changed(current, initial, repo, attempt):
607+
def _run_changed(current, initial, repo, attempt, max_attempts=MAX_ATTEMPTS):
601608
identity = ('id', 'run_attempt', 'head_sha', 'path', 'event', 'head_branch')
602-
return _gate(current, repo, attempt) or any(current[key] != initial[key] for key in identity)
609+
return _gate(current, repo, attempt, max_attempts=max_attempts) or any(
610+
current[key] != initial[key] for key in identity)
611+
612+
613+
def _pr_identity(run, repo):
614+
repository = run.get('repository')
615+
head_repository = run.get('head_repository')
616+
if not isinstance(repository, dict) or not isinstance(head_repository, dict):
617+
return None, 'pull_request_metadata_unavailable'
618+
repository_name = repository.get('full_name')
619+
head_repository_name = head_repository.get('full_name')
620+
if not isinstance(repository_name, str) or not isinstance(head_repository_name, str):
621+
return None, 'pull_request_metadata_unavailable'
622+
if repository_name != repo or head_repository_name != repository_name:
623+
return None, 'pull_request_head_repository_mismatch'
624+
pull_requests = run.get('pull_requests')
625+
if (not isinstance(pull_requests, list) or len(pull_requests) != 1
626+
or not isinstance(pull_requests[0], dict)):
627+
return None, 'pull_request_metadata_unavailable'
628+
number = pull_requests[0].get('number')
629+
if type(number) is not int or number <= 0:
630+
return None, 'pull_request_metadata_unavailable'
631+
return number, None
632+
633+
634+
def _same_run(left, right):
635+
left_id, right_id = left.get('id'), right.get('id')
636+
if left_id is None or right_id is None:
637+
return False
638+
if left_id == right_id:
639+
return True
640+
return (type(left_id) is int and type(right_id) is int
641+
and left_id > 0 and left_id == right_id)
642+
643+
644+
def _created_at(value):
645+
if not isinstance(value, str):
646+
return None
647+
try:
648+
timestamp = datetime.fromisoformat(value.replace('Z', '+00:00'))
649+
except ValueError:
650+
return None
651+
return timestamp if timestamp.tzinfo is not None else None
603652

604653

605-
def _completed_jobs(api, endpoint, attempt, initial_run, repo):
654+
def _newer_run(candidate, run):
655+
if _same_run(candidate, run):
656+
return False
657+
compared = False
658+
candidate_number, run_number = candidate.get('run_number'), run.get('run_number')
659+
if (type(candidate_number) is int and candidate_number > 0
660+
and type(run_number) is int and run_number > 0):
661+
compared = True
662+
if candidate_number > run_number:
663+
return True
664+
candidate_created, run_created = (_created_at(candidate.get('created_at')),
665+
_created_at(run.get('created_at')))
666+
if candidate_created is not None and run_created is not None:
667+
compared = True
668+
if candidate_created > run_created:
669+
return True
670+
return False if compared else None
671+
672+
673+
def _pr_run_history(api, repo, run, number):
674+
head_branch = run.get('head_branch')
675+
if not isinstance(head_branch, str):
676+
return 'pull_request_metadata_unavailable'
677+
endpoint = (f'repos/{repo}/actions/workflows/gke-pr.yml/runs'
678+
f'?event={PR_EVENT}&branch={quote(head_branch, safe="")}&per_page=100')
679+
pages = api.pages(endpoint)
680+
if not isinstance(pages, list) or not pages:
681+
return 'pull_request_runs_unavailable'
682+
runs, total_count = [], None
683+
for page in pages:
684+
if not isinstance(page, dict) or not isinstance(page.get('workflow_runs'), list):
685+
return 'pull_request_runs_unavailable'
686+
page_total = page.get('total_count')
687+
if page_total is not None:
688+
if type(page_total) is not int or page_total < 0:
689+
return 'pull_request_runs_unavailable'
690+
total_count = page_total if total_count is None else max(total_count, page_total)
691+
runs.extend(page['workflow_runs'])
692+
if total_count is not None and len(runs) < total_count:
693+
return 'pull_request_runs_unavailable'
694+
found_current = False
695+
for candidate in runs:
696+
if not isinstance(candidate, dict):
697+
return 'pull_request_runs_unavailable'
698+
if _same_run(candidate, run):
699+
found_current = True
700+
continue
701+
pull_requests = candidate.get('pull_requests')
702+
if not isinstance(pull_requests, list):
703+
return 'pull_request_runs_unavailable'
704+
numbers = []
705+
for pull_request in pull_requests:
706+
if not isinstance(pull_request, dict):
707+
return 'pull_request_runs_unavailable'
708+
candidate_number = pull_request.get('number')
709+
if type(candidate_number) is not int or candidate_number <= 0:
710+
return 'pull_request_runs_unavailable'
711+
numbers.append(candidate_number)
712+
if number not in numbers:
713+
continue
714+
newer = _newer_run(candidate, run)
715+
if newer is None:
716+
return 'pull_request_runs_unavailable'
717+
if newer:
718+
return 'superseded_pull_request_head'
719+
return None if found_current else 'pull_request_runs_unavailable'
720+
721+
722+
def _pr_guard(api, repo, run):
723+
number, reason = _pr_identity(run, repo)
724+
if reason:
725+
return reason
726+
head_sha = run.get('head_sha')
727+
if not isinstance(head_sha, str):
728+
return 'pull_request_metadata_unavailable'
729+
pull_request = api.get(f'repos/{repo}/pulls/{number}')
730+
if not isinstance(pull_request, dict) or pull_request.get('number') != number:
731+
return 'pull_request_metadata_unavailable'
732+
head = pull_request.get('head')
733+
head_repository = head.get('repo') if isinstance(head, dict) else None
734+
if not isinstance(head, dict) or not isinstance(head_repository, dict):
735+
return 'pull_request_metadata_unavailable'
736+
fork = head_repository.get('fork')
737+
if type(fork) is not bool:
738+
return 'pull_request_metadata_unavailable'
739+
if fork:
740+
return 'pull_request_is_fork'
741+
head_repository_name = head_repository.get('full_name')
742+
if not isinstance(head_repository_name, str):
743+
return 'pull_request_metadata_unavailable'
744+
if head_repository_name != repo:
745+
return 'pull_request_head_repository_mismatch'
746+
if pull_request.get('state') != 'open':
747+
return 'superseded_pull_request_head'
748+
current_head_sha = head.get('sha')
749+
if not isinstance(current_head_sha, str):
750+
return 'pull_request_metadata_unavailable'
751+
if current_head_sha != head_sha:
752+
return 'superseded_pull_request_head'
753+
return _pr_run_history(api, repo, run, number)
754+
755+
756+
def _completed_jobs(api, endpoint, attempt, initial_run, repo, max_attempts=MAX_ATTEMPTS):
606757
jobs = _jobs(api, endpoint, attempt)
607758
if jobs and all(job['status'] == 'completed' for job in jobs):
608759
return jobs, None, None
@@ -617,29 +768,40 @@ def _completed_jobs(api, endpoint, attempt, initial_run, repo):
617768
_sleep(min(JOB_POLL_INTERVAL, remaining))
618769
waited = round(max(0.0, _clock() - started), 3)
619770
current = api.get(endpoint)
620-
if _run_changed(current, initial_run, repo, attempt):
771+
if _run_changed(current, initial_run, repo, attempt, max_attempts=max_attempts):
621772
return jobs, waited, 'run_changed_during_classification'
622773
jobs = _jobs(api, endpoint, attempt)
623774
if jobs and all(job['status'] == 'completed' for job in jobs):
624775
return jobs, waited, None
625776

626777

627-
def process(api, repo, run_id, *, dry_run=False, expected_attempt=None):
778+
def process(api, repo, run_id, *, dry_run=False, expected_attempt=None,
779+
max_attempts=MAX_ATTEMPTS):
628780
"""Return a bounded decision; lookup ambiguity always refuses the POST."""
629781
result = {'run_id': run_id, 'dry_run': dry_run, 'rerun': False,
630782
'failed_job_ids': [], 'dependent_job_ids': []}
631783
try:
632784
if not re.fullmatch(r'[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+', repo) or int(run_id) <= 0:
633785
raise ValueError('invalid repository or run id')
786+
if type(max_attempts) is not int or max_attempts < 1:
787+
result['reason'] = 'invalid_max_attempts'
788+
return result
634789
endpoint = f'repos/{repo}/actions/runs/{int(run_id)}'
635790
run = api.get(endpoint)
636-
reason = _gate(run, repo, expected_attempt)
791+
reason = _gate(run, repo, expected_attempt, max_attempts=max_attempts)
637792
result['run_attempt'] = run['run_attempt']
638793
if reason:
639794
result['reason'] = reason
640795
return result
796+
is_pr = run['path'] == PR_WORKFLOW
797+
if is_pr:
798+
reason = _pr_guard(api, repo, run)
799+
if reason:
800+
result['reason'] = reason
801+
return result
641802
attempt = int(run['run_attempt'])
642-
jobs, waited, reason = _completed_jobs(api, endpoint, attempt, run, repo)
803+
jobs, waited, reason = _completed_jobs(
804+
api, endpoint, attempt, run, repo, max_attempts=max_attempts)
643805
if waited is not None:
644806
result['waited_seconds'] = waited
645807
if reason:
@@ -708,9 +870,14 @@ def process(api, repo, run_id, *, dry_run=False, expected_attempt=None):
708870
return result
709871
# Re-read just before mutation; concurrency serializes controller runs too.
710872
current = api.get(endpoint)
711-
if _run_changed(current, run, repo, attempt):
873+
if _run_changed(current, run, repo, attempt, max_attempts=max_attempts):
712874
result['reason'] = 'run_changed_during_classification'
713875
return result
876+
if is_pr:
877+
reason = _pr_guard(api, repo, current)
878+
if reason:
879+
result['reason'] = reason
880+
return result
714881
if not dry_run:
715882
api.post(f'{endpoint}/rerun-failed-jobs')
716883
result.update(rerun=True, reason=('runner_loss_with_dependents' if dependent
@@ -725,9 +892,12 @@ def main():
725892
parser.add_argument('--repo', default=os.environ.get('GITHUB_REPOSITORY'), required=not os.environ.get('GITHUB_REPOSITORY'))
726893
parser.add_argument('--run-id', required=True, type=int)
727894
parser.add_argument('--expected-attempt', type=int)
895+
parser.add_argument('--max-attempts', type=int, default=MAX_ATTEMPTS)
728896
parser.add_argument('--dry-run', action='store_true')
729897
args = parser.parse_args()
730-
decision = process(GitHubAPI(), args.repo, args.run_id, dry_run=args.dry_run, expected_attempt=args.expected_attempt)
898+
decision = process(GitHubAPI(), args.repo, args.run_id, dry_run=args.dry_run,
899+
expected_attempt=args.expected_attempt,
900+
max_attempts=args.max_attempts)
731901
print(json.dumps(decision, sort_keys=True))
732902
# Refusals, including lookup failures, are successful controller decisions.
733903
# Their JSON diagnostics remain visible; missing proof never permits a POST.

‎scripts/test_ci_rerun_preempted.py‎

Lines changed: 58 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -195,9 +195,63 @@ def test_plain_failure_and_pr_run_refused(self):
195195
'independent_failure_diagnostic')
196196
self.assert_no_post(api)
197197
api = FakeAPI(run=CAPTURED['run-41515706477.json'])
198-
self.assert_refused(api, 'workflow_or_event_not_allowlisted')
198+
# The PR workflow is now allowlisted; this old capture lacks PR identity fields.
199+
self.assert_refused(api, 'pull_request_metadata_unavailable')
199200
self.assertEqual(api.calls, [('GET', api.endpoint)])
200201

202+
def test_pr_superseded_head_refused_without_post(self):
203+
class PRAPI(FakeAPI):
204+
def __init__(self, run, pull_request, history):
205+
super().__init__(run=run)
206+
self.pull_request = deepcopy(pull_request)
207+
self.history = deepcopy(history)
208+
209+
def get(self, path):
210+
if path == f'{self.endpoint.rsplit("/actions", 1)[0]}/pulls/{self.pull_request["number"]}':
211+
self.calls.append(('GET', path))
212+
return deepcopy(self.pull_request)
213+
return super().get(path)
214+
215+
def pages(self, path):
216+
if path == (f'{self.endpoint.rsplit("/actions", 1)[0]}/actions/workflows/'
217+
f'gke-pr.yml/runs?event={controller.PR_EVENT}&branch=feature&per_page=100'):
218+
self.calls.append(('PAGES', path))
219+
return deepcopy(self.history)
220+
return super().pages(path)
221+
222+
old_sha, new_sha = '1' * 40, '2' * 40
223+
run = completed_wave_run()
224+
run.update(path=controller.PR_WORKFLOW, event=controller.PR_EVENT,
225+
head_branch='feature', head_repository={'full_name': REPO},
226+
pull_requests=[{'number': 736}], head_sha=old_sha,
227+
run_number=10, created_at='2026-11-05T00:00:00Z')
228+
current = deepcopy(run)
229+
newer = deepcopy(run)
230+
newer.update(id=run['id'] + 1, run_number=11, head_sha=new_sha,
231+
created_at='2026-11-05T00:01:00Z')
232+
233+
def pull_request(state, sha):
234+
return {'number': 736, 'state': state,
235+
'head': {'sha': sha, 'repo': {'full_name': REPO, 'fork': False}}}
236+
237+
cases = (
238+
('closed', pull_request('closed', old_sha),
239+
[{'total_count': 1, 'workflow_runs': [current]}]),
240+
('head_changed', pull_request('open', new_sha),
241+
[{'total_count': 1, 'workflow_runs': [current]}]),
242+
('newer_run_head_match', pull_request('open', old_sha),
243+
[{'total_count': 2, 'workflow_runs': [newer, current]}]),
244+
('newer_run_head_new', pull_request('open', new_sha),
245+
[{'total_count': 2, 'workflow_runs': [newer, current]}]),
246+
)
247+
for name, pr, history in cases:
248+
with self.subTest(case=name):
249+
api = PRAPI(run, pr, history)
250+
result = self.decide(api)
251+
self.assertFalse(result['rerun'], result)
252+
self.assertEqual(result['reason'], 'superseded_pull_request_head', result)
253+
self.assert_no_post(api)
254+
201255
def test_mixed_failure_later_page_refused(self):
202256
for pages in ([{'jobs': [CUT_JOB, ORDINARY_JOB]}],
203257
[{'jobs': [CUT_JOB]}, {'jobs': [ORDINARY_JOB]}]):
@@ -509,11 +563,13 @@ def test_allowlist_and_trusted_checkout(self):
509563
self.assertEqual(controller.PRIMARY_BRANCH, 'main')
510564
self.assertEqual(controller.ALLOWED_WORKFLOWS, {
511565
'.github/workflows/ci-post.yml': frozenset({'push', 'schedule'}),
566+
'.github/workflows/gke-pr.yml': frozenset({'pull_request_target'}),
512567
})
513568
workflow = (ROOT / '.github/workflows/rerun-preempted.yml').read_text()
514569
trigger = workflow.split('\npermissions:', 1)[0]
515570
self.assertEqual(re.findall(r'^ (\w+):', trigger, re.M), ['workflow_run'])
516-
self.assertEqual(re.findall(r'^ - (.+)$', trigger, re.M), ['CI post-merge'])
571+
self.assertEqual(re.findall(r'^ - (.+)$', trigger, re.M),
572+
['CI post-merge', 'GKE PR'])
517573
self.assertIn('types: [completed]', trigger)
518574
self.assertIn('runs-on: ubuntu-24.04', workflow)
519575
self.assertNotIn('pf-gcp-pr', workflow)

0 commit comments

Comments
 (0)