Skip to content

Commit 74d0476

Browse files
committed
Merge current main into release archive compatibility changes
2 parents dcbffbe + ace3079 commit 74d0476

2 files changed

Lines changed: 89 additions & 10 deletions

File tree

‎scripts/ci/rerun_preempted.py‎

Lines changed: 55 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,13 @@
1919
import subprocess
2020
import sys
2121
import textwrap
22+
import time
2223

2324
PRIMARY_BRANCH = 'main'
2425
MAX_ATTEMPTS = 3
26+
# Let the jobs API converge after the workflow_run completion event.
27+
JOB_POLL_INTERVAL = 30
28+
JOB_POLL_BOUND = 10 * 60
2529
# Exact entrypoints/events, independently checked in addition to workflow_run names.
2630
ALLOWED_WORKFLOWS = {
2731
'.github/workflows/ci-post.yml': frozenset({'push', 'schedule'}),
@@ -133,6 +137,14 @@ def post(self, path):
133137
self._call(path, method='POST')
134138

135139

140+
def _clock():
141+
return time.monotonic()
142+
143+
144+
def _sleep(seconds):
145+
time.sleep(seconds)
146+
147+
136148
def _gate(run, repo, expected_attempt):
137149
if run['repository']['full_name'] != repo:
138150
return 'repository_mismatch'
@@ -580,6 +592,38 @@ def _job_evidence(api, repo, job, *, diagnostics=None):
580592
return reason
581593

582594

595+
def _jobs(api, endpoint, attempt):
596+
return [job for page in api.pages(f'{endpoint}/attempts/{attempt}/jobs?per_page=100')
597+
for job in page['jobs']]
598+
599+
600+
def _run_changed(current, initial, repo, attempt):
601+
identity = ('id', 'run_attempt', 'head_sha', 'path', 'event', 'head_branch')
602+
return _gate(current, repo, attempt) or any(current[key] != initial[key] for key in identity)
603+
604+
605+
def _completed_jobs(api, endpoint, attempt, initial_run, repo):
606+
jobs = _jobs(api, endpoint, attempt)
607+
if jobs and all(job['status'] == 'completed' for job in jobs):
608+
return jobs, None, None
609+
610+
started = _clock()
611+
deadline = started + JOB_POLL_BOUND
612+
while True:
613+
remaining = deadline - _clock()
614+
if remaining <= 0:
615+
waited = round(max(0.0, _clock() - started), 3)
616+
return jobs, waited, 'missing_or_incomplete_jobs'
617+
_sleep(min(JOB_POLL_INTERVAL, remaining))
618+
waited = round(max(0.0, _clock() - started), 3)
619+
current = api.get(endpoint)
620+
if _run_changed(current, initial_run, repo, attempt):
621+
return jobs, waited, 'run_changed_during_classification'
622+
jobs = _jobs(api, endpoint, attempt)
623+
if jobs and all(job['status'] == 'completed' for job in jobs):
624+
return jobs, waited, None
625+
626+
583627
def process(api, repo, run_id, *, dry_run=False, expected_attempt=None):
584628
"""Return a bounded decision; lookup ambiguity always refuses the POST."""
585629
result = {'run_id': run_id, 'dry_run': dry_run, 'rerun': False,
@@ -595,9 +639,11 @@ def process(api, repo, run_id, *, dry_run=False, expected_attempt=None):
595639
result['reason'] = reason
596640
return result
597641
attempt = int(run['run_attempt'])
598-
jobs = [job for page in api.pages(f'{endpoint}/attempts/{attempt}/jobs?per_page=100') for job in page['jobs']]
599-
if not jobs or any(job['status'] != 'completed' for job in jobs):
600-
result['reason'] = 'missing_or_incomplete_jobs'
642+
jobs, waited, reason = _completed_jobs(api, endpoint, attempt, run, repo)
643+
if waited is not None:
644+
result['waited_seconds'] = waited
645+
if reason:
646+
result['reason'] = reason
601647
return result
602648
if any(job['conclusion'] not in {'success', 'skipped', 'neutral', 'failure'} for job in jobs):
603649
result['reason'] = 'ambiguous_unsuccessful_job'
@@ -611,7 +657,11 @@ def process(api, repo, run_id, *, dry_run=False, expected_attempt=None):
611657
specs = _workflow_jobs(api, repo, run)
612658
# Map failures first so an unresolvable failure names its blocker.
613659
mapped = _map_jobs(specs, failed)
614-
mapped.update(_map_jobs(specs, [j for j in jobs if j not in failed]))
660+
# A skipped matrix row may collapse to an aggregate display name
661+
# when a docs-only condition is evaluated before expansion. It
662+
# cannot be a failed DAG node, so only non-skipped jobs need aliases.
663+
mapped.update(_map_jobs(specs, [j for j in jobs
664+
if j not in failed and j['conclusion'] != 'skipped']))
615665
failed_needs = {
616666
int(job['id']): {int(other['id']) for other in failed
617667
if any(mapped[int(other['id'])] in specs[need]['members']
@@ -658,8 +708,7 @@ def process(api, repo, run_id, *, dry_run=False, expected_attempt=None):
658708
return result
659709
# Re-read just before mutation; concurrency serializes controller runs too.
660710
current = api.get(endpoint)
661-
identity = ('id', 'run_attempt', 'head_sha', 'path', 'event', 'head_branch')
662-
if _gate(current, repo, attempt) or any(current[key] != run[key] for key in identity):
711+
if _run_changed(current, run, repo, attempt):
663712
result['reason'] = 'run_changed_during_classification'
664713
return result
665714
if not dry_run:

‎scripts/test_ci_rerun_preempted.py‎

Lines changed: 34 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,8 @@ def __init__(self, *, run=None, job_pages=None, current=None):
5454
self.endpoint = f'repos/{REPO}/actions/runs/{self.run["id"]}'
5555
self.jobs_path = f'{self.endpoint}/attempts/{self.run["run_attempt"]}/jobs?per_page=100'
5656
self.job_pages = deepcopy([{'jobs': [CUT_JOB]}] if job_pages is None else job_pages)
57+
self.job_pages_after = []
58+
self.job_reads = 0
5759
self.calls = []
5860
self.reads = 0
5961
self.annotation_error = None
@@ -105,7 +107,10 @@ def get(self, path):
105107
def pages(self, path):
106108
self.calls.append(('PAGES', path))
107109
if path == self.jobs_path:
108-
return deepcopy(self.job_pages)
110+
self.job_reads += 1
111+
pages = (self.job_pages_after.pop(0)
112+
if self.job_reads > 1 and self.job_pages_after else self.job_pages)
113+
return deepcopy(pages)
109114
if path not in self.annotations:
110115
raise AssertionError('unexpected paginated lookup')
111116
if self.annotation_error:
@@ -270,8 +275,15 @@ def test_missing_loss_proof_refused(self):
270275
job.update(mutation)
271276
self.assert_refused(FakeAPI(job_pages=[{'jobs': [job]}]), reason)
272277

273-
def test_incomplete_ambiguous_jobs_refused(self):
274-
self.assert_refused(FakeAPI(job_pages=[]), 'missing_or_incomplete_jobs')
278+
def test_incomplete_jobs_poll_before_deciding(self):
279+
pending = deepcopy(CUT_JOB)
280+
pending['status'] = 'in_progress'
281+
api = FakeAPI(job_pages=[{'jobs': [pending]}])
282+
api.job_pages_after = [[{'jobs': [CUT_JOB]}]]
283+
with patch.object(controller, '_sleep'):
284+
result = self.decide(api)
285+
self.assertTrue(result['rerun'], result)
286+
self.assertIn('waited_seconds', result)
275287
for mutation, reason in (
276288
({'status': 'in_progress'}, 'missing_or_incomplete_jobs'),
277289
({'conclusion': 'cancelled'}, 'ambiguous_unsuccessful_job'),
@@ -280,7 +292,15 @@ def test_incomplete_ambiguous_jobs_refused(self):
280292
with self.subTest(mutation=mutation):
281293
job = deepcopy(CUT_JOB)
282294
job.update(mutation)
283-
self.assert_refused(FakeAPI(job_pages=[{'jobs': [job]}]), reason)
295+
api = FakeAPI(job_pages=[{'jobs': [job]}])
296+
if mutation.get('status') == 'in_progress':
297+
with patch.object(controller, '_clock',
298+
side_effect=(0, controller.JOB_POLL_BOUND,
299+
controller.JOB_POLL_BOUND)):
300+
result = self.assert_refused(api, reason)
301+
self.assertEqual(result['waited_seconds'], controller.JOB_POLL_BOUND)
302+
else:
303+
self.assert_refused(api, reason)
284304

285305
def test_optional_annotations_and_dry_run(self):
286306
api = FakeAPI()
@@ -1116,6 +1136,16 @@ def test_mapping_ambiguity(self):
11161136
api = case_api(SMALL_FLOW + '\n other:\n name: Result\n',
11171137
('Worker (linux, one)', 'loss'), ('Result', 'gate'))
11181138
self.refused(api, 'aggregate_mapping_unavailable', 101)
1139+
flow = SMALL_FLOW + '''
1140+
other:
1141+
name: Hidden
1142+
duplicate:
1143+
name: Hidden
1144+
'''
1145+
api = case_api(flow, ('Worker (linux, one)', 'loss'),
1146+
('Result', 'gate'), ('Hidden', 'ok'))
1147+
api.job_pages[0]['jobs'][2]['conclusion'] = 'skipped'
1148+
self.accepted(api, [101])
11191149

11201150
def test_needs_not_name_or_order(self):
11211151
flow = SMALL_FLOW.replace('needs: work', 'needs: unrelated') + '''

0 commit comments

Comments
 (0)