Skip to content

Commit 611e000

Browse files
committed
ci: enforce the standard runner contract in preflight
Replace the event-based runner contract and mutation cases introduced by d093560 with ubuntu-24.04 and matrix.os. Refuse pf-linux-x64-16, every *-xlarge or *-large label and every larger_runner key, including in the release matrix, while retaining the remaining workflow checks. Add rejection cases for paid labels, obsolete keys and conditional paid routing. Preserve comment tolerance and core-count checks. Tests intentionally not run under the executor no-run rule.
1 parent 29b2c62 commit 611e000

2 files changed

Lines changed: 86 additions & 77 deletions

File tree

‎scripts/ci_preflight.py‎

Lines changed: 20 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -37,33 +37,17 @@
3737
# The names live only in tests/CMakeLists.txt. A digest pins that measured
3838
# population without maintaining a second row list in this Python guard.
3939
PR_SLOW_ROWS_SHA256 = 'e8bddea797abc46ca0deb671d5d35db3491b67a82e1413d08278530d12f7b67b'
40-
# A larger runner bills the organization even for a public repository, so only
41-
# an event no fork can raise -- a push, the schedule, a maintainer's dispatch --
42-
# or a pull_request whose head is a branch of this repository runs the heavy
43-
# jobs on the organization's 16-core Linux runner or GitHub's M2 runner for the
44-
# macos-26 image. Any other event -- a fork's pull request, any
45-
# pull_request_target, a review or review comment (anyone can post one on a
46-
# public repository's pull request), a comment, a workflow_run -- gets the
47-
# standard runner. In a workflow_call the github context is the caller's.
48-
TRUSTED_EVENT = ("github.event_name == 'push' || github.event_name == 'schedule' || "
49-
"github.event_name == 'workflow_dispatch' || "
50-
"(github.event_name == 'pull_request' && "
51-
"github.event.pull_request.head.repo.full_name == github.repository)")
52-
LINUX_RUNNER = '${{ (' + TRUSTED_EVENT + ") && 'pf-linux-x64-16' || 'ubuntu-24.04' }}"
53-
MATRIX_RUNNER = '${{ (' + TRUSTED_EVENT + ') && matrix.larger_runner || matrix.os }}'
54-
# The build job's whole strategy, exactly (comments and blank lines aside): an
55-
# include entry more -- a leg whose os is a larger runner, or one that
56-
# overrides a larger_runner -- is a finding.
40+
# Standard runners only for every event (owner cost stop, 2026-10-18), until
41+
# the ci-spot pool is live. Larger runners bill even for a public repository.
42+
LINUX_RUNNER = 'ubuntu-24.04'
43+
MATRIX_RUNNER = '${{ matrix.os }}'
44+
# The build job's whole strategy, exactly (comments and blank lines aside):
45+
# only the two standard images and the two build types, with no include overrides.
5746
BUILD_STRATEGY = (' strategy:\n'
5847
' fail-fast: false\n'
5948
' matrix:\n'
6049
' os: [ubuntu-24.04, macos-26]\n'
61-
' build_type: [Release, Debug]\n'
62-
' include:\n'
63-
' - os: ubuntu-24.04\n'
64-
' larger_runner: pf-linux-x64-16\n'
65-
' - os: macos-26\n'
66-
' larger_runner: macos-26-xlarge\n')
50+
' build_type: [Release, Debug]\n')
6751
BUILD_NAME = 'build (${{ matrix.os }}, ${{ matrix.build_type }})'
6852
NATIVE_STRATEGY = ''' strategy:
6953
fail-fast: false
@@ -107,6 +91,11 @@
10791
# schedule, a dispatch; a dispatch alone today); any other workflow file is
10892
# checked, whatever starts it.
10993
STANDARD_RUNNERS = ('ubuntu-24.04', 'ubuntu-latest')
94+
# Refuse paid labels and the old matrix key in every workflow, including the
95+
# otherwise exempt release matrix. Scan code so nested/flow mappings count too.
96+
_LARGER_RUNNER_LABEL = re.compile(
97+
r'(?<![\w.-])(?:pf-linux-x64-16|[\w.-]+-(?:xlarge|large))(?![\w.-])')
98+
_LARGER_RUNNER_KEY = re.compile(r'(?<![\w.-])["\']?larger_runner["\']?[ \t]*:')
11099
TRUSTED_ONLY_WORKFLOWS = ('release.yml',)
111100
TRUSTED_ONLY_EVENTS = {'push', 'schedule', 'workflow_dispatch'}
112101
# A docs-only change (scripts/ci_docs_only.py) skips the proof jobs. Each waits
@@ -295,9 +284,14 @@ def _events(workflow: str) -> set[str] | None:
295284

296285

297286
def runner_findings(workflows: dict[str, str]) -> list[str]:
298-
"""Pin every CI job's runner, time limit and parallelism; keep forks off larger runners."""
287+
"""Pin every CI job's runner, time limit and parallelism; refuse paid runners."""
299288
findings = []
300289
for name, workflow in workflows.items():
290+
code = '\n'.join(re.sub(r'\s+#.*$', '', line) for line in _code(workflow))
291+
if _LARGER_RUNNER_LABEL.search(code):
292+
findings.append(f'{name} must not use larger-runner labels; standard runners only')
293+
if _LARGER_RUNNER_KEY.search(code):
294+
findings.append(f'{name} must not define a larger_runner key; standard runners only')
301295
pinned = JOB_RUNNERS.get(name, {})
302296
events = _events(workflow)
303297
if (not pinned and name in TRUSTED_ONLY_WORKFLOWS and events is not None
@@ -327,12 +321,12 @@ def runner_findings(workflows: dict[str, str]) -> list[str]:
327321
findings.append(f'{name} job {job} must run on {runner}')
328322
if _job_values(body, 'timeout-minutes') != [str(minutes)]:
329323
findings.append(f'{name} job {job} must allow {minutes} minutes')
330-
# A job on 16 cores that still asks for 4 wastes the runner it pays for.
324+
# Keep parallelism tied to the runner's available cores.
331325
if not _sized(_counted_text(body), command):
332326
findings.append(f'{name} job {job} must size its parallelism to the runner')
333327
build = _jobs(workflows.get('ci.yml', '')).get('build', '')
334328
if _job_block(build, 'strategy') != _code(BUILD_STRATEGY):
335-
findings.append('ci.yml build must pair each image with its larger runner, exactly')
329+
findings.append('ci.yml build must use exactly the standard images and build types')
336330
if _job_values(build, 'name') != [BUILD_NAME]:
337331
findings.append(f'ci.yml build legs must keep their names: {BUILD_NAME}')
338332
return findings

‎scripts/test_ci_preflight.py‎

Lines changed: 66 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -142,59 +142,50 @@ def test_ci_contract_pins_parallel_pr_light_and_full_events(self):
142142
changed[index] = changed[index].replace(before, after, 1)
143143
self.assertNotEqual(ci_workflow_findings(*changed), [])
144144

145-
def test_ci_contract_pins_runners_time_limits_and_the_fork_guard(self):
145+
def test_ci_contract_pins_standard_runners_time_limits_and_parallelism(self):
146146
original, others = self.ci_sources(), self.other_sources()
147+
self.assertEqual(LINUX_RUNNER, 'ubuntu-24.04')
148+
self.assertEqual(MATRIX_RUNNER, '${{ matrix.os }}')
147149
self.assertEqual(ci_workflow_findings(*original, others=others), [])
148-
fork_head = 'github.event.pull_request.head.repo.full_name'
149150
mutations = (
150-
# A heavy job moved back to the slow standard runner.
151-
(0, 'preflight', LINUX_RUNNER, 'ubuntu-24.04', 'ci.yml job preflight must run on'),
152-
(0, 'sanitizers', LINUX_RUNNER, 'ubuntu-24.04', 'ci.yml job sanitizers must run on'),
153-
(0, 'kernel-only', LINUX_RUNNER, 'ubuntu-24.04', 'ci.yml job kernel-only must run on'),
154-
(0, 'build', MATRIX_RUNNER, '${{ matrix.os }}', 'ci.yml job build must run on'),
155-
(0, 'build', 'larger_runner: macos-26-xlarge', 'larger_runner: macos-26',
156-
'ci.yml build must pair each image'),
157-
(0, 'build', 'larger_runner: pf-linux-x64-16', 'larger_runner: ubuntu-24.04',
158-
'ci.yml build must pair each image'),
159-
# A later include entry overrides one leg's larger runner.
160-
(0, 'build', ' larger_runner: macos-26-xlarge\n',
161-
' larger_runner: macos-26-xlarge\n - os: ubuntu-24.04\n'
162-
' build_type: Debug\n larger_runner: ubuntu-24.04\n',
163-
'ci.yml build must pair each image'),
164-
(1, 'native-live', LINUX_RUNNER, 'ubuntu-24.04', 'native-live.yml job native-live must run on'),
165-
(4, 'corpus-parity', LINUX_RUNNER, 'ubuntu-24.04', 'corpus-parity.yml job corpus-parity must run on'),
166-
(4, 'corpus-parity-subset', LINUX_RUNNER, 'ubuntu-24.04',
167-
'corpus-parity.yml job corpus-parity-subset must run on'),
168-
# A fork's pull request let onto a larger runner, which bills the org.
151+
# Every heavy job stays on standard runners, whatever starts it.
169152
(0, 'preflight', LINUX_RUNNER, 'pf-linux-x64-16', 'ci.yml job preflight must run on'),
170-
(0, 'sanitizers', fork_head, 'github.event.pull_request.base.repo.full_name',
171-
'ci.yml job sanitizers must run on'),
172-
(0, 'kernel-only', f'{fork_head} == github.repository', f"{fork_head} != ''",
173-
'ci.yml job kernel-only must run on'),
174-
# Only named events no fork can raise are trusted; everything else a
175-
# fork can start -- pull_request_target, a comment, a workflow_run --
176-
# falls to the standard runner.
177-
(0, 'preflight', "github.event_name == 'push' || github.event_name == 'schedule' || "
178-
"github.event_name == 'workflow_dispatch'", "github.event_name != 'pull_request'",
153+
(0, 'sanitizers', LINUX_RUNNER, 'macos-26-xlarge', 'ci.yml job sanitizers must run on'),
154+
(0, 'kernel-only', LINUX_RUNNER, 'ubuntu-latest', 'ci.yml job kernel-only must run on'),
155+
(0, 'build', MATRIX_RUNNER, 'macos-26-large', 'ci.yml job build must run on'),
156+
(0, 'build', MATRIX_RUNNER, '${{ matrix.larger_runner }}', 'ci.yml job build must run on'),
157+
(1, 'native-live', LINUX_RUNNER, 'pf-linux-x64-16', 'native-live.yml job native-live must run on'),
158+
(4, 'corpus-parity', LINUX_RUNNER, 'pf-linux-x64-16', 'corpus-parity.yml job corpus-parity must run on'),
159+
(4, 'corpus-parity-subset', LINUX_RUNNER, 'pf-linux-x64-16',
160+
'corpus-parity.yml job corpus-parity-subset must run on'),
161+
# An event-based paid-runner route is forbidden, even for trusted events.
162+
(0, 'preflight', LINUX_RUNNER,
163+
"${{ github.event_name == 'push' && 'pf-linux-x64-16' || 'ubuntu-24.04' }}",
179164
'ci.yml job preflight must run on'),
180-
(1, 'native-live', "github.event_name == 'workflow_dispatch'",
181-
"github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request_target'",
165+
(1, 'native-live', LINUX_RUNNER,
166+
"${{ github.event_name == 'workflow_dispatch' && 'pf-linux-x64-16' || 'ubuntu-24.04' }}",
182167
'native-live.yml job native-live must run on'),
183-
# The head test counts only for a pull_request: anyone can review one.
184-
(0, 'sanitizers', "(github.event_name == 'pull_request' && "
185-
"github.event.pull_request.head.repo.full_name == github.repository)",
186-
'github.event.pull_request.head.repo.full_name == github.repository',
168+
(4, 'corpus-parity', LINUX_RUNNER,
169+
"${{ github.event_name == 'schedule' && 'pf-linux-x64-16' || 'ubuntu-24.04' }}",
170+
'corpus-parity.yml job corpus-parity must run on'),
171+
(0, 'sanitizers', LINUX_RUNNER,
172+
"${{ github.event.pull_request.head.repo.full_name == github.repository "
173+
"&& 'pf-linux-x64-16' || 'ubuntu-24.04' }}",
187174
'ci.yml job sanitizers must run on'),
188175
(0, 'native-live', 'uses: ./.github/workflows/native-live.yml',
189176
'uses: someone/else/.github/workflows/native-live.yml@main',
190177
'ci.yml job native-live calls a workflow the runner contract cannot see'),
191-
(0, 'build', MATRIX_RUNNER, '${{ matrix.larger_runner }}', 'ci.yml job build must run on'),
192-
(0, 'build', 'larger_runner: macos-26-xlarge', 'larger_runner: macos-26-large',
193-
'ci.yml build must pair each image'),
194-
# A leg whose os is a larger runner: the fork's fallback is the larger runner.
195-
(0, 'build', ' larger_runner: macos-26-xlarge\n',
196-
' larger_runner: macos-26-xlarge\n - os: pf-linux-x64-16\n'
197-
' build_type: Release\n', 'ci.yml build must pair each image'),
178+
# No paid image or include override, even one using a standard image.
179+
(0, 'build', 'os: [ubuntu-24.04, macos-26]', 'os: [ubuntu-24.04, macos-26-large]',
180+
'ci.yml build must use exactly the standard images'),
181+
(0, 'build', ' build_type: [Release, Debug]\n',
182+
' build_type: [Release, Debug]\n include:\n'
183+
' - os: ubuntu-24.04\n build_type: Debug\n',
184+
'ci.yml build must use exactly the standard images'),
185+
(0, 'build', ' build_type: [Release, Debug]\n',
186+
' build_type: [Release, Debug]\n include:\n'
187+
' - os: ubuntu-24.04\n larger_runner: ubuntu-24.04\n',
188+
'ci.yml must not define a larger_runner key'),
198189
# Block forms of runs-on, and job ids the parser must still see.
199190
(5, 'build', ' runs-on: ubuntu-latest\n', ' runs-on:\n group: pineforge-ci-large\n',
200191
'docs.yml job build must stay on a standard runner'),
@@ -216,8 +207,6 @@ def test_ci_contract_pins_runners_time_limits_and_the_fork_guard(self):
216207
' - run: "true"\n\n', 'docs.yml job pre_build must stay on a standard runner'),
217208
(5, None, '\njobs:\n', '\njobs:\n "pre-build":\n runs-on: macos-26-xlarge\n steps:\n'
218209
' - run: "true"\n\n', 'docs.yml has jobs lines the runner contract cannot read'),
219-
(1, 'native-live', "|| 'ubuntu-24.04'", "|| 'pf-linux-x64-16'",
220-
'native-live.yml job native-live must run on'),
221210
(4, 'corpus-parity-subset', f'runs-on: {LINUX_RUNNER}', 'runs-on:\n group: pineforge-ci-large',
222211
'corpus-parity.yml job corpus-parity-subset must run on'),
223212
(0, 'build-gate', 'runs-on: ubuntu-24.04', 'runs-on: pf-linux-x64-16',
@@ -251,7 +240,7 @@ def test_ci_contract_pins_runners_time_limits_and_the_fork_guard(self):
251240
'corpus-parity.yml job corpus-parity must allow 360'),
252241
(4, 'corpus-parity-subset', 'timeout-minutes: 360', 'timeout-minutes: 10',
253242
'corpus-parity.yml job corpus-parity-subset must allow 360'),
254-
# Four jobs asked of a 16-core runner.
243+
# A fixed count cannot follow the runner's available cores.
255244
(0, 'build', f'--jobs {CORES}', '--jobs 4', 'ci.yml job build must size'),
256245
(0, 'sanitizers', f'--jobs {CORES}', '--jobs 4', 'ci.yml job sanitizers must size'),
257246
(0, 'kernel-only', f'--jobs {CORES}', '--jobs "$(nproc)"', 'ci.yml job kernel-only must size'),
@@ -507,7 +496,9 @@ def test_harmless_spellings_are_not_findings(self):
507496
edits = (
508497
(0, 'kernel-only', ' steps:\n',
509498
' # Was --jobs 4, and JOBS: "4", on the standard runner.\n steps:\n'),
510-
(0, 'build', ' include:\n', ' include:\n\n # one per image\n'),
499+
(0, 'build', ' matrix:\n', ' matrix:\n\n # standard images only\n'),
500+
(0, 'build', ' matrix:\n',
501+
' matrix:\n # Was larger_runner: macos-26-xlarge\n'),
511502
(0, 'build', "&& '--exclude-label slow' || '' }}\n",
512503
"&& '--exclude-label slow' || '' }} # was --jobs 4\n"),
513504
(0, 'kernel-only', verify,
@@ -553,6 +544,27 @@ def test_a_finding_is_reported_once(self):
553544
findings = ci_workflow_findings(*changed, others=self.other_sources())
554545
self.assertEqual(findings.count('ci.yml is missing job kernel-only'), 1, findings)
555546

547+
def test_paid_labels_and_larger_runner_keys_are_refused_even_in_release(self):
548+
original, others = self.ci_sources(), self.other_sources()
549+
standard = ' runs-on: ubuntu-24.04\n'
550+
workflow = ('name: lint\non: workflow_dispatch\njobs:\n lint:\n' + standard
551+
+ ' steps:\n - run: "true"\n')
552+
mutations = [(f' runs-on: "{label}"\n', 'must not use larger-runner labels')
553+
for label in ('pf-linux-x64-16', 'macos-26-xlarge', 'macos-26-large',
554+
'ubuntu-24.04-xlarge', 'windows-2025-large')]
555+
mutations += [(f' strategy:\n matrix: {{{key}: ubuntu-24.04}}\n' + standard,
556+
'must not define a larger_runner key')
557+
for key in ('larger_runner', '"larger_runner"', "'larger_runner'")]
558+
for name in ('lint.yml', 'release.yml'):
559+
self.assertEqual(ci_workflow_findings(
560+
*original, others=dict(others, **{name: workflow})), [])
561+
for replacement, finding in mutations:
562+
with self.subTest(name=name, replacement=replacement):
563+
changed = workflow.replace(standard, replacement, 1)
564+
findings = ci_workflow_findings(
565+
*original, others=dict(others, **{name: changed}))
566+
self.assertTrue(any(f'{name} {finding}' in line for line in findings), findings)
567+
556568
def test_every_workflow_a_pull_request_can_start_stays_on_standard_runners(self):
557569
original, others = self.ci_sources(), self.other_sources()
558570
# release.yml starts only on a dispatch, so no pull request reaches its runners.
@@ -627,14 +639,17 @@ def contract_stage(self, mutation):
627639
cwd=tree, capture_output=True, text=True)
628640
return result.returncode, result.stdout + result.stderr
629641

630-
def test_the_preflight_stage_fails_a_slow_or_fork_reachable_runner(self):
642+
def test_the_preflight_stage_fails_a_paid_runner_or_larger_runner_key(self):
631643
code, output = self.contract_stage(None)
632644
self.assertEqual(code, 0, output)
633645
self.assertIn('runners and time limits OK', output)
634646
for mutation, finding in (
635-
(('sanitizers', LINUX_RUNNER, 'ubuntu-24.04'), 'ci.yml job sanitizers must run on'),
636-
(('sanitizers', 'pull_request.head.repo', 'pull_request.base.repo'),
637-
'ci.yml job sanitizers must run on')):
647+
(('sanitizers', LINUX_RUNNER, 'pf-linux-x64-16'), 'ci.yml job sanitizers must run on'),
648+
(('build', 'os: [ubuntu-24.04, macos-26]', 'os: [ubuntu-24.04, macos-26-xlarge]'),
649+
'ci.yml must not use larger-runner labels'),
650+
(('build', ' build_type: [Release, Debug]\n',
651+
' build_type: [Release, Debug]\n larger_runner: ubuntu-24.04\n'),
652+
'ci.yml must not define a larger_runner key')):
638653
with self.subTest(mutation=mutation):
639654
code, output = self.contract_stage(mutation)
640655
self.assertEqual(code, 1, output)

0 commit comments

Comments
 (0)