Skip to content

Commit 7f99a31

Browse files
committed
ci: retry lost on-demand shards
Treat on-demand amd64 labels as part of the runner-loss retry fleet, cover the new labels with a real acceptance row, remove the unused old label, and document the rollback path.
1 parent 9f4ad0c commit 7f99a31

5 files changed

Lines changed: 7 additions & 4 deletions

File tree

‎.github/actionlint.yaml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@ self-hosted-runner:
77
- gke-2cpu-4gb-amd64
88
- gke-4cpu-8gb-amd64
99
- gke-32cpu-64gb-amd64
10-
- gke-32cpu-64gb-amd64-256gb
1110
- gke-ondemand-32cpu-64gb-amd64
1211
- gke-ondemand-32cpu-64gb-amd64-256gb
1312
- gke-16cpu-32gb-amd64

‎docs/ci.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,9 @@ refuse the retry; three total attempts are allowed. PRs, manual dispatches and
3636
release/publish/deploy/promotion workflows are excluded. The controller runs on
3737
hosted `ubuntu-24.04`, uses its trusted default-branch workflow commit, and never
3838
prints log bodies. Missing `gh`, `python3` or the log escape flag fails the job.
39+
The post-merge on-demand amd64 labels are included in the same loss-retry fleet.
40+
If the on-demand routing is rolled back, land a revert PR for the label commit
41+
before the next post-merge run.
3942

4043
A GitHub-only PR changes at least one path, all under `.github/`, including both sides of a rename.
4144
`scripts/ci_*.py` changes, alone or mixed with `.github/`, follow the ordinary proof plan.

‎scripts/ci/rerun_preempted.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@
3737
}
3838
# Bare amd64 GKE scale sets, optionally with the fleet's disk-size suffix.
3939
# No arm64, gkp pools or operator-dispatched entrypoints are admitted.
40-
FLEET_LABEL = re.compile(r'gke-(?:2|4|8|16|32)cpu-(?:4|8|16|32|64)gb-amd64(?:-(?:64|128|256|512)gb)?')
40+
FLEET_LABEL = re.compile(r'gke-(?:ondemand-)?(?:2|4|8|16|32)cpu-(?:4|8|16|32|64)gb-amd64(?:-(?:64|128|256|512)gb)?')
4141
LOSS_PATTERNS = (
4242
re.compile(r'The runner has received a shutdown signal\..*', re.I),
4343
re.compile(r'(?:The runner[: ]+.*?)?lost communication with the server\b.*', re.I),

‎scripts/ci_preflight.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,6 @@
122122
'release-ruleset-probe.yml': {'probe': 'release'},
123123
'promote-baseline.yml': {'promote': 'release-bot'},
124124
}
125-
GKE_32_256_LABEL = 'gke-32cpu-64gb-amd64-256gb'
126125
GKE_ONDEMAND_32_256_LABEL = 'gke-ondemand-32cpu-64gb-amd64-256gb'
127126
GKE_16_128_LABEL = 'gke-16cpu-32gb-amd64-128gb'
128127
GKE_16_256_LABEL = 'gke-16cpu-32gb-amd64-256gb'

‎scripts/test_ci_rerun_preempted.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -182,7 +182,9 @@ def test_wave6_cut_reruns_only_failed_jobs(self):
182182

183183
def test_plain_and_disk_sized_amd64_labels(self):
184184
for label in ('gke-8cpu-16gb-amd64', 'gke-32cpu-64gb-amd64',
185-
'gke-32cpu-64gb-amd64-256gb', 'gke-16cpu-32gb-amd64-128gb'):
185+
'gke-ondemand-32cpu-64gb-amd64',
186+
'gke-ondemand-32cpu-64gb-amd64-256gb',
187+
'gke-16cpu-32gb-amd64-128gb'):
186188
with self.subTest(label=label):
187189
job = deepcopy(CUT_JOB)
188190
job['labels'] = [label]

0 commit comments

Comments
 (0)