Skip to content

Commit 35938b5

Browse files
committed
Explain dependencies from the image a RunsOn canary keeps in S3
Since #74889 scheduled canaries run on RunsOn, where the CI image stash goes to RunsOn's S3 cache and only its digest is a GitHub artifact, so every report entry was skipped as "CI image missing". The plan now also accepts that digest artifact; the explain job then runs on a RunsOn runner, downloads exactly that tarball by its digest and checks it like stash-restore does. It prepares the disk with the workflow version's runner scripts, as the image build job now does.
1 parent 1e66c2b commit 35938b5

3 files changed

Lines changed: 120 additions & 16 deletions

File tree

‎.github/workflows/dependency-upgrade-report.yml‎

Lines changed: 60 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,9 @@ jobs:
7676
const inputName = new RegExp('^dependency-report-inputs-(3\\.[0-9]+)-' +
7777
'(constraints(?:-source-providers|-no-providers)?)-(true|false)-([1-9][0-9]*)$');
7878
const saved = artifacts.map(a => [a, inputName.exec(a.name)]).filter(([, match]) => match);
79+
// Only RunsOn runners can read RunsOn's S3 cache, where a canary on RunsOn keeps its CI image.
80+
const runsOnRunner = `runs-on=${process.env.GITHUB_RUN_ID}/family=m7a+m7i+m6a+m6i` +
81+
'/image=ubuntu22-full-x64/volume=150gb/cpu=8/ram=32';
7982
for (const [input, [, python, mode, useUv, attempt]] of saved) {
8083
if (attempt !== rawAttempt) continue;
8184
const entry = `${python}:${mode}`;
@@ -86,16 +89,21 @@ jobs:
8689
continue;
8790
}
8891
// The stash action names the artifact after the stash key, with '/' as '_', and the branch.
89-
const image = artifacts.find(a => a.name === `ci-image-save-v3-linux_amd64-${python}-main`);
92+
// On RunsOn runners the image itself goes to RunsOn's S3 cache and the artifact is only
93+
// its digest, stashed under the key plus '-s3' (see .github/actions/stash-save).
94+
const imageName = `ci-image-save-v3-linux_amd64-${python}-main`;
95+
const image = artifacts.find(a => a.name === imageName) ||
96+
artifacts.find(a => a.name === `ci-image-save-v3-linux_amd64-${python}-s3-main`);
9097
const missing = input.expired ? 'inputs expired' :
9198
!image ? 'CI image missing' : image.expired ? 'CI image expired' : '';
9299
if (missing) {
93100
core.warning(`Skipping ${entry}: original inputs or CI image unavailable (${missing})`);
94101
skipped.push(`${entry} (${missing})`);
95102
continue;
96103
}
97-
include.push({ python, mode, 'use-uv': useUv,
98-
'inputs-id': input.id, 'image-id': image.id });
104+
const fromS3 = image.name !== imageName;
105+
include.push({ python, mode, 'use-uv': useUv, 'inputs-id': input.id, 'image-id': image.id,
106+
'image-source': fromS3 ? 's3' : 'artifact', runner: fromS3 ? runsOnRunner : 'ubuntu-22.04' });
99107
}
100108
core.setOutput('matrix', JSON.stringify({ include }));
101109
core.setOutput('has-reports', include.length > 0 ? 'true' : 'false');
@@ -120,7 +128,7 @@ jobs:
120128
needs: plan
121129
if: needs.plan.outputs.has-reports == 'true'
122130
name: "Explain ${{ matrix.python }}:${{ matrix.mode }}"
123-
runs-on: ubuntu-22.04
131+
runs-on: ${{ matrix.runner }}
124132
# Explaining constraints-source-providers took up to 115 minutes in the canary (Oct 2026).
125133
timeout-minutes: 150
126134
strategy:
@@ -139,14 +147,23 @@ jobs:
139147
with:
140148
ref: ${{ needs.plan.outputs.source-sha }}
141149
persist-credentials: false
142-
# Docker moves to /mnt as in the image build jobs: the loaded image and uv resolutions outgrow /.
150+
# The disk is prepared as in the image build jobs: the loaded image and uv resolutions outgrow /,
151+
# and the runner scripts come from the workflow's own version, which knows RunsOn's disks.
152+
- name: "Checkout the workflow version for its actions and runner scripts"
153+
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
154+
with:
155+
persist-credentials: false
156+
path: workflow-version
157+
sparse-checkout: |
158+
.github/actions
159+
scripts/ci
143160
- name: "Free up disk space"
144161
shell: bash
145162
run: ./scripts/tools/free_up_disk_space.sh
146163
- name: "Make /mnt writeable"
147-
run: ./scripts/ci/make_mnt_writeable.sh
164+
run: ./workflow-version/scripts/ci/make_mnt_writeable.sh
148165
- name: "Move docker to /mnt"
149-
run: ./scripts/ci/move_docker_to_mnt.sh
166+
run: ./workflow-version/scripts/ci/move_docker_to_mnt.sh
150167
- name: "Install Breeze"
151168
uses: ./.github/actions/breeze
152169
- name: "Download the original CI image"
@@ -157,6 +174,42 @@ jobs:
157174
github-token: ${{ secrets.GITHUB_TOKEN }}
158175
merge-multiple: true
159176
path: /mnt
177+
if: matrix.image-source == 'artifact'
178+
- name: "Download the digest of the original CI image"
179+
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
180+
with:
181+
artifact-ids: ${{ matrix.image-id }}
182+
run-id: ${{ needs.plan.outputs.source-run-id }}
183+
github-token: ${{ secrets.GITHUB_TOKEN }}
184+
merge-multiple: true
185+
path: ${{ runner.temp }}/ci-image-digest
186+
if: matrix.image-source == 's3'
187+
# The tarball is the one the canary stashed, addressed by its digest as in
188+
# .github/actions/stash-restore, so a later build on `main` cannot replace it.
189+
- name: "Restore the original CI image from the RunsOn cache"
190+
shell: bash
191+
run: |
192+
set -euo pipefail
193+
digest="$(cat "${RUNNER_TEMP}/ci-image-digest/sha256")"
194+
if [[ ! "${digest}" =~ ^[0-9a-f]{64}$ ]]; then
195+
echo "::error::Malformed digest of the original CI image"
196+
exit 1
197+
fi
198+
export AWS_DEFAULT_REGION="${RUNS_ON_AWS_REGION:?not running on a RunsOn runner}"
199+
bucket="${RUNS_ON_S3_BUCKET_CACHE:?not running on a RunsOn runner}"
200+
object="s3://${bucket}/cache/airflow-stash/sha256/${digest}.tar"
201+
payload="${RUNNER_TEMP}/ci-image-stash.tar"
202+
aws s3 cp --only-show-errors "${object}" "${payload}"
203+
echo "${digest} ${payload}" | sha256sum --check --status
204+
tar -tPf "${payload}" | while IFS= read -r entry; do
205+
if [[ "${entry}" != /mnt/* ]]; then
206+
echo "::error::The original CI image stash holds ${entry} outside /mnt"
207+
exit 1
208+
fi
209+
done
210+
tar -xPf "${payload}"
211+
rm -f "${payload}"
212+
if: matrix.image-source == 's3'
160213
- name: "Load the original CI image"
161214
env:
162215
PYTHON_VERSION: ${{ matrix.python }}

‎dev/breeze/doc/ci/05_workflows.md‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -467,7 +467,9 @@ ID from that run. It never substitutes the latest branch image or constraints: a
467467
saved constraints or image are missing or expired is skipped with a warning and listed in the
468468
summary, and the others are still explained. The image comes from the CI image stash, which
469469
keeps only the newest image per Python version and branch, so a later build on `main` can
470-
replace it within hours. Re-running the canary triggers the workflow again; inputs that an earlier attempt
470+
replace it within hours. A canary on RunsOn keeps the image in RunsOn's S3 cache and stashes
471+
only its digest; the report job then runs on a RunsOn runner and downloads exactly that tarball
472+
by its digest. Re-running the canary triggers the workflow again; inputs that an earlier attempt
471473
already saved are not explained again.
472474

473475
The saved constraints are the constraints branch as the summary saw it, not constraints

‎dev/breeze/tests/test_dependency_upgrade_report.py‎

Lines changed: 57 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import pytest
2626
import yaml
2727

28+
from airflow_breeze.global_constants import RUNS_ON_AMD_RUNNER_TEMPLATE
2829
from airflow_breeze.utils.path_utils import AIRFLOW_ROOT_PATH
2930

3031
NODE_HARNESS = """
@@ -33,6 +34,7 @@
3334
process.env.SOURCE_RUN_ID = input.id;
3435
process.env.SOURCE_RUN_ATTEMPT = input.sourceAttempt;
3536
process.env.GITHUB_RUN_ATTEMPT = String(input.attempt);
37+
process.env.GITHUB_RUN_ID = '456';
3638
const result = { outputs: {}, requests: [], notices: [], warnings: [], summary: '' };
3739
const summary = {
3840
addHeading() { return this; }, addLink() { return this; },
@@ -141,6 +143,8 @@ def test_plan_keeps_original_inputs_when_the_report_is_rerun(source_run, source_
141143
"use-uv": "true",
142144
"inputs-id": 1,
143145
"image-id": 2,
146+
"image-source": "artifact",
147+
"runner": "ubuntu-22.04",
144148
}
145149
]
146150
assert all(request["run_id"] == 123 for request in result["requests"])
@@ -251,21 +255,64 @@ def stash_artifact_name(key, ref):
251255
return "-".join(re.sub(r"[^_\-.\w]", "_", part) for part in (key, ref))
252256

253257

254-
@requires_node
255-
def test_plan_finds_the_image_under_the_name_the_canary_stashes_it(source_run, source_artifacts):
258+
def image_stash_key():
256259
steps = load_workflow("workflows/ci-image-build.yml")["jobs"]["build-ci-images"]["steps"]
257260
keys = [
258261
step["with"]["key"]
259262
for step in steps
260-
if step.get("uses", "").startswith("apache/infrastructure-actions/stash/save@")
263+
if step.get("uses", "").endswith("/stash-save")
261264
and step["with"]["key"].startswith("ci-image-save-")
262265
and "image-stash-ref" not in step["with"]["key"]
263266
]
264267
assert keys == ["ci-image-save-v3-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}"]
265268
key = keys[0].replace("${{ inputs.platform }}", "linux/amd64")
266-
key = key.replace("${{ env.PYTHON_MAJOR_MINOR_VERSION }}", "3.14")
269+
return key.replace("${{ env.PYTHON_MAJOR_MINOR_VERSION }}", "3.14")
270+
271+
272+
def runs_on_digest_suffix():
273+
"""The suffix .github/actions/stash-save adds to the key of the digest it stashes on RunsOn."""
274+
steps = load_workflow("actions/stash-save/action.yml")["runs"]["steps"]
275+
digest = next(step for step in steps if step["name"] == "Save stash digest to GitHub artifacts")
276+
return digest["with"]["key"].removeprefix("${{ inputs.key }}")
277+
278+
279+
@requires_node
280+
@pytest.mark.parametrize(
281+
("runs_on", "image_source", "runner"),
282+
[
283+
pytest.param(False, "artifact", "ubuntu-22.04", id="github-hosted-canary"),
284+
pytest.param(
285+
True, "s3", RUNS_ON_AMD_RUNNER_TEMPLATE.format(run_id="456").format(8, 32), id="runs-on-canary"
286+
),
287+
],
288+
)
289+
def test_plan_finds_the_image_under_the_name_the_canary_stashes_it(
290+
source_run, source_artifacts, runs_on, image_source, runner
291+
):
292+
key = image_stash_key() + (runs_on_digest_suffix() if runs_on else "")
267293
source_artifacts[1]["name"] = stash_artifact_name(key, "main")
268-
assert included_inputs(run_plan(source_run, source_artifacts)) == [1]
294+
[entry] = json.loads(run_plan(source_run, source_artifacts)["outputs"]["matrix"])["include"]
295+
assert entry["inputs-id"] == 1
296+
assert entry["image-id"] == 2
297+
assert entry["image-source"] == image_source
298+
assert entry["runner"] == runner
299+
300+
301+
def test_explain_restores_a_runs_on_image_where_stash_save_put_it():
302+
stash_save = (AIRFLOW_ROOT_PATH / ".github" / "actions" / "stash-save" / "action.yml").read_text()
303+
assert 'prefix="s3://${bucket}/cache/airflow-stash"' in stash_save
304+
assert 'object="${prefix}/sha256/${digest}.tar"' in stash_save
305+
steps = load_workflow("workflows/dependency-upgrade-report.yml")["jobs"]["explain"]["steps"]
306+
by_name = {step["name"]: step for step in steps}
307+
restore = by_name["Restore the original CI image from the RunsOn cache"]
308+
assert 'object="s3://${bucket}/cache/airflow-stash/sha256/${digest}.tar"' in restore["run"]
309+
assert "sha256sum --check" in restore["run"]
310+
assert restore["if"] == "matrix.image-source == 's3'"
311+
assert by_name["Download the digest of the original CI image"]["if"] == "matrix.image-source == 's3'"
312+
assert by_name["Download the original CI image"]["if"] == "matrix.image-source == 'artifact'"
313+
assert load_workflow("workflows/dependency-upgrade-report.yml")["jobs"]["explain"]["runs-on"] == (
314+
"${{ matrix.runner }}"
315+
)
269316

270317

271318
def test_each_canary_attempt_triggers_its_own_report():
@@ -282,12 +329,14 @@ def test_explain_prepares_the_disk_like_the_image_build_jobs():
282329
build_steps = load_workflow("workflows/ci-image-build.yml")["jobs"]["build-ci-images"]["steps"]
283330
steps = load_workflow("workflows/dependency-upgrade-report.yml")["jobs"]["explain"]["steps"]
284331
names = [step["name"] for step in steps]
332+
workflow_version = "Checkout the workflow version for its actions and runner scripts"
285333
disk_steps = ["Free up disk space", "Make /mnt writeable", "Move docker to /mnt"]
286-
assert names[1:5] == [*disk_steps, "Install Breeze"]
334+
assert names[1:6] == [workflow_version, *disk_steps, "Install Breeze"]
287335
assert names.index("Install Breeze") < names.index("Download the original CI image")
288-
build_runs = {step["name"]: step.get("run") for step in build_steps}
336+
build = {step["name"]: step for step in build_steps}
337+
assert steps[names.index(workflow_version)]["with"] == build[workflow_version]["with"]
289338
for name in disk_steps:
290-
assert steps[names.index(name)]["run"] == build_runs[name]
339+
assert steps[names.index(name)]["run"] == build[name]["run"]
291340

292341

293342
@pytest.mark.parametrize(

0 commit comments

Comments
 (0)