Skip to content

Commit b5bc253

Browse files
shahar1shahar1
authored
Run maintainer and canary CI on self-hosted RunsOn runners (#74889)
This PR implements AIP-118 for integrating self-hosted runners in Apache Airflow's CI, utilizing the project's AWS instance and runs-on.com's runners.
1 parent 054d093 commit b5bc253

33 files changed

Lines changed: 1112 additions & 152 deletions

‎.github/actions/breeze/action.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@ outputs:
2929
runs:
3030
using: "composite"
3131
steps:
32+
- name: "RunsOn cost summary"
33+
uses: ./.github/actions/runs-on-cost-summary
3234
- name: "Setup python"
3335
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
3436
with:

‎.github/actions/prepare_breeze_and_image/action.yml‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -82,12 +82,13 @@ runs:
8282
- name: >
8383
Restore ${{ inputs.image-type }} docker image ${{ inputs.platform }}:${{ inputs.python }}
8484
${{ inputs.image-stash-ref != '' && format('built for ref {0}', inputs.image-stash-ref) || '' }}
85-
# yamllint disable-line rule:line-length
86-
uses: apache/infrastructure-actions/stash/restore@61dcea11f19e2bbe1263f14d72235e8da17d3ad0 # restore/v1.0.0
85+
uses: ./.github/actions/stash-restore
8786
with:
8887
key: "${{ inputs.image-type }}-image-save-v3-${{ inputs.platform }}-${{ inputs.python }}\
8988
${{ inputs.image-stash-ref != '' && format('-{0}', inputs.image-stash-ref) || '' }}"
9089
path: "/mnt/"
90+
# Must stay true: runs on RunsOn (canaries and release-branch pushes) save these image
91+
# stashes as S3 digests only.
9192
only-current-branch: 'true'
9293
fail-on-download: 'true'
9394
- name: "Load ${{ inputs.image-type }} image ${{ inputs.platform }}:${{ inputs.python }}"

‎.github/actions/prepare_single_ci_image/action.yml‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,11 +36,12 @@ runs:
3636
using: "composite"
3737
steps:
3838
- name: "Restore CI docker images ${{ inputs.platform }}:${{ inputs.python }}"
39-
# yamllint disable-line rule:line-length
40-
uses: apache/infrastructure-actions/stash/restore@61dcea11f19e2bbe1263f14d72235e8da17d3ad0 # restore/v1.0.0
39+
uses: ./.github/actions/stash-restore
4140
with:
4241
key: ci-image-save-v3-${{ inputs.platform }}-${{ inputs.python }}
4342
path: "/mnt/"
43+
# Must stay true: runs on RunsOn (canaries and release-branch pushes) save these image
44+
# stashes as S3 digests only.
4445
only-current-branch: 'true'
4546
fail-on-download: 'true'
4647
if: contains(inputs.python-versions-list-as-string, inputs.python)

‎.github/actions/runs-on-cost-summary/action.yml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
#
18+
---
19+
# Puts the job's EC2 cost next to the GitHub-hosted price in the job summary. Used by every job
20+
# that can run on RunsOn: through the breeze action, or directly by jobs that do not install Breeze.
21+
name: 'RunsOn cost summary'
22+
description: 'Add the EC2 cost of a RunsOn job to its summary; a no-op elsewhere.'
23+
runs:
24+
using: 'composite'
25+
steps:
26+
- name: "RunsOn cost summary"
27+
if: runner.environment == 'self-hosted'
28+
uses: runs-on/action@17b59092eb37ebfdbb034460892887c09100eaa3 # v2.5.0
29+
with:
30+
show_costs: summary

‎.github/actions/stash-restore/action.yml‎

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
#
18+
---
19+
# apache/infrastructure-actions/stash/restore on GitHub-hosted runners. On RunsOn (the only
20+
# self-hosted runners) multi-GB image tarballs stay in the RunsOn stack's in-region S3 cache
21+
# bucket instead of leaving AWS as GitHub artifacts.
22+
#
23+
# Every RunsOn job can write anywhere in that bucket, so the bucket is not trusted: objects are
24+
# named by their sha256, and the digest itself travels as a small ASF stash artifact, which
25+
# GitHub scopes to the run / branch exactly like a regular stash. A blob that does not match the
26+
# digest the ASF restore found for this run, branch and repository is never extracted. The bucket's
27+
# lifecycle rule (S3CacheExpirationInDays) must outlive the longest retention-days of a digest
28+
# (6 days today), or a digest hit finds no blob.
29+
name: 'Restore stash'
30+
description: 'Restore a stash from GitHub artifacts, or from S3 on RunsOn runners.'
31+
inputs:
32+
key:
33+
description: 'Stash key.'
34+
required: true
35+
path:
36+
description: >
37+
Directory to restore into (GitHub-hosted only; the S3 tarball restores entries to the
38+
paths they were saved from, and the restore then fails if any of them is outside this directory).
39+
required: false
40+
default: ${{ github.workspace }}
41+
only-current-branch:
42+
description: 'Do not fall back to the base branch stash.'
43+
required: false
44+
default: 'false'
45+
fail-on-download:
46+
description: 'Fail when a stash was found but could not be restored.'
47+
required: false
48+
default: 'false'
49+
outputs:
50+
stash-hit:
51+
description: "'true' if a stash was restored, 'false' otherwise."
52+
value: ${{ steps.s3.outputs.stash-hit || steps.artifact.outputs.stash-hit }}
53+
runs:
54+
using: 'composite'
55+
steps:
56+
- name: "Restore stash from GitHub artifacts"
57+
id: artifact
58+
if: runner.environment != 'self-hosted'
59+
# yamllint disable-line rule:line-length
60+
uses: apache/infrastructure-actions/stash/restore@c968777141a2afc98d1a09c9f6979455bfebaae3 # restore/v1.0.1
61+
with:
62+
key: ${{ inputs.key }}
63+
path: ${{ inputs.path }}
64+
only-current-branch: ${{ inputs.only-current-branch }}
65+
fail-on-download: ${{ inputs.fail-on-download }}
66+
- name: "Restore stash digest from GitHub artifacts"
67+
id: digest
68+
if: runner.environment == 'self-hosted'
69+
# yamllint disable-line rule:line-length
70+
uses: apache/infrastructure-actions/stash/restore@c968777141a2afc98d1a09c9f6979455bfebaae3 # restore/v1.0.1
71+
with:
72+
key: ${{ inputs.key }}-s3
73+
path: ${{ runner.temp }}/stash-digest-restore/${{ inputs.key }}
74+
only-current-branch: ${{ inputs.only-current-branch }}
75+
fail-on-download: ${{ inputs.fail-on-download }}
76+
- name: "Restore stash from S3"
77+
id: s3
78+
if: runner.environment == 'self-hosted'
79+
shell: bash
80+
env:
81+
STASH_KEY: ${{ inputs.key }}
82+
STASH_PATH: ${{ inputs.path }}
83+
DIGEST_HIT: ${{ steps.digest.outputs.stash-hit }}
84+
DIGEST_FILE: ${{ runner.temp }}/stash-digest-restore/${{ inputs.key }}/sha256
85+
FAIL_ON_DOWNLOAD: ${{ inputs.fail-on-download }}
86+
run: |
87+
set -uo pipefail
88+
payload="${RUNNER_TEMP}/stash-restore-$$.tar"
89+
entries="${RUNNER_TEMP}/stash-restore-$$.entries"
90+
miss() {
91+
rm -f "${payload}" "${entries}"
92+
echo "stash-hit=false" >> "${GITHUB_OUTPUT}"
93+
if [[ "${FAIL_ON_DOWNLOAD}" == "true" && "${DIGEST_HIT}" == "true" ]]; then
94+
echo "::error::$1"
95+
exit 1
96+
fi
97+
echo "$1"
98+
exit 0
99+
}
100+
if [[ "${DIGEST_HIT}" != "true" ]]; then
101+
miss "No stash '${STASH_KEY}'"
102+
fi
103+
digest="$(cat "${DIGEST_FILE}")"
104+
if [[ ! "${digest}" =~ ^[0-9a-f]{64}$ ]]; then
105+
miss "Malformed digest for stash '${STASH_KEY}'"
106+
fi
107+
bucket="${RUNS_ON_S3_BUCKET_CACHE:?not running on a RunsOn runner}"
108+
export AWS_DEFAULT_REGION="${RUNS_ON_AWS_REGION:?not running on a RunsOn runner}"
109+
object="s3://${bucket}/cache/airflow-stash/sha256/${digest}.tar"
110+
if ! aws s3 cp --only-show-errors "${object}" "${payload}"; then
111+
miss "Could not download ${object} for stash '${STASH_KEY}'"
112+
fi
113+
if ! echo "${digest} ${payload}" | sha256sum --check --status; then
114+
miss "${object} does not match the digest recorded for stash '${STASH_KEY}'"
115+
fi
116+
# GNU tar detects the compression stash-save chose.
117+
if ! tar -xPvf "${payload}" > "${entries}"; then
118+
miss "Could not extract ${object} for stash '${STASH_KEY}'"
119+
fi
120+
rm -f "${payload}"
121+
# The S3 tarball ignores `path`: fail loudly if a caller saves and restores different
122+
# directories, which would work on GitHub-hosted runners only.
123+
restore_dir="$(realpath -ms "${STASH_PATH}")"
124+
restore_dir="${restore_dir%/}"
125+
outside=()
126+
while IFS= read -r entry; do
127+
[[ "${entry}" == /* ]] || entry="${GITHUB_WORKSPACE}/${entry}"
128+
[[ "${entry}" == "${restore_dir}/"* ]] || outside+=("${entry}")
129+
done < "${entries}"
130+
rm -f "${entries}"
131+
if [[ ${#outside[@]} -gt 0 ]]; then
132+
printf '%s\n' "${outside[@]}"
133+
echo "::error::Stash '${STASH_KEY}' restored the entries above outside of ${restore_dir}/"
134+
exit 1
135+
fi
136+
echo "stash-hit=true" >> "${GITHUB_OUTPUT}"
137+
echo "Restored stash '${STASH_KEY}' from ${object}"

‎.github/actions/stash-save/action.yml‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
#
18+
---
19+
# apache/infrastructure-actions/stash/save on GitHub-hosted runners. On RunsOn the tarball goes
20+
# to the stack's S3 cache bucket under its sha256 and only the digest is saved as an ASF stash
21+
# artifact. See stash-restore for why.
22+
name: 'Save stash'
23+
description: 'Save a stash to GitHub artifacts, or to S3 on RunsOn runners.'
24+
inputs:
25+
key:
26+
description: 'Stash key.'
27+
required: true
28+
path:
29+
description: 'File, directory or glob to stash.'
30+
required: true
31+
if-no-files-found:
32+
description: "'error' fails when path matches nothing, 'warn' or 'ignore' do not."
33+
required: false
34+
default: 'warn'
35+
retention-days:
36+
description: >
37+
Days the stash (on RunsOn: its digest) is kept. On RunsOn the stack's
38+
S3CacheExpirationInDays lifecycle rule expires the tarball itself.
39+
required: false
40+
default: '5'
41+
compression-level:
42+
description: "Zlib compression level (0-9). On RunsOn, '0' stores the tarball uncompressed."
43+
required: false
44+
default: '6'
45+
runs:
46+
using: 'composite'
47+
steps:
48+
- name: "Save stash to GitHub artifacts"
49+
if: runner.environment != 'self-hosted'
50+
uses: apache/infrastructure-actions/stash/save@7d3afde57801d8c9a2db9730c28ae4e036cc2ab9 # save/v1.0.2
51+
with:
52+
key: ${{ inputs.key }}
53+
path: ${{ inputs.path }}
54+
if-no-files-found: ${{ inputs.if-no-files-found }}
55+
retention-days: ${{ inputs.retention-days }}
56+
compression-level: ${{ inputs.compression-level }}
57+
- name: "Save stash to S3"
58+
id: s3
59+
if: runner.environment == 'self-hosted'
60+
shell: bash
61+
env:
62+
STASH_KEY: ${{ inputs.key }}
63+
STASH_PATH: ${{ inputs.path }}
64+
DIGEST_DIR: ${{ runner.temp }}/stash-digest-save/${{ inputs.key }}
65+
IF_NO_FILES_FOUND: ${{ inputs.if-no-files-found }}
66+
COMPRESSION_LEVEL: ${{ inputs.compression-level }}
67+
run: |
68+
set -euo pipefail
69+
bucket="${RUNS_ON_S3_BUCKET_CACHE:?not running on a RunsOn runner}"
70+
export AWS_DEFAULT_REGION="${RUNS_ON_AWS_REGION:?not running on a RunsOn runner}"
71+
shopt -s nullglob
72+
# shellcheck disable=SC2206
73+
candidates=( ${STASH_PATH} )
74+
existing=()
75+
for p in "${candidates[@]}"; do
76+
if [[ -e "${p}" ]]; then
77+
existing+=("${p}")
78+
fi
79+
done
80+
if [[ ${#existing[@]} -eq 0 ]]; then
81+
if [[ "${IF_NO_FILES_FOUND}" == "error" ]]; then
82+
echo "::error::No files found for path '${STASH_PATH}'"
83+
exit 1
84+
fi
85+
if [[ "${IF_NO_FILES_FOUND}" == "warn" ]]; then
86+
echo "::warning::No files found for path '${STASH_PATH}', skipping save"
87+
fi
88+
exit 0
89+
fi
90+
# The tarball is streamed to S3 while being hashed: a multi-GB image never touches the
91+
# runner's disk. Its key is only known at the end, so it lands under a temporary key and
92+
# is copied server-side to its digest key.
93+
prefix="s3://${bucket}/cache/airflow-stash"
94+
upload="${prefix}/uploads/$(cat /proc/sys/kernel/random/uuid).tar"
95+
digest_fifo="${RUNNER_TEMP}/stash-save-$$.fifo"
96+
digest_file="${RUNNER_TEMP}/stash-save-$$.sha256"
97+
trap 'rm -f "${digest_fifo}" "${digest_file}"' EXIT
98+
mkfifo "${digest_fifo}"
99+
sha256sum < "${digest_fifo}" | cut -d ' ' -f 1 > "${digest_file}" &
100+
digest_pid=$!
101+
compression=()
102+
if [[ "${COMPRESSION_LEVEL}" != "0" ]]; then
103+
compression=(-z)
104+
fi
105+
tar -cP "${compression[@]}" -f - "${existing[@]}" \
106+
| tee "${digest_fifo}" \
107+
| aws s3 cp --only-show-errors - "${upload}"
108+
wait "${digest_pid}"
109+
digest="$(cat "${digest_file}")"
110+
if [[ ! "${digest}" =~ ^[0-9a-f]{64}$ ]]; then
111+
echo "::error::Could not hash the stash '${STASH_KEY}'"
112+
exit 1
113+
fi
114+
object="${prefix}/sha256/${digest}.tar"
115+
aws s3 cp --only-show-errors --copy-props none "${upload}" "${object}"
116+
# Leaving the upload behind only costs storage until the bucket's lifecycle rule expires it.
117+
aws s3 rm --only-show-errors "${upload}" || echo "::warning::Could not remove ${upload}"
118+
mkdir -p "${DIGEST_DIR}"
119+
echo "${digest}" > "${DIGEST_DIR}/sha256"
120+
echo "saved=true" >> "${GITHUB_OUTPUT}"
121+
echo "Saved ${existing[*]} to ${object}"
122+
- name: "Save stash digest to GitHub artifacts"
123+
if: runner.environment == 'self-hosted' && steps.s3.outputs.saved == 'true'
124+
uses: apache/infrastructure-actions/stash/save@7d3afde57801d8c9a2db9730c28ae4e036cc2ab9 # save/v1.0.2
125+
with:
126+
key: ${{ inputs.key }}-s3
127+
path: ${{ runner.temp }}/stash-digest-save/${{ inputs.key }}
128+
if-no-files-found: error
129+
retention-days: ${{ inputs.retention-days }}

‎.github/workflows/additional-ci-image-checks.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ jobs:
6262
# layer that builds Python from source, which takes 11-18 minutes for Python 3.11+ on hosted runners.
6363
timeout-minutes: 35
6464
name: Check that image builds quickly
65-
runs-on: ${{ fromJSON(inputs.runners) }}
65+
runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
6666
env:
6767
UPGRADE_TO_NEWER_DEPENDENCIES: false
6868
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"

0 commit comments

Comments
 (0)