Skip to content

Commit 5b35171

Browse files
borsbors
committed
Auto merge of #124175 - Kobzol:ci-dynamic-job, r=<try>
CI: dynamic jobs Opening as a draft to experiment with [dynamic CI jobs](https://rust-lang.zulipchat.com/#narrow/stream/242791-t-infra/topic/job.20matrix.20re-ordered.20PR.20list). r? `@ghost`
2 parents ce3263e + 8b945c3 commit 5b35171

16 files changed

+130
-215
lines changed

‎.github/workflows/ci.yml

+9-148
Original file line numberDiff line numberDiff line change
@@ -45,23 +45,23 @@ jobs:
4545
- name: Checkout the source code
4646
uses: actions/checkout@v4
4747
- name: Calculate the CI job matrix
48-
run: python3 src/ci/scripts/calculate-job-matrix.py >> $GITHUB_OUTPUT
48+
env:
49+
GITHUB_CTX: "${{ toJSON(github) }}"
50+
run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT
4951
id: jobs
50-
pr:
51-
name: "PR - ${{ matrix.name }}"
52+
job:
53+
name: "${{ matrix.name }}"
5254
needs:
5355
- calculate_matrix
5456
env:
55-
PR_CI_JOB: 1
56-
CI_JOB_NAME: "${{ matrix.name }}"
57+
CI_JOB_NAME: "${{ matrix.image }}"
5758
CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
5859
HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}"
5960
DOCKER_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
6061
SCCACHE_BUCKET: rust-lang-ci-sccache2
6162
TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate"
6263
CACHE_DOMAIN: ci-caches.rust-lang.org
63-
if: "github.event_name == 'pull_request'"
64-
continue-on-error: "${{ matrix.name == 'mingw-check-tidy' }}"
64+
continue-on-error: "${{ matrix.continue_on_error || false }}"
6565
strategy:
6666
matrix:
6767
include: "${{ fromJSON(needs.calculate_matrix.outputs.jobs) }}"
@@ -589,145 +589,6 @@ jobs:
589589
AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}"
590590
AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}"
591591
if: "success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')"
592-
try:
593-
name: "try - ${{ matrix.name }}"
594-
env:
595-
DIST_TRY_BUILD: 1
596-
CI_JOB_NAME: "${{ matrix.name }}"
597-
CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
598-
HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}"
599-
DOCKER_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
600-
SCCACHE_BUCKET: rust-lang-ci-sccache2
601-
DEPLOY_BUCKET: rust-lang-ci2
602-
TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate"
603-
TOOLSTATE_ISSUES_API_URL: "https://api.github.com/repos/rust-lang/rust/issues"
604-
TOOLSTATE_PUBLISH: 1
605-
CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZI5DHEBFL
606-
ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZN24CBO55
607-
AWS_REGION: us-west-1
608-
CACHE_DOMAIN: ci-caches.rust-lang.org
609-
if: "github.event_name == 'push' && (((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust') || ((github.ref == 'refs/heads/automation/bors/try') && github.repository == 'rust-lang/rust'))"
610-
strategy:
611-
matrix:
612-
include:
613-
- name: dist-x86_64-linux
614-
env:
615-
CODEGEN_BACKENDS: "llvm,cranelift"
616-
os: ubuntu-20.04-16core-64gb
617-
defaults:
618-
run:
619-
shell: "${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }}"
620-
timeout-minutes: 600
621-
runs-on: "${{ matrix.os }}"
622-
steps:
623-
- if: "contains(matrix.os, 'windows')"
624-
uses: msys2/[email protected]
625-
with:
626-
msystem: "${{ contains(matrix.name, 'i686') && 'mingw32' || 'mingw64' }}"
627-
update: false
628-
release: true
629-
path-type: inherit
630-
install: "make dos2unix diffutils\n"
631-
- name: disable git crlf conversion
632-
run: git config --global core.autocrlf false
633-
- name: checkout the source code
634-
uses: actions/checkout@v4
635-
with:
636-
fetch-depth: 2
637-
- name: configure the PR in which the error message will be posted
638-
run: "echo \"[CI_PR_NUMBER=$num]\""
639-
env:
640-
num: "${{ github.event.number }}"
641-
if: "success() && !env.SKIP_JOB && github.event_name == 'pull_request'"
642-
- name: add extra environment variables
643-
run: src/ci/scripts/setup-environment.sh
644-
env:
645-
EXTRA_VARIABLES: "${{ toJson(matrix.env) }}"
646-
if: success() && !env.SKIP_JOB
647-
- name: decide whether to skip this job
648-
run: src/ci/scripts/should-skip-this.sh
649-
if: success() && !env.SKIP_JOB
650-
- name: ensure the channel matches the target branch
651-
run: src/ci/scripts/verify-channel.sh
652-
if: success() && !env.SKIP_JOB
653-
- name: collect CPU statistics
654-
run: src/ci/scripts/collect-cpu-stats.sh
655-
if: success() && !env.SKIP_JOB
656-
- name: show the current environment
657-
run: src/ci/scripts/dump-environment.sh
658-
if: success() && !env.SKIP_JOB
659-
- name: install awscli
660-
run: src/ci/scripts/install-awscli.sh
661-
if: success() && !env.SKIP_JOB
662-
- name: install sccache
663-
run: src/ci/scripts/install-sccache.sh
664-
if: success() && !env.SKIP_JOB
665-
- name: select Xcode
666-
run: src/ci/scripts/select-xcode.sh
667-
if: success() && !env.SKIP_JOB
668-
- name: install clang
669-
run: src/ci/scripts/install-clang.sh
670-
if: success() && !env.SKIP_JOB
671-
- name: install tidy
672-
run: src/ci/scripts/install-tidy.sh
673-
if: success() && !env.SKIP_JOB
674-
- name: install WIX
675-
run: src/ci/scripts/install-wix.sh
676-
if: success() && !env.SKIP_JOB
677-
- name: disable git crlf conversion
678-
run: src/ci/scripts/disable-git-crlf-conversion.sh
679-
if: success() && !env.SKIP_JOB
680-
- name: checkout submodules
681-
run: src/ci/scripts/checkout-submodules.sh
682-
if: success() && !env.SKIP_JOB
683-
- name: install MSYS2
684-
run: src/ci/scripts/install-msys2.sh
685-
if: success() && !env.SKIP_JOB
686-
- name: install MinGW
687-
run: src/ci/scripts/install-mingw.sh
688-
if: success() && !env.SKIP_JOB
689-
- name: install ninja
690-
run: src/ci/scripts/install-ninja.sh
691-
if: success() && !env.SKIP_JOB
692-
- name: enable ipv6 on Docker
693-
run: src/ci/scripts/enable-docker-ipv6.sh
694-
if: success() && !env.SKIP_JOB
695-
- name: disable git crlf conversion
696-
run: src/ci/scripts/disable-git-crlf-conversion.sh
697-
if: success() && !env.SKIP_JOB
698-
- name: ensure line endings are correct
699-
run: src/ci/scripts/verify-line-endings.sh
700-
if: success() && !env.SKIP_JOB
701-
- name: ensure backported commits are in upstream branches
702-
run: src/ci/scripts/verify-backported-commits.sh
703-
if: success() && !env.SKIP_JOB
704-
- name: ensure the stable version number is correct
705-
run: src/ci/scripts/verify-stable-version-number.sh
706-
if: success() && !env.SKIP_JOB
707-
- name: run the build
708-
run: src/ci/scripts/run-build-from-ci.sh 2>&1
709-
env:
710-
AWS_ACCESS_KEY_ID: "${{ env.CACHES_AWS_ACCESS_KEY_ID }}"
711-
AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}"
712-
TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}"
713-
if: success() && !env.SKIP_JOB
714-
- name: create github artifacts
715-
run: src/ci/scripts/create-doc-artifacts.sh
716-
if: success() && !env.SKIP_JOB
717-
- name: upload artifacts to github
718-
uses: actions/upload-artifact@v4
719-
with:
720-
name: "${{ env.DOC_ARTIFACT_NAME }}"
721-
path: obj/artifacts/doc
722-
if-no-files-found: ignore
723-
retention-days: 5
724-
if: success() && !env.SKIP_JOB
725-
- name: upload artifacts to S3
726-
run: src/ci/scripts/upload-artifacts.sh
727-
env:
728-
AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}"
729-
AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}"
730-
if: "success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')"
731592
master:
732593
name: master
733594
runs-on: ubuntu-latest
@@ -755,7 +616,7 @@ jobs:
755616
if: success() && !env.SKIP_JOB
756617
try-success:
757618
needs:
758-
- try
619+
- job
759620
if: "success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'"
760621
steps:
761622
- name: mark the job as a success
@@ -765,7 +626,7 @@ jobs:
765626
runs-on: ubuntu-latest
766627
try-failure:
767628
needs:
768-
- try
629+
- job
769630
if: "!success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'"
770631
steps:
771632
- name: mark the job as a failure

‎library/backtrace

‎library/stdarch

‎src/ci/github-actions/calculate-job-matrix.py

+51
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
#!/usr/bin/env python3
2+
3+
"""
4+
This script serves for generating a matrix of jobs that should
5+
be executed on CI.
6+
7+
It reads job definitions from `src/ci/github-actions/jobs.yml`
8+
and filters them based on the event that happened on CI.
9+
10+
Currently, it only supports PR builds.
11+
"""
12+
13+
import json
14+
import os
15+
from pathlib import Path
16+
from typing import List, Dict
17+
18+
import yaml
19+
20+
JOBS_YAML_PATH = Path(__file__).absolute().parent / "jobs.yml"
21+
22+
23+
def name_jobs(jobs: List[Dict], prefix: str) -> List[Dict]:
24+
for job in jobs:
25+
job["name"] = f"{prefix} - {job['image']}"
26+
return jobs
27+
28+
29+
if __name__ == "__main__":
30+
github_ctx = json.loads(os.environ["GITHUB_CTX"])
31+
32+
with open(JOBS_YAML_PATH) as f:
33+
data = yaml.safe_load(f)
34+
35+
event_name = github_ctx["event_name"]
36+
ref = github_ctx["ref"]
37+
repository = github_ctx["repository"]
38+
39+
old_bors_try_build = ref in ("refs/heads/try", "refs/heads/try-perf") and repository == "rust-lang-ci/rust"
40+
new_bors_try_build = ref == "refs/heads/automation/bors/try" and repository == "rust-lang/rust"
41+
try_build = old_bors_try_build or new_bors_try_build
42+
43+
jobs = []
44+
# Pull request CI jobs. Their name is 'PR - <image>'
45+
if event_name == "pull_request":
46+
jobs = name_jobs(data["pr"], "PR")
47+
# Try builds
48+
elif event_name == "push" and try_build:
49+
jobs = name_jobs(data["try"], "try")
50+
51+
print(f"jobs={json.dumps(jobs)}")

‎src/ci/github-actions/ci.yml

+16-26
Original file line numberDiff line numberDiff line change
@@ -351,17 +351,24 @@ jobs:
351351
- name: Checkout the source code
352352
uses: actions/checkout@v4
353353
- name: Calculate the CI job matrix
354-
run: python3 src/ci/scripts/calculate-job-matrix.py >> $GITHUB_OUTPUT
354+
env:
355+
GITHUB_CTX: ${{ toJSON(github) }}
356+
run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT
355357
id: jobs
356-
pr:
358+
job:
357359
<<: *base-ci-job
358-
name: PR - ${{ matrix.name }}
360+
name: ${{ matrix.name }}
359361
needs: [ calculate_matrix ]
360362
env:
361-
<<: [*shared-ci-variables, *public-variables]
362-
PR_CI_JOB: 1
363-
if: github.event_name == 'pull_request'
364-
continue-on-error: ${{ matrix.name == 'mingw-check-tidy' }}
363+
CI_JOB_NAME: ${{ matrix.image }}
364+
CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
365+
# commit of PR sha or commit sha. `GITHUB_SHA` is not accurate for PRs.
366+
HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
367+
DOCKER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
368+
SCCACHE_BUCKET: rust-lang-ci-sccache2
369+
TOOLSTATE_REPO: https://github.com/rust-lang-nursery/rust-toolstate
370+
CACHE_DOMAIN: ci-caches.rust-lang.org
371+
continue-on-error: ${{ matrix.continue_on_error || false }}
365372
strategy:
366373
matrix:
367374
# Check the `calculate_matrix` job to see how is the matrix defined.
@@ -751,23 +758,6 @@ jobs:
751758
SCRIPT: python x.py dist bootstrap --include-default-paths
752759
<<: *job-windows-8c
753760

754-
try:
755-
<<: *base-ci-job
756-
name: try - ${{ matrix.name }}
757-
env:
758-
DIST_TRY_BUILD: 1
759-
<<: [*shared-ci-variables, *prod-variables]
760-
if: github.event_name == 'push' && (((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust') || ((github.ref == 'refs/heads/automation/bors/try') && github.repository == 'rust-lang/rust'))
761-
strategy:
762-
matrix:
763-
include:
764-
- &dist-x86_64-linux
765-
name: dist-x86_64-linux
766-
env:
767-
CODEGEN_BACKENDS: llvm,cranelift
768-
<<: *job-linux-16c
769-
770-
771761
master:
772762
name: master
773763
runs-on: ubuntu-latest
@@ -791,11 +781,11 @@ jobs:
791781
# build completed, as there is no practical way to detect when a workflow is
792782
# successful listening to webhooks only.
793783
try-success:
794-
needs: [try]
784+
needs: [ job ]
795785
if: "success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'"
796786
<<: *base-success-job
797787
try-failure:
798-
needs: [try]
788+
needs: [ job ]
799789
if: "!success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'"
800790
<<: *base-failure-job
801791
auto-success:

‎src/ci/github-actions/jobs.yml

+43-5
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
# dynamically in CI from ci.yml.
33
# You *do not* need to re-run `src/tools/expand-yaml-anchors` when you
44
# modify this file.
5-
shared_defs:
5+
runners:
66
- &base-job
77
env: { }
88

@@ -37,14 +37,52 @@ shared_defs:
3737
- &job-aarch64-linux
3838
os: [ self-hosted, ARM64, linux ]
3939

40+
envs:
41+
- &pr-env
42+
PR_CI_JOB: 1
43+
44+
- &prod-variables
45+
SCCACHE_BUCKET: rust-lang-ci-sccache2
46+
DEPLOY_BUCKET: rust-lang-ci2
47+
TOOLSTATE_REPO: https://github.com/rust-lang-nursery/rust-toolstate
48+
TOOLSTATE_ISSUES_API_URL: https://api.github.com/repos/rust-lang/rust/issues
49+
TOOLSTATE_PUBLISH: 1
50+
# AWS_SECRET_ACCESS_KEYs are stored in GitHub's secrets storage, named
51+
# AWS_SECRET_ACCESS_KEY_<keyid>. Including the key id in the name allows to
52+
# rotate them in a single branch while keeping the old key in another
53+
# branch, which wouldn't be possible if the key was named with the kind
54+
# (caches, artifacts...).
55+
CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZI5DHEBFL
56+
ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZN24CBO55
57+
AWS_REGION: us-west-1
58+
CACHE_DOMAIN: ci-caches.rust-lang.org
59+
60+
# Pull request jobs
4061
pr:
41-
- name: mingw-check
62+
- image: mingw-check
63+
env:
64+
<<: *pr-env
4265
<<: *job-linux-4c
43-
- name: mingw-check-tidy
66+
- image: mingw-check-tidy
67+
continue_on_error: true
68+
env:
69+
<<: *pr-env
4470
<<: *job-linux-4c
45-
- name: x86_64-gnu-llvm-17
71+
- image: x86_64-gnu-llvm-17
4672
env:
4773
ENABLE_GCC_CODEGEN: "1"
74+
<<: *pr-env
4875
<<: *job-linux-16c
49-
- name: x86_64-gnu-tools
76+
- image: x86_64-gnu-tools
77+
env:
78+
<<: *pr-env
79+
<<: *job-linux-16c
80+
81+
# Try build jobs
82+
try:
83+
- image: dist-x86_64-linux
84+
env:
85+
DIST_TRY_BUILD: 1
86+
CODEGEN_BACKENDS: llvm,cranelift
87+
<<: *prod-variables
5088
<<: *job-linux-16c

0 commit comments

Comments
 (0)