@@ -31,15 +31,7 @@ on: # yamllint disable-line rule:truthy
3131 description : " Compute the changes without applying them to pull requests"
3232 type : boolean
3333 default : false
34- permissions :
35- # Listing the runs and jobs of the gate tampering check, and deleting the runs that only gated a
36- # pull request to fork CI; unlisted scopes are none.
37- actions : write
38- contents : read
39- # Labels, draft state, comments and closing of gated pull requests.
40- pull-requests : write
41- # The `Fork CI / Tests (AMD)` commit status on pull request head commits.
42- statuses : write
34+ permissions : {}
4335concurrency :
4436 group : fork-ci-monitor
4537 cancel-in-progress : false
@@ -48,42 +40,196 @@ jobs:
4840 reconcile :
4941 name : " Reconcile pull requests gated to fork CI"
5042 if : github.repository == 'apache/airflow'
43+ permissions :
44+ # Listing the runs and jobs of the gate tampering check, and deleting the runs that only
45+ # gated a pull request to fork CI.
46+ actions : write
47+ contents : read
48+ # Labels, comments and closing of gated pull requests.
49+ pull-requests : write
50+ # The `Fork CI / Tests (AMD)` commit status on pull request head commits.
51+ statuses : write
52+ outputs :
53+ draft-changes : ${{ steps.reconcile.outputs.draft-changes }}
5154 # The breeze action frees disk space with sudo and docker, which ubuntu-slim does not provide.
5255 runs-on : ubuntu-latest
5356 timeout-minutes : 20
5457 steps :
58+ # What earlier runs reported, so that a problem that persists is posted to Slack once
59+ # rather than every 15 minutes. Losing it, for example to cache eviction, re-posts once.
60+ # Restored before anything that can fail, so a failing setup is reported once too, and
61+ # outside the checkout, where nothing but the steps below reads it.
62+ - name : " Restore what earlier runs reported"
63+ id : restore-reported
64+ uses : actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
65+ with :
66+ path : ${{ runner.temp }}/fork-ci-monitor
67+ key : fork-ci-monitor-reported-${{ github.run_id }}-${{ github.run_attempt }}
68+ restore-keys : fork-ci-monitor-reported-
5569 - name : " Checkout main"
5670 uses : actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
5771 with :
5872 ref : main
5973 persist-credentials : false
6074 - name : " Install Breeze"
6175 uses : ./.github/actions/breeze
76+ # The reconciler updates a working copy; it replaces the recorded one only once Slack has
77+ # the message, so a failed post is retried by the next run instead of being suppressed.
6278 - name : " Reconcile"
6379 id : reconcile
6480 env :
6581 GH_TOKEN : ${{ secrets.GITHUB_TOKEN }}
6682 DRY_RUN : ${{ inputs.dry-run && 'true' || 'false' }}
6783 RUN_URL : ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
68- run : breeze ci fork-ci-reconcile --output-file slack-message.json
84+ run : |
85+ cp "${RUNNER_TEMP}/fork-ci-monitor/reported.json" reported.json 2>/dev/null || true
86+ breeze ci fork-ci-reconcile --output-file slack-message.json --reported-file reported.json \
87+ --draft-changes-file draft-changes.json
88+ echo "draft-changes=$(jq -c . draft-changes.json)" >> "${GITHUB_OUTPUT}"
6989 # A dry run reports what it would have done, which must not read as done on Slack.
7090 - name : " Post to Slack"
91+ id : post
7192 if : steps.reconcile.outputs.has-alerts == 'true' && !inputs.dry-run
7293 uses : slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0
7394 with :
7495 method : chat.postMessage
7596 token : ${{ secrets.SLACK_BOT_TOKEN }}
97+ # Fail the step on a rejected post, so it is not recorded as reported.
98+ errors : true
7699 payload-file-path : " slack-message.json"
100+ - name : " Record what this run reported"
101+ if : steps.reconcile.outcome == 'success' && steps.post.outcome != 'failure' && !inputs.dry-run
102+ run : |
103+ mkdir -p "${RUNNER_TEMP}/fork-ci-monitor"
104+ cp reported.json "${RUNNER_TEMP}/fork-ci-monitor/reported.json"
105+ rm -f "${RUNNER_TEMP}/fork-ci-monitor/failed"
77106 # A run that fails before writing its message, for example on a discovery query or a rate
78- # limit it cannot wait out, would otherwise stay silent.
79- - name : " Report a failed run to Slack"
107+ # limit it cannot wait out, would otherwise stay silent. Only the first of consecutive failed
108+ # runs is reported.
109+ - name : " Check whether the previous run's failure was reported"
110+ id : previous-failure
80111 if : failure() && !inputs.dry-run
112+ run : |
113+ if [[ -f "${RUNNER_TEMP}/fork-ci-monitor/failed" ]]; then
114+ echo "reported=true" >> "${GITHUB_OUTPUT}"
115+ else
116+ echo "reported=false" >> "${GITHUB_OUTPUT}"
117+ fi
118+ - name : " Report a failed run to Slack"
119+ id : report-failure
120+ if : failure() && !inputs.dry-run && steps.previous-failure.outputs.reported == 'false'
81121 uses : slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0
82122 with :
83123 method : chat.postMessage
84124 token : ${{ secrets.SLACK_BOT_TOKEN }}
125+ errors : true
85126 payload : |
86127 channel: "internal-airflow-ci-cd"
87128 text: >-
88129 Fork CI monitor run failed:
89130 ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
131+ - name : " Record the reported failure"
132+ if : failure() && steps.report-failure.outcome == 'success'
133+ run : mkdir -p "${RUNNER_TEMP}/fork-ci-monitor" && touch "${RUNNER_TEMP}/fork-ci-monitor/failed"
134+ # A dry run posts nothing, so what it saw must not suppress the next run's message.
135+ - name : " Save what was reported"
136+ if : always() && !inputs.dry-run && steps.restore-reported.outcome == 'success'
137+ uses : actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
138+ with :
139+ path : ${{ runner.temp }}/fork-ci-monitor
140+ key : fork-ci-monitor-reported-${{ github.run_id }}-${{ github.run_attempt }}
141+
142+ # GitHub authorises the convertPullRequestToDraft and markPullRequestReadyForReview mutations
143+ # against `contents`, not `pull-requests`: with `contents: read` they fail with "Resource not
144+ # accessible by integration". `contents: write` would also let the token push to the
145+ # repository, so it is kept out of the reconcile job, which installs and runs Breeze and its
146+ # dependencies. This job runs no code but `gh` and `jq` on the reconciler's list.
147+ draft-state :
148+ name : " Change the draft state of pull requests gated to fork CI"
149+ needs : reconcile
150+ # Runs whenever the reconciler wrote its list, even when only its Slack post failed, and with
151+ # an empty list too, so that a failure set that cleared is recorded as cleared.
152+ if : >-
153+ !cancelled() && !inputs.dry-run && github.repository == 'apache/airflow'
154+ && needs.reconcile.outputs.draft-changes != ''
155+ permissions :
156+ contents : write
157+ # The draft state, and removing `fork ci required` from a released pull request once it
158+ # is undrafted.
159+ pull-requests : write
160+ runs-on : ubuntu-latest
161+ timeout-minutes : 10
162+ steps :
163+ # The pull requests whose change failed last time, so a persisting failure is posted to
164+ # Slack once, as in the reconcile job.
165+ - name : " Restore the previously failed changes"
166+ id : restore-failed
167+ uses : actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
168+ with :
169+ path : ${{ runner.temp }}/fork-ci-draft-state
170+ key : fork-ci-monitor-draft-state-${{ github.run_id }}-${{ github.run_attempt }}
171+ restore-keys : fork-ci-monitor-draft-state-
172+ - name : " Change the draft state"
173+ id : change
174+ env :
175+ GH_TOKEN : ${{ secrets.GITHUB_TOKEN }}
176+ REPO : ${{ github.repository }}
177+ DRAFT_CHANGES : ${{ needs.reconcile.outputs.draft-changes }}
178+ run : |
179+ failed=()
180+ while IFS=$'\t' read -r number node_id draft release; do
181+ # This token can push to the repository; act only on well-formed entries.
182+ if [[ ! "${number}" =~ ^[0-9]+$ || ! "${node_id}" =~ ^PR_[A-Za-z0-9_-]+$ \
183+ || ! "${draft}" =~ ^(true|false)$ || ! "${release}" =~ ^(true|false)$ ]]; then
184+ echo "::error::Skipped a malformed draft change"
185+ failed+=("malformed")
186+ continue
187+ fi
188+ if [[ "${draft}" == "true" ]]; then
189+ mutation="convertPullRequestToDraft"
190+ else
191+ mutation="markPullRequestReadyForReview"
192+ fi
193+ echo "#${number}: ${mutation}"
194+ if ! gh api graphql -f id="${node_id}" -f query="mutation(\$id: ID!) {
195+ ${mutation}(input: {pullRequestId: \$id}) { clientMutationId } }" > /dev/null; then
196+ failed+=("${number}")
197+ continue
198+ fi
199+ # The label brings the pull request back to the reconciler, which retries the
200+ # undraft while it is there.
201+ label="repos/${REPO}/issues/${number}/labels/fork%20ci%20required"
202+ if [[ "${release}" == "true" ]] && ! gh api -X DELETE "${label}" > /dev/null; then
203+ failed+=("${number}")
204+ fi
205+ done < <(jq -r '.[] | [.number, .node_id, .draft, .release] | @tsv' <<< "${DRAFT_CHANGES}")
206+ mkdir -p "${RUNNER_TEMP}/fork-ci-draft-state"
207+ printf '%s\n' "${failed[@]}" | sort -n | sed '/^$/d' > failed.txt
208+ previous="${RUNNER_TEMP}/fork-ci-draft-state/failed.txt"
209+ if [[ -s failed.txt ]] && ! cmp -s failed.txt "${previous}"; then
210+ echo "new-failures=true" >> "${GITHUB_OUTPUT}"
211+ fi
212+ echo "failed=$(paste -sd, failed.txt)" >> "${GITHUB_OUTPUT}"
213+ - name : " Post failed changes to Slack"
214+ id : post
215+ if : steps.change.outputs.new-failures == 'true'
216+ uses : slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0
217+ with :
218+ method : chat.postMessage
219+ token : ${{ secrets.SLACK_BOT_TOKEN }}
220+ errors : true
221+ payload : |
222+ channel: "internal-airflow-ci-cd"
223+ text: >-
224+ Fork CI monitor could not change the draft state of pull requests
225+ ${{ steps.change.outputs.failed }}:
226+ ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
227+ - name : " Record the failed changes"
228+ if : steps.change.outcome == 'success' && steps.post.outcome != 'failure'
229+ run : cp failed.txt "${RUNNER_TEMP}/fork-ci-draft-state/failed.txt"
230+ - name : " Save the failed changes"
231+ if : always() && steps.restore-failed.outcome == 'success'
232+ uses : actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
233+ with :
234+ path : ${{ runner.temp }}/fork-ci-draft-state
235+ key : fork-ci-monitor-draft-state-${{ github.run_id }}-${{ github.run_attempt }}
0 commit comments