gha: add guardrails timeouts on all jobs

thaJeztah · thaJeztah · commit 6b7e2783d1e6 · 2024-10-10T13:43:59.000+02:00
We had a few "runaway jobs" recently, where the job got stuck, and kept
running for 6 hours (in one case even 24 hours, probably due some github
outage). Some of those jobs could not be terminated.

While running these actions on public repositories doesn't cost us, it's
still not desirable to have jobs running for that long (as they can still
hold up the queue).

This patch adds a blanket "2 hours" time-limit to all jobs that didn't
have a limit set. We should look at tweaking those limits to actually
expected duration, but having a default at least is a start.

Also changed the position of some existing timeouts so that we have a
consistent order in which it's set; making it easier to spot locations
where no limit is defined.

Signed-off-by: Sebastiaan van Stijn &lt;github@gone.nl&gt;
diff --git a/.github/workflows/.dco.yml b/.github/workflows/.dco.yml
@@ -21,6 +21,7 @@ env:
 jobs:
   run:
     runs-on: ubuntu-20.04
+    timeout-minutes: 120 # guardrails timeout for the whole job
     steps:
       -
         name: Checkout
diff --git a/.github/workflows/.test-prepare.yml b/.github/workflows/.test-prepare.yml
@@ -22,6 +22,7 @@ on:
 jobs:
   run:
     runs-on: ubuntu-20.04
+    timeout-minutes: 120 # guardrails timeout for the whole job
     outputs:
       matrix: ${{ steps.set.outputs.matrix }}
     steps:
diff --git a/.github/workflows/.test.yml b/.github/workflows/.test.yml
@@ -34,8 +34,8 @@ env:
 jobs:
   unit:
     runs-on: ubuntu-20.04
+    timeout-minutes: 120 # guardrails timeout for the whole job
     continue-on-error: ${{ github.event_name != 'pull_request' }}
-    timeout-minutes: 120
     steps:
       -
         name: Checkout
@@ -89,8 +89,8 @@ jobs:
 
   unit-report:
     runs-on: ubuntu-20.04
-    continue-on-error: ${{ github.event_name != 'pull_request' }}
     timeout-minutes: 10
+    continue-on-error: ${{ github.event_name != 'pull_request' }}
     if: always()
     needs:
       - unit
@@ -117,8 +117,8 @@ jobs:
 
   docker-py:
     runs-on: ubuntu-20.04
+    timeout-minutes: 120 # guardrails timeout for the whole job
     continue-on-error: ${{ github.event_name != 'pull_request' }}
-    timeout-minutes: 120
     steps:
       -
         name: Checkout
@@ -174,8 +174,8 @@ jobs:
 
   integration-flaky:
     runs-on: ubuntu-20.04
+    timeout-minutes: 120 # guardrails timeout for the whole job
     continue-on-error: ${{ github.event_name != 'pull_request' }}
-    timeout-minutes: 120
     steps:
       -
         name: Checkout
@@ -206,8 +206,8 @@ jobs:
 
   integration:
     runs-on: ${{ matrix.os }}
+    timeout-minutes: 120 # guardrails timeout for the whole job
     continue-on-error: ${{ github.event_name != 'pull_request' }}
-    timeout-minutes: 120
     strategy:
       fail-fast: false
       matrix:
@@ -304,8 +304,8 @@ jobs:
 
   integration-report:
     runs-on: ubuntu-20.04
-    continue-on-error: ${{ github.event_name != 'pull_request' }}
     timeout-minutes: 10
+    continue-on-error: ${{ github.event_name != 'pull_request' }}
     if: always()
     needs:
       - integration
@@ -333,6 +333,7 @@ jobs:
 
   integration-cli-prepare:
     runs-on: ubuntu-20.04
+    timeout-minutes: 120 # guardrails timeout for the whole job
     continue-on-error: ${{ github.event_name != 'pull_request' }}
     outputs:
       matrix: ${{ steps.tests.outputs.matrix }}
@@ -368,8 +369,8 @@ jobs:
 
   integration-cli:
     runs-on: ubuntu-20.04
+    timeout-minutes: 120 # guardrails timeout for the whole job
     continue-on-error: ${{ github.event_name != 'pull_request' }}
-    timeout-minutes: 120
     needs:
       - integration-cli-prepare
     strategy:
@@ -448,8 +449,8 @@ jobs:
 
   integration-cli-report:
     runs-on: ubuntu-20.04
-    continue-on-error: ${{ github.event_name != 'pull_request' }}
     timeout-minutes: 10
+    continue-on-error: ${{ github.event_name != 'pull_request' }}
     if: always()
     needs:
       - integration-cli
diff --git a/.github/workflows/.windows.yml b/.github/workflows/.windows.yml
@@ -12,6 +12,7 @@ name: .windows
 permissions:
   contents: read
 
+
 on:
   workflow_call:
     inputs:
@@ -42,6 +43,7 @@ env:
 jobs:
   build:
     runs-on: ${{ inputs.os }}
+    timeout-minutes: 120 # guardrails timeout for the whole job
     env:
       GOPATH: ${{ github.workspace }}\go
       GOBIN: ${{ github.workspace }}\go\bin
@@ -121,7 +123,7 @@ jobs:
 
   unit-test:
     runs-on: ${{ inputs.os }}
-    timeout-minutes: 120
+    timeout-minutes: 120 # guardrails timeout for the whole job
     env:
       GOPATH: ${{ github.workspace }}\go
       GOBIN: ${{ github.workspace }}\go\bin
@@ -203,6 +205,7 @@ jobs:
 
   unit-test-report:
     runs-on: ubuntu-latest
+    timeout-minutes: 120 # guardrails timeout for the whole job
     if: always()
     needs:
       - unit-test
@@ -229,6 +232,7 @@ jobs:
 
   integration-test-prepare:
     runs-on: ubuntu-latest
+    timeout-minutes: 120 # guardrails timeout for the whole job
     outputs:
       matrix: ${{ steps.tests.outputs.matrix }}
     steps:
@@ -262,8 +266,8 @@ jobs:
 
   integration-test:
     runs-on: ${{ inputs.os }}
+    timeout-minutes: 120 # guardrails timeout for the whole job
     continue-on-error: ${{ inputs.storage == 'snapshotter' && github.event_name != 'pull_request' }}
-    timeout-minutes: 120
     needs:
       - build
       - integration-test-prepare
@@ -522,6 +526,7 @@ jobs:
 
   integration-test-report:
     runs-on: ubuntu-latest
+    timeout-minutes: 120 # guardrails timeout for the whole job
     continue-on-error: ${{ inputs.storage == 'snapshotter' && github.event_name != 'pull_request' }}
     if: always()
     needs:
diff --git a/.github/workflows/bin-image.yml b/.github/workflows/bin-image.yml
@@ -41,6 +41,7 @@ jobs:
 
   prepare:
     runs-on: ubuntu-20.04
+    timeout-minutes: 120 # guardrails timeout for the whole job
     outputs:
       platforms: ${{ steps.platforms.outputs.matrix }}
     steps:
@@ -93,6 +94,7 @@ jobs:
 
   build:
     runs-on: ubuntu-20.04
+    timeout-minutes: 120 # guardrails timeout for the whole job
     needs:
       - validate-dco
       - prepare
@@ -167,6 +169,7 @@ jobs:
 
   merge:
     runs-on: ubuntu-20.04
+    timeout-minutes: 120 # guardrails timeout for the whole job
     needs:
       - build
     if: always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') && github.event_name != 'pull_request' && github.repository == 'moby/moby'
diff --git a/.github/workflows/buildkit.yml b/.github/workflows/buildkit.yml
@@ -33,6 +33,7 @@ jobs:
 
   build:
     runs-on: ubuntu-20.04
+    timeout-minutes: 120 # guardrails timeout for the whole job
     needs:
       - validate-dco
     steps:
@@ -62,7 +63,7 @@ jobs:
 
   test:
     runs-on: ubuntu-20.04
-    timeout-minutes: 120
+    timeout-minutes: 120 # guardrails timeout for the whole job
     needs:
       - build
     env:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -32,6 +32,7 @@ jobs:
 
   build:
     runs-on: ubuntu-20.04
+    timeout-minutes: 120 # guardrails timeout for the whole job
     needs:
       - validate-dco
     strategy:
@@ -69,6 +70,7 @@ jobs:
 
   prepare-cross:
     runs-on: ubuntu-latest
+    timeout-minutes: 120 # guardrails timeout for the whole job
     needs:
       - validate-dco
     outputs:
@@ -90,6 +92,7 @@ jobs:
 
   cross:
     runs-on: ubuntu-20.04
+    timeout-minutes: 120 # guardrails timeout for the whole job
     needs:
       - validate-dco
       - prepare-cross
@@ -133,6 +136,7 @@ jobs:
 
   govulncheck:
     runs-on: ubuntu-24.04
+    timeout-minutes: 120 # guardrails timeout for the whole job
     permissions:
       # required to write sarif report
       security-events: write
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -34,6 +34,7 @@ jobs:
 
   build-dev:
     runs-on: ubuntu-20.04
+    timeout-minutes: 120 # guardrails timeout for the whole job
     needs:
       - validate-dco
     strategy:
@@ -86,6 +87,7 @@ jobs:
 
   validate-prepare:
     runs-on: ubuntu-20.04
+    timeout-minutes: 120 # guardrails timeout for the whole job
     needs:
       - validate-dco
     outputs:
@@ -107,7 +109,7 @@ jobs:
 
   validate:
     runs-on: ubuntu-20.04
-    timeout-minutes: 120
+    timeout-minutes: 120 # guardrails timeout for the whole job
     needs:
       - validate-prepare
       - build-dev
@@ -145,6 +147,7 @@ jobs:
 
   smoke-prepare:
     runs-on: ubuntu-20.04
+    timeout-minutes: 120 # guardrails timeout for the whole job
     needs:
       - validate-dco
     outputs:
@@ -166,6 +169,7 @@ jobs:
 
   smoke:
     runs-on: ubuntu-20.04
+    timeout-minutes: 120 # guardrails timeout for the whole job
     needs:
       - smoke-prepare
     strategy:
diff --git a/.github/workflows/validate-pr.yml b/.github/workflows/validate-pr.yml
@@ -16,6 +16,7 @@ on:
 jobs:
   check-area-label:
     runs-on: ubuntu-20.04
+    timeout-minutes: 120 # guardrails timeout for the whole job
     steps:
       - name: Missing `area/` label
         if: contains(join(github.event.pull_request.labels.*.name, ','), 'impact/') && !contains(join(github.event.pull_request.labels.*.name, ','), 'area/')
@@ -28,6 +29,7 @@ jobs:
   check-changelog:
     if: contains(join(github.event.pull_request.labels.*.name, ','), 'impact/')
     runs-on: ubuntu-20.04
+    timeout-minutes: 120 # guardrails timeout for the whole job
     env:
       PR_BODY: |
         ${{ github.event.pull_request.body }}
@@ -56,6 +58,7 @@ jobs:
 
   check-pr-branch:
     runs-on: ubuntu-20.04
+    timeout-minutes: 120 # guardrails timeout for the whole job
     env:
       PR_TITLE: ${{ github.event.pull_request.title }}
     steps:

Original file line number	Diff line number	Diff line change
`@@ -21,6 +21,7 @@ env:`
`21`	`21`	`jobs:`
`22`	`22`	`run:`
`23`	`23`	`runs-on: ubuntu-20.04`
	`24`	`+ timeout-minutes: 120 # guardrails timeout for the whole job`
`24`	`25`	`steps:`
`25`	`26`	`-`
`26`	`27`	`name: Checkout`