Skip to content

Commit 8363d65

Browse files
steipetesteipete
committed
ci: retry transient checkout fetch timeouts
1 parent 6fab00a commit 8363d65

2 files changed

Lines changed: 54 additions & 12 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 51 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -88,10 +88,22 @@ jobs:
8888
git -C "$GITHUB_WORKSPACE" remote add origin "https://github.com/${CHECKOUT_REPO}.git"
8989
fetch_checkout_ref() {
9090
local ref="$1"
91-
timeout --signal=TERM --kill-after=10s 30s git -C "$GITHUB_WORKSPACE" \
92-
-c protocol.version=2 \
93-
fetch --no-tags --prune --no-recurse-submodules --depth=1 origin \
94-
"+${ref}:refs/remotes/origin/checkout"
91+
local fetch_status
92+
for attempt in 1 2 3; do
93+
timeout --signal=TERM --kill-after=10s 30s git -C "$GITHUB_WORKSPACE" \
94+
-c protocol.version=2 \
95+
fetch --no-tags --prune --no-recurse-submodules --depth=1 origin \
96+
"+${ref}:refs/remotes/origin/checkout" && return 0
97+
fetch_status="$?"
98+
if [ "$fetch_status" != "124" ] && [ "$fetch_status" != "137" ]; then
99+
return "$fetch_status"
100+
fi
101+
if [ "$attempt" = "3" ]; then
102+
return "$fetch_status"
103+
fi
104+
echo "::warning::checkout fetch for '$ref' timed out on attempt $attempt; retrying"
105+
sleep 5
106+
done
95107
}
96108
if fetch_checkout_ref "$CHECKOUT_REF"; then
97109
:
@@ -337,10 +349,22 @@ jobs:
337349
git -C "$GITHUB_WORKSPACE" remote add origin "https://github.com/${CHECKOUT_REPO}.git"
338350
fetch_checkout_ref() {
339351
local ref="$1"
340-
timeout --signal=TERM --kill-after=10s 30s git -C "$GITHUB_WORKSPACE" \
341-
-c protocol.version=2 \
342-
fetch --no-tags --prune --no-recurse-submodules --depth=1 origin \
343-
"+${ref}:refs/remotes/origin/checkout"
352+
local fetch_status
353+
for attempt in 1 2 3; do
354+
timeout --signal=TERM --kill-after=10s 30s git -C "$GITHUB_WORKSPACE" \
355+
-c protocol.version=2 \
356+
fetch --no-tags --prune --no-recurse-submodules --depth=1 origin \
357+
"+${ref}:refs/remotes/origin/checkout" && return 0
358+
fetch_status="$?"
359+
if [ "$fetch_status" != "124" ] && [ "$fetch_status" != "137" ]; then
360+
return "$fetch_status"
361+
fi
362+
if [ "$attempt" = "3" ]; then
363+
return "$fetch_status"
364+
fi
365+
echo "::warning::checkout fetch for '$ref' timed out on attempt $attempt; retrying"
366+
sleep 5
367+
done
344368
}
345369
if fetch_checkout_ref "$CHECKOUT_REF"; then
346370
:
@@ -1583,10 +1607,25 @@ jobs:
15831607
git init "$GITHUB_WORKSPACE"
15841608
git -C "$GITHUB_WORKSPACE" config gc.auto 0
15851609
git -C "$GITHUB_WORKSPACE" remote add origin "https://github.com/${CHECKOUT_REPO}.git"
1586-
timeout --signal=TERM --kill-after=10s 30s git -C "$GITHUB_WORKSPACE" \
1587-
-c protocol.version=2 \
1588-
fetch --no-tags --prune --no-recurse-submodules --depth=1 origin \
1589-
"+${CHECKOUT_SHA}:refs/remotes/origin/checkout"
1610+
fetch_checkout_ref() {
1611+
local fetch_status
1612+
for attempt in 1 2 3; do
1613+
timeout --signal=TERM --kill-after=10s 30s git -C "$GITHUB_WORKSPACE" \
1614+
-c protocol.version=2 \
1615+
fetch --no-tags --prune --no-recurse-submodules --depth=1 origin \
1616+
"+${CHECKOUT_SHA}:refs/remotes/origin/checkout" && return 0
1617+
fetch_status="$?"
1618+
if [ "$fetch_status" != "124" ] && [ "$fetch_status" != "137" ]; then
1619+
return "$fetch_status"
1620+
fi
1621+
if [ "$attempt" = "3" ]; then
1622+
return "$fetch_status"
1623+
fi
1624+
echo "::warning::checkout fetch for '$CHECKOUT_SHA' timed out on attempt $attempt; retrying"
1625+
sleep 5
1626+
done
1627+
}
1628+
fetch_checkout_ref
15901629
git -C "$GITHUB_WORKSPACE" checkout --detach refs/remotes/origin/checkout
15911630
15921631
- name: Setup Python

‎test/scripts/ci-workflow-guards.test.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,9 @@ describe("ci workflow guards", () => {
5050
expect(checkoutStep.run, jobName).toContain(
5151
'timeout --signal=TERM --kill-after=10s 30s git -C "$GITHUB_WORKSPACE"',
5252
);
53+
expect(checkoutStep.run, jobName).toContain("for attempt in 1 2 3");
54+
expect(checkoutStep.run, jobName).toContain("timed out on attempt $attempt; retrying");
55+
expect(checkoutStep.run, jobName).not.toContain("if timeout --signal=TERM");
5356
expect(checkoutStep.run, jobName).toContain("-c protocol.version=2");
5457
expect(checkoutStep.run, jobName).toContain(
5558
"fetch --no-tags --prune --no-recurse-submodules --depth=1 origin",

0 commit comments

Comments
 (0)