Skip to content

Commit 15de9d8

Browse files
RomneyDaRomneyDa
authored
ci: retry platform checkout fetch timeouts (#97912)
1 parent 29f787f commit 15de9d8

2 files changed

Lines changed: 32 additions & 98 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 21 additions & 98 deletions
Original file line numberDiff line numberDiff line change
@@ -1885,7 +1885,8 @@ jobs:
18851885
max-parallel: 2
18861886
matrix: ${{ fromJson(needs.preflight.outputs.checks_windows_matrix) }}
18871887
steps:
1888-
- name: Checkout
1888+
- &platform_checkout_step
1889+
name: Checkout
18891890
env:
18901891
CHECKOUT_REPO: ${{ github.repository }}
18911892
CHECKOUT_SHA: ${{ needs.preflight.outputs.checkout_revision }}
@@ -1895,7 +1896,7 @@ jobs:
18951896
git -C "$GITHUB_WORKSPACE" config gc.auto 0
18961897
git -C "$GITHUB_WORKSPACE" remote add origin "https://github.com/${CHECKOUT_REPO}.git"
18971898
fetch_timeout_seconds=90
1898-
fetch_checkout_ref() {
1899+
fetch_checkout_ref_once() {
18991900
git -C "$GITHUB_WORKSPACE" \
19001901
-c protocol.version=2 \
19011902
fetch --no-tags --prune --no-recurse-submodules --depth=1 origin \
@@ -1915,6 +1916,21 @@ jobs:
19151916
done
19161917
wait "$fetch_pid"
19171918
}
1919+
fetch_checkout_ref() {
1920+
local fetch_status
1921+
for attempt in 1 2 3; do
1922+
fetch_checkout_ref_once && return 0
1923+
fetch_status="$?"
1924+
if [ "$fetch_status" != "124" ] && [ "$fetch_status" != "137" ]; then
1925+
return "$fetch_status"
1926+
fi
1927+
if [ "$attempt" = "3" ]; then
1928+
return "$fetch_status"
1929+
fi
1930+
echo "::warning::checkout fetch for '$CHECKOUT_SHA' timed out on attempt $attempt; retrying"
1931+
sleep 5
1932+
done
1933+
}
19181934
fetch_checkout_ref
19191935
git -C "$GITHUB_WORKSPACE" checkout --detach refs/remotes/origin/checkout
19201936
@@ -2006,38 +2022,7 @@ jobs:
20062022
fail-fast: false
20072023
matrix: ${{ fromJson(needs.preflight.outputs.macos_node_matrix) }}
20082024
steps:
2009-
- name: Checkout
2010-
env:
2011-
CHECKOUT_REPO: ${{ github.repository }}
2012-
CHECKOUT_SHA: ${{ needs.preflight.outputs.checkout_revision }}
2013-
run: |
2014-
set -euo pipefail
2015-
git init "$GITHUB_WORKSPACE"
2016-
git -C "$GITHUB_WORKSPACE" config gc.auto 0
2017-
git -C "$GITHUB_WORKSPACE" remote add origin "https://github.com/${CHECKOUT_REPO}.git"
2018-
fetch_timeout_seconds=90
2019-
fetch_checkout_ref() {
2020-
git -C "$GITHUB_WORKSPACE" \
2021-
-c protocol.version=2 \
2022-
fetch --no-tags --prune --no-recurse-submodules --depth=1 origin \
2023-
"+${CHECKOUT_SHA}:refs/remotes/origin/checkout" &
2024-
local fetch_pid="$!"
2025-
local elapsed=0
2026-
while kill -0 "$fetch_pid" 2>/dev/null; do
2027-
if [ "$elapsed" -ge "$fetch_timeout_seconds" ]; then
2028-
kill -TERM "$fetch_pid" 2>/dev/null || true
2029-
sleep 10
2030-
kill -KILL "$fetch_pid" 2>/dev/null || true
2031-
wait "$fetch_pid" || true
2032-
return 124
2033-
fi
2034-
sleep 1
2035-
elapsed=$((elapsed + 1))
2036-
done
2037-
wait "$fetch_pid"
2038-
}
2039-
fetch_checkout_ref
2040-
git -C "$GITHUB_WORKSPACE" checkout --detach refs/remotes/origin/checkout
2025+
- *platform_checkout_step
20412026

20422027
- name: Setup Node environment
20432028
uses: ./.github/actions/setup-node-env
@@ -2073,38 +2058,7 @@ jobs:
20732058
runs-on: ${{ github.event_name == 'workflow_dispatch' && 'macos-26' || (github.repository == 'openclaw/openclaw' && 'blacksmith-12vcpu-macos-26' || 'macos-26') }}
20742059
timeout-minutes: 20
20752060
steps:
2076-
- name: Checkout
2077-
env:
2078-
CHECKOUT_REPO: ${{ github.repository }}
2079-
CHECKOUT_SHA: ${{ needs.preflight.outputs.checkout_revision }}
2080-
run: |
2081-
set -euo pipefail
2082-
git init "$GITHUB_WORKSPACE"
2083-
git -C "$GITHUB_WORKSPACE" config gc.auto 0
2084-
git -C "$GITHUB_WORKSPACE" remote add origin "https://github.com/${CHECKOUT_REPO}.git"
2085-
fetch_timeout_seconds=90
2086-
fetch_checkout_ref() {
2087-
git -C "$GITHUB_WORKSPACE" \
2088-
-c protocol.version=2 \
2089-
fetch --no-tags --prune --no-recurse-submodules --depth=1 origin \
2090-
"+${CHECKOUT_SHA}:refs/remotes/origin/checkout" &
2091-
local fetch_pid="$!"
2092-
local elapsed=0
2093-
while kill -0 "$fetch_pid" 2>/dev/null; do
2094-
if [ "$elapsed" -ge "$fetch_timeout_seconds" ]; then
2095-
kill -TERM "$fetch_pid" 2>/dev/null || true
2096-
sleep 10
2097-
kill -KILL "$fetch_pid" 2>/dev/null || true
2098-
wait "$fetch_pid" || true
2099-
return 124
2100-
fi
2101-
sleep 1
2102-
elapsed=$((elapsed + 1))
2103-
done
2104-
wait "$fetch_pid"
2105-
}
2106-
fetch_checkout_ref
2107-
git -C "$GITHUB_WORKSPACE" checkout --detach refs/remotes/origin/checkout
2061+
- *platform_checkout_step
21082062

21092063
- name: Install XcodeGen / SwiftLint / SwiftFormat
21102064
run: brew install xcodegen swiftlint swiftformat
@@ -2211,38 +2165,7 @@ jobs:
22112165
runs-on: ${{ github.event_name == 'workflow_dispatch' && 'macos-26' || (github.repository == 'openclaw/openclaw' && 'blacksmith-12vcpu-macos-26' || 'macos-26') }}
22122166
timeout-minutes: 45
22132167
steps:
2214-
- name: Checkout
2215-
env:
2216-
CHECKOUT_REPO: ${{ github.repository }}
2217-
CHECKOUT_SHA: ${{ needs.preflight.outputs.checkout_revision }}
2218-
run: |
2219-
set -euo pipefail
2220-
git init "$GITHUB_WORKSPACE"
2221-
git -C "$GITHUB_WORKSPACE" config gc.auto 0
2222-
git -C "$GITHUB_WORKSPACE" remote add origin "https://github.com/${CHECKOUT_REPO}.git"
2223-
fetch_timeout_seconds=90
2224-
fetch_checkout_ref() {
2225-
git -C "$GITHUB_WORKSPACE" \
2226-
-c protocol.version=2 \
2227-
fetch --no-tags --prune --no-recurse-submodules --depth=1 origin \
2228-
"+${CHECKOUT_SHA}:refs/remotes/origin/checkout" &
2229-
local fetch_pid="$!"
2230-
local elapsed=0
2231-
while kill -0 "$fetch_pid" 2>/dev/null; do
2232-
if [ "$elapsed" -ge "$fetch_timeout_seconds" ]; then
2233-
kill -TERM "$fetch_pid" 2>/dev/null || true
2234-
sleep 10
2235-
kill -KILL "$fetch_pid" 2>/dev/null || true
2236-
wait "$fetch_pid" || true
2237-
return 124
2238-
fi
2239-
sleep 1
2240-
elapsed=$((elapsed + 1))
2241-
done
2242-
wait "$fetch_pid"
2243-
}
2244-
fetch_checkout_ref
2245-
git -C "$GITHUB_WORKSPACE" checkout --detach refs/remotes/origin/checkout
2168+
- *platform_checkout_step
22462169

22472170
- name: Select Xcode 26
22482171
run: |

‎test/scripts/ci-workflow-guards.test.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -396,12 +396,19 @@ describe("ci workflow guards", () => {
396396
});
397397

398398
it("bounds platform checkout fetches without GNU timeout", () => {
399+
const source = readFileSync(".github/workflows/ci.yml", "utf8");
399400
const workflow = readCiWorkflow();
400401

402+
expect(source.match(/&platform_checkout_step/gu) ?? []).toHaveLength(1);
403+
expect(source.match(/\*platform_checkout_step/gu) ?? []).toHaveLength(3);
404+
expect(source.match(/fetch_checkout_ref_once\(\)/gu) ?? []).toHaveLength(1);
405+
401406
for (const jobName of ["checks-windows", "macos-node", "macos-swift", "ios-build"]) {
402407
const checkoutStep = workflow.jobs[jobName].steps.find((step) => step.name === "Checkout");
403408

404409
expect(checkoutStep.run, jobName).toContain("fetch_checkout_ref()");
410+
expect(checkoutStep.run, jobName).toContain("fetch_checkout_ref_once()");
411+
expect(checkoutStep.run, jobName).toContain("for attempt in 1 2 3");
405412
expect(checkoutStep.run, jobName).toContain("fetch_timeout_seconds=90");
406413
expect(checkoutStep.run, jobName).toContain("-c protocol.version=2");
407414
expect(checkoutStep.run, jobName).toContain(
@@ -412,6 +419,10 @@ describe("ci workflow guards", () => {
412419
);
413420
expect(checkoutStep.run, jobName).toContain('kill -TERM "$fetch_pid"');
414421
expect(checkoutStep.run, jobName).toContain('kill -KILL "$fetch_pid"');
422+
expect(checkoutStep.run, jobName).toContain(
423+
'if [ "$fetch_status" != "124" ] && [ "$fetch_status" != "137" ]; then',
424+
);
425+
expect(checkoutStep.run, jobName).toContain("timed out on attempt $attempt; retrying");
415426
expect(checkoutStep.run, jobName).not.toContain(
416427
'git -C "$GITHUB_WORKSPACE" fetch --no-tags --depth=1',
417428
);

0 commit comments

Comments
 (0)