Skip to content

Commit aca96d2

Browse files
steipetesteipete
authored
ci: retry transient release workflow reads (#102896)
1 parent 13821c9 commit aca96d2

2 files changed

Lines changed: 100 additions & 9 deletions

File tree

‎.github/workflows/full-release-validation.yml‎

Lines changed: 33 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,8 @@ concurrency:
119119
env:
120120
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
121121
GH_REPO: ${{ github.repository }}
122+
# Read retries and one-shot dispatch recovery share this classifier; dispatch POSTs never retry.
123+
GH_TRANSIENT_SERVER_OR_NETWORK_PATTERN: "HTTP 5[0-9][0-9]|Server Error|error connecting to|context deadline exceeded|connection reset by peer|connection refused|TLS handshake timeout|i/o timeout|network is unreachable|(^|[^A-Za-z0-9_])EOF([^A-Za-z0-9_]|$)|ETIMEDOUT|ECONNRESET|EAI_AGAIN"
122124
NODE_VERSION: "24.15.0"
123125

124126
jobs:
@@ -290,7 +292,7 @@ jobs:
290292
printf '%s\n' "$output"
291293
return 0
292294
fi
293-
if [[ "$output" == *"Bad credentials"* || "$output" == *"HTTP 401"* || "$output" == *"secondary rate limit"* || "$output" == *"API rate limit"* || "$output" == *"Sorry. Your account was suspended"* ]]; then
295+
if [[ "$output" == *"Bad credentials"* || "$output" == *"HTTP 401"* || "$output" == *"secondary rate limit"* || "$output" == *"API rate limit"* || "$output" == *"HTTP 429"* || "$output" == *"abuse detection"* || "$output" == *"Sorry. Your account was suspended"* || "$output" =~ $GH_TRANSIENT_SERVER_OR_NETWORK_PATTERN ]]; then
294296
echo "::warning::gh $* failed on attempt ${attempt}: ${output}" >&2
295297
sleep $((attempt * 10))
296298
continue
@@ -308,6 +310,11 @@ jobs:
308310
set -e
309311
printf '%s\n' "$dispatch_output"
310312
313+
if [[ "$dispatch_status" -ne 0 && ! "$dispatch_output" =~ $GH_TRANSIENT_SERVER_OR_NETWORK_PATTERN ]]; then
314+
echo "::error::${workflow} dispatch failed with non-ambiguous status ${dispatch_status}; refusing adoption polling." >&2
315+
exit "$dispatch_status"
316+
fi
317+
311318
run_id=""
312319
for _ in $(seq 1 60); do
313320
if matches_json="$(
@@ -450,7 +457,7 @@ jobs:
450457
printf '%s\n' "$output"
451458
return 0
452459
fi
453-
if [[ "$output" == *"Bad credentials"* || "$output" == *"HTTP 401"* || "$output" == *"secondary rate limit"* || "$output" == *"API rate limit"* || "$output" == *"Sorry. Your account was suspended"* ]]; then
460+
if [[ "$output" == *"Bad credentials"* || "$output" == *"HTTP 401"* || "$output" == *"secondary rate limit"* || "$output" == *"API rate limit"* || "$output" == *"HTTP 429"* || "$output" == *"abuse detection"* || "$output" == *"Sorry. Your account was suspended"* || "$output" =~ $GH_TRANSIENT_SERVER_OR_NETWORK_PATTERN ]]; then
454461
echo "::warning::gh $* failed on attempt ${attempt}: ${output}" >&2
455462
sleep $((attempt * 10))
456463
continue
@@ -468,6 +475,11 @@ jobs:
468475
set -e
469476
printf '%s\n' "$dispatch_output"
470477
478+
if [[ "$dispatch_status" -ne 0 && ! "$dispatch_output" =~ $GH_TRANSIENT_SERVER_OR_NETWORK_PATTERN ]]; then
479+
echo "::error::${workflow} dispatch failed with non-ambiguous status ${dispatch_status}; refusing adoption polling." >&2
480+
exit "$dispatch_status"
481+
fi
482+
471483
run_id=""
472484
for _ in $(seq 1 60); do
473485
if matches_json="$(
@@ -620,7 +632,7 @@ jobs:
620632
printf '%s\n' "$output"
621633
return 0
622634
fi
623-
if [[ "$output" == *"Bad credentials"* || "$output" == *"HTTP 401"* || "$output" == *"secondary rate limit"* || "$output" == *"API rate limit"* || "$output" == *"Sorry. Your account was suspended"* ]]; then
635+
if [[ "$output" == *"Bad credentials"* || "$output" == *"HTTP 401"* || "$output" == *"secondary rate limit"* || "$output" == *"API rate limit"* || "$output" == *"HTTP 429"* || "$output" == *"abuse detection"* || "$output" == *"Sorry. Your account was suspended"* || "$output" =~ $GH_TRANSIENT_SERVER_OR_NETWORK_PATTERN ]]; then
624636
echo "::warning::gh $* failed on attempt ${attempt}: ${output}" >&2
625637
sleep $((attempt * 10))
626638
continue
@@ -638,6 +650,11 @@ jobs:
638650
set -e
639651
printf '%s\n' "$dispatch_output"
640652
653+
if [[ "$dispatch_status" -ne 0 && ! "$dispatch_output" =~ $GH_TRANSIENT_SERVER_OR_NETWORK_PATTERN ]]; then
654+
echo "::error::${workflow} dispatch failed with non-ambiguous status ${dispatch_status}; refusing adoption polling." >&2
655+
exit "$dispatch_status"
656+
fi
657+
641658
run_id=""
642659
for _ in $(seq 1 60); do
643660
if matches_json="$(
@@ -883,7 +900,7 @@ jobs:
883900
printf '%s\n' "$output"
884901
return 0
885902
fi
886-
if [[ "$output" == *"Bad credentials"* || "$output" == *"HTTP 401"* || "$output" == *"secondary rate limit"* || "$output" == *"API rate limit"* || "$output" == *"Sorry. Your account was suspended"* ]]; then
903+
if [[ "$output" == *"Bad credentials"* || "$output" == *"HTTP 401"* || "$output" == *"secondary rate limit"* || "$output" == *"API rate limit"* || "$output" == *"HTTP 429"* || "$output" == *"abuse detection"* || "$output" == *"Sorry. Your account was suspended"* || "$output" =~ $GH_TRANSIENT_SERVER_OR_NETWORK_PATTERN ]]; then
887904
echo "::warning::gh $* failed on attempt ${attempt}: ${output}" >&2
888905
sleep $((attempt * 10))
889906
continue
@@ -911,6 +928,11 @@ jobs:
911928
set -e
912929
printf '%s\n' "$dispatch_output"
913930
931+
if [[ "$dispatch_status" -ne 0 && ! "$dispatch_output" =~ $GH_TRANSIENT_SERVER_OR_NETWORK_PATTERN ]]; then
932+
echo "::error::npm-telegram-beta-e2e.yml dispatch failed with non-ambiguous status ${dispatch_status}; refusing adoption polling." >&2
933+
exit "$dispatch_status"
934+
fi
935+
914936
run_id=""
915937
for _ in $(seq 1 60); do
916938
if matches_json="$(
@@ -1026,7 +1048,7 @@ jobs:
10261048
printf '%s\n' "$output"
10271049
return 0
10281050
fi
1029-
if [[ "$output" == *"Bad credentials"* || "$output" == *"HTTP 401"* || "$output" == *"secondary rate limit"* || "$output" == *"API rate limit"* || "$output" == *"Sorry. Your account was suspended"* ]]; then
1051+
if [[ "$output" == *"Bad credentials"* || "$output" == *"HTTP 401"* || "$output" == *"secondary rate limit"* || "$output" == *"API rate limit"* || "$output" == *"HTTP 429"* || "$output" == *"abuse detection"* || "$output" == *"Sorry. Your account was suspended"* || "$output" =~ $GH_TRANSIENT_SERVER_OR_NETWORK_PATTERN ]]; then
10301052
echo "::warning::gh $* failed on attempt ${attempt}: ${output}" >&2
10311053
sleep $((attempt * 10))
10321054
continue
@@ -1067,6 +1089,11 @@ jobs:
10671089
set -e
10681090
printf '%s\n' "$dispatch_output"
10691091
1092+
if [[ "$dispatch_status" -ne 0 && ! "$dispatch_output" =~ $GH_TRANSIENT_SERVER_OR_NETWORK_PATTERN ]]; then
1093+
echo "::error::openclaw-performance.yml dispatch failed with non-ambiguous status ${dispatch_status}; refusing adoption polling." >&2
1094+
exit "$dispatch_status"
1095+
fi
1096+
10701097
run_id=""
10711098
for _ in $(seq 1 60); do
10721099
if matches_json="$(
@@ -1181,7 +1208,7 @@ jobs:
11811208
printf '%s\n' "$output"
11821209
return 0
11831210
fi
1184-
if [[ "$output" == *"Bad credentials"* || "$output" == *"HTTP 401"* || "$output" == *"secondary rate limit"* || "$output" == *"API rate limit"* || "$output" == *"Sorry. Your account was suspended"* ]]; then
1211+
if [[ "$output" == *"Bad credentials"* || "$output" == *"HTTP 401"* || "$output" == *"secondary rate limit"* || "$output" == *"API rate limit"* || "$output" == *"HTTP 429"* || "$output" == *"abuse detection"* || "$output" == *"Sorry. Your account was suspended"* || "$output" =~ $GH_TRANSIENT_SERVER_OR_NETWORK_PATTERN ]]; then
11851212
echo "::warning::gh $* failed on attempt ${attempt}: ${output}" >&2
11861213
sleep $((attempt * 10))
11871214
continue

‎test/scripts/package-acceptance-workflow.test.ts‎

Lines changed: 67 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -589,26 +589,90 @@ describe("package acceptance workflow", () => {
589589
["npm_telegram", "Dispatch and monitor npm Telegram E2E"],
590590
["performance", "Dispatch and monitor OpenClaw Performance"],
591591
] as const;
592-
593-
for (const [jobName, stepName] of childDispatches) {
592+
const dispatchScripts = childDispatches.map(([jobName, stepName]) => {
594593
const job = workflowJob(FULL_RELEASE_VALIDATION_WORKFLOW, jobName);
595-
const script = workflowStep(job, stepName).run ?? "";
594+
return workflowStep(job, stepName).run ?? "";
595+
});
596596

597+
for (const script of dispatchScripts) {
597598
expect(script.match(/gh workflow run/gu)).toHaveLength(1);
598599
expect(script).not.toContain("gh_with_retry workflow run");
599600
expectTextToIncludeAll(script, [
600601
"A failed dispatch POST can still create a run. Never retry it",
601602
"set +e",
602603
"dispatch_status=$?",
604+
'if [[ "$dispatch_status" -ne 0 && ! "$dispatch_output" =~ $GH_TRANSIENT_SERVER_OR_NETWORK_PATTERN ]]',
605+
"dispatch failed with non-ambiguous status ${dispatch_status}; refusing adoption polling.",
603606
'DISPATCH_RUN_NAME="$dispatch_run_name" CHILD_WORKFLOW_REF="$CHILD_WORKFLOW_REF"',
604607
".display_title == env.DISPATCH_RUN_NAME and .head_branch == env.CHILD_WORKFLOW_REF",
605608
"Multiple runs matched ${dispatch_run_name}; refusing to guess.",
606609
"The dispatch was not retried to avoid creating a duplicate child.",
607610
"adopted exact run ${run_id}",
608611
]);
612+
expect(script.indexOf("dispatch failed with non-ambiguous status")).toBeLessThan(
613+
script.indexOf('run_id=""'),
614+
);
615+
}
616+
617+
const parsedWorkflow = readWorkflow(FULL_RELEASE_VALIDATION_WORKFLOW);
618+
const transientPattern = parsedWorkflow.env?.GH_TRANSIENT_SERVER_OR_NETWORK_PATTERN;
619+
expect(transientPattern).toBeDefined();
620+
const transientError = new RegExp(transientPattern ?? "", "u");
621+
for (const message of [
622+
"could not create workflow dispatch event: HTTP 500: Failed to run workflow dispatch",
623+
"gh: HTTP 502",
624+
"500 Internal Server Error",
625+
"error connecting to api.github.com",
626+
"context deadline exceeded",
627+
"read: connection reset by peer",
628+
"connect: connection refused",
629+
"net/http: TLS handshake timeout",
630+
"read: i/o timeout",
631+
"network is unreachable",
632+
"unexpected EOF",
633+
'Post "https://api.github.com/repos/openclaw/openclaw/actions/workflows/ci.yml/dispatches": EOF',
634+
"EOF",
635+
"ETIMEDOUT",
636+
"ECONNRESET",
637+
"EAI_AGAIN",
638+
]) {
639+
expect(transientError.test(message), message).toBe(true);
640+
}
641+
for (const message of [
642+
"HTTP 400: Bad Request",
643+
"HTTP 401: Bad credentials",
644+
"HTTP 403: Resource not accessible by integration",
645+
"HTTP 404: workflow not found",
646+
"HTTP 422: Validation Failed",
647+
"HTTP 429: too many requests",
648+
"unknown flag --field",
649+
"EOFError while parsing local input",
650+
]) {
651+
expect(transientError.test(message), message).toBe(false);
652+
}
653+
654+
const summaryScript =
655+
workflowStep(
656+
workflowJob(FULL_RELEASE_VALIDATION_WORKFLOW, "summary"),
657+
"Verify child workflow results",
658+
).run ?? "";
659+
for (const script of [...dispatchScripts, summaryScript]) {
660+
expect(script.match(/gh_with_retry\(\)/gu)).toHaveLength(1);
661+
expectTextToIncludeAll(script, [
662+
'"$output" == *"HTTP 429"*',
663+
'"$output" == *"abuse detection"*',
664+
'"$output" =~ $GH_TRANSIENT_SERVER_OR_NETWORK_PATTERN',
665+
'return "$status"',
666+
]);
609667
}
610668

611669
const workflow = readFileSync(FULL_RELEASE_VALIDATION_WORKFLOW, "utf8");
670+
const retryCalls = workflow.split("\n").filter((line) => line.includes("gh_with_retry "));
671+
expect(retryCalls).toHaveLength(28);
672+
for (const call of retryCalls) {
673+
expect(call).toMatch(/gh_with_retry (api|run view)/u);
674+
}
675+
expect(workflow).not.toMatch(/gh_with_retry (workflow run|run cancel)/u);
612676
expectTextToIncludeAll(workflow, [
613677
'dispatch_id="full-release-validation-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-ci"',
614678
'dispatch_id="full-release-validation-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-plugin-prerelease"',

0 commit comments

Comments
 (0)