Skip to content

Commit 89de454

Browse files
RomneyDaRomneyDa
authored
ci: add manual release qa evidence workflow (#95876)
1 parent de9c94c commit 89de454

1 file changed

Lines changed: 313 additions & 0 deletions

File tree

‎.github/workflows/release-qa-profile-evidence.yml‎

Lines changed: 313 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,313 @@
1+
name: Release QA Profile Evidence
2+
3+
run-name: ${{ format('Release QA Profile Evidence {0}', inputs.ref) }}
4+
5+
on:
6+
workflow_dispatch:
7+
inputs:
8+
ref:
9+
description: OpenClaw branch, tag, or SHA to run
10+
required: true
11+
default: main
12+
type: string
13+
expected_sha:
14+
description: Optional full SHA that ref must resolve to
15+
required: false
16+
default: ""
17+
type: string
18+
workflow_call:
19+
inputs:
20+
ref:
21+
description: OpenClaw branch, tag, or SHA to run
22+
required: true
23+
type: string
24+
expected_sha:
25+
description: Optional full SHA that ref must resolve to
26+
required: false
27+
default: ""
28+
type: string
29+
outputs:
30+
artifact_name:
31+
description: Uploaded release QA profile evidence artifact name
32+
value: ${{ jobs.run_release_profile.outputs.artifact_name }}
33+
target_sha:
34+
description: Resolved OpenClaw SHA that produced the evidence
35+
value: ${{ jobs.run_release_profile.outputs.target_sha }}
36+
trusted_reason:
37+
description: Trust reason accepted before the secret-bearing QA job
38+
value: ${{ jobs.run_release_profile.outputs.trusted_reason }}
39+
qa_evidence_path:
40+
description: Path to qa-evidence.json inside the uploaded artifact
41+
value: ${{ jobs.run_release_profile.outputs.qa_evidence_path }}
42+
43+
permissions:
44+
contents: read
45+
46+
concurrency:
47+
group: release-qa-profile-evidence-${{ inputs.expected_sha || inputs.ref }}
48+
cancel-in-progress: false
49+
50+
env:
51+
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
52+
NODE_VERSION: "24.x"
53+
OPENCLAW_BUILD_PRIVATE_QA: "1"
54+
OPENCLAW_ENABLE_PRIVATE_QA_CLI: "1"
55+
OPENCLAW_QA_REDACT_PUBLIC_METADATA: "1"
56+
OPENCLAW_QA_TRANSPORT_READY_TIMEOUT_MS: "180000"
57+
58+
jobs:
59+
authorize_actor:
60+
name: Authorize workflow actor
61+
runs-on: blacksmith-8vcpu-ubuntu-2404
62+
outputs:
63+
authorized: ${{ steps.permission.outputs.authorized }}
64+
steps:
65+
- name: Require maintainer-level repository access
66+
id: permission
67+
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9
68+
with:
69+
script: |
70+
if (context.eventName !== "workflow_dispatch") {
71+
core.info(`Skipping manual actor permission check for ${context.eventName}.`);
72+
core.setOutput("authorized", "true");
73+
return;
74+
}
75+
const allowed = new Set(["admin", "maintain", "write"]);
76+
const { owner, repo } = context.repo;
77+
const { data } = await github.rest.repos.getCollaboratorPermissionLevel({
78+
owner,
79+
repo,
80+
username: context.actor,
81+
});
82+
const permission = data.permission;
83+
core.info(`Actor ${context.actor} permission: ${permission}`);
84+
if (!allowed.has(permission)) {
85+
core.notice(
86+
`Workflow requires write/maintain/admin access. Actor "${context.actor}" has "${permission}".`,
87+
);
88+
core.setOutput("authorized", "false");
89+
return;
90+
}
91+
core.setOutput("authorized", "true");
92+
93+
validate_selected_ref:
94+
name: Validate selected ref
95+
needs: authorize_actor
96+
if: needs.authorize_actor.outputs.authorized == 'true'
97+
runs-on: blacksmith-8vcpu-ubuntu-2404
98+
outputs:
99+
selected_revision: ${{ steps.validate.outputs.selected_revision }}
100+
trusted_reason: ${{ steps.validate.outputs.trusted_reason }}
101+
steps:
102+
- name: Checkout selected ref
103+
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
104+
with:
105+
persist-credentials: false
106+
ref: ${{ inputs.ref }}
107+
fetch-depth: 0
108+
109+
- name: Validate selected ref
110+
id: validate
111+
env:
112+
EXPECTED_SHA: ${{ inputs.expected_sha }}
113+
INPUT_REF: ${{ inputs.ref }}
114+
shell: bash
115+
run: |
116+
set -euo pipefail
117+
118+
selected_revision="$(git rev-parse HEAD)"
119+
expected_sha="${EXPECTED_SHA,,}"
120+
trusted_reason=""
121+
122+
if [[ -n "${expected_sha// }" && ! "$expected_sha" =~ ^[0-9a-f]{40}$ ]]; then
123+
echo "expected_sha must be a full 40-character SHA; got: ${EXPECTED_SHA}" >&2
124+
exit 1
125+
fi
126+
if [[ -n "${expected_sha// }" && "${selected_revision,,}" != "$expected_sha" ]]; then
127+
echo "Ref '${INPUT_REF}' resolved to ${selected_revision}, expected ${EXPECTED_SHA}." >&2
128+
exit 1
129+
fi
130+
131+
git fetch --no-tags origin +refs/heads/main:refs/remotes/origin/main
132+
133+
if git merge-base --is-ancestor "$selected_revision" refs/remotes/origin/main; then
134+
trusted_reason="main-ancestor"
135+
elif git tag --points-at "$selected_revision" | grep -Eq '^v'; then
136+
trusted_reason="release-tag"
137+
elif [[ "$INPUT_REF" =~ ^release/[0-9]{4}\.[0-9]+\.[0-9]+$ ]]; then
138+
git fetch --no-tags origin "+refs/heads/${INPUT_REF}:refs/remotes/origin/${INPUT_REF}"
139+
release_branch_sha="$(git rev-parse "refs/remotes/origin/${INPUT_REF}")"
140+
if [[ "$selected_revision" == "$release_branch_sha" ]]; then
141+
trusted_reason="release-branch-head"
142+
fi
143+
fi
144+
145+
if [[ -z "$trusted_reason" ]]; then
146+
echo "Ref '${INPUT_REF}' resolved to $selected_revision, which is not trusted for this secret-bearing QA evidence run." >&2
147+
echo "Allowed refs must be on main, point to a release tag, or match a release branch head." >&2
148+
exit 1
149+
fi
150+
151+
echo "selected_revision=$selected_revision" >> "$GITHUB_OUTPUT"
152+
echo "trusted_reason=$trusted_reason" >> "$GITHUB_OUTPUT"
153+
{
154+
echo "### Target"
155+
echo
156+
echo "- Requested ref: \`${INPUT_REF}\`"
157+
echo "- Resolved SHA: \`$selected_revision\`"
158+
echo "- Trust reason: \`$trusted_reason\`"
159+
} >> "$GITHUB_STEP_SUMMARY"
160+
161+
run_release_profile:
162+
name: Generate release QA profile evidence
163+
needs: validate_selected_ref
164+
runs-on: blacksmith-8vcpu-ubuntu-2404
165+
timeout-minutes: 60
166+
permissions:
167+
contents: read
168+
outputs:
169+
artifact_name: ${{ steps.evidence.outputs.artifact_name }}
170+
target_sha: ${{ steps.evidence.outputs.target_sha }}
171+
trusted_reason: ${{ steps.evidence.outputs.trusted_reason }}
172+
qa_evidence_path: ${{ steps.evidence.outputs.qa_evidence_path }}
173+
environment: qa-live-shared
174+
steps:
175+
- name: Checkout selected ref
176+
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
177+
with:
178+
persist-credentials: false
179+
ref: ${{ needs.validate_selected_ref.outputs.selected_revision }}
180+
fetch-depth: 1
181+
182+
- name: Setup Node environment
183+
uses: ./.github/actions/setup-node-env
184+
with:
185+
node-version: ${{ env.NODE_VERSION }}
186+
install-bun: "true"
187+
188+
- name: Validate required QA credential env
189+
env:
190+
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
191+
shell: bash
192+
run: |
193+
set -euo pipefail
194+
if [[ -z "${OPENAI_API_KEY:-}" ]]; then
195+
echo "Missing required OPENAI_API_KEY." >&2
196+
exit 1
197+
fi
198+
199+
- name: Build private QA runtime
200+
env:
201+
NODE_OPTIONS: --max-old-space-size=8192
202+
run: node scripts/build-all.mjs qaRuntime
203+
204+
- name: Run release QA profile
205+
id: run_profile
206+
env:
207+
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
208+
shell: bash
209+
run: |
210+
set -euo pipefail
211+
212+
output_dir=".artifacts/qa-e2e/release-profile-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
213+
echo "output_dir=${output_dir}" >> "$GITHUB_OUTPUT"
214+
215+
qa_exit_code=0
216+
pnpm openclaw qa run \
217+
--repo-root . \
218+
--qa-profile release \
219+
--output-dir "${output_dir}" || qa_exit_code=$?
220+
221+
echo "qa_exit_code=${qa_exit_code}" >> "$GITHUB_OUTPUT"
222+
223+
- name: Validate release QA evidence
224+
id: evidence
225+
if: always()
226+
env:
227+
ARTIFACT_NAME: release-qa-profile-evidence-${{ needs.validate_selected_ref.outputs.selected_revision }}
228+
OUTPUT_DIR: ${{ steps.run_profile.outputs.output_dir }}
229+
REQUESTED_REF: ${{ inputs.ref }}
230+
TARGET_SHA: ${{ needs.validate_selected_ref.outputs.selected_revision }}
231+
TRUSTED_REASON: ${{ needs.validate_selected_ref.outputs.trusted_reason }}
232+
shell: bash
233+
run: |
234+
set -euo pipefail
235+
236+
node --input-type=module <<'NODE'
237+
import fs from "node:fs";
238+
import path from "node:path";
239+
240+
const outputDir = process.env.OUTPUT_DIR;
241+
if (!outputDir) {
242+
throw new Error("OUTPUT_DIR is required");
243+
}
244+
245+
const evidencePath = path.join(outputDir, "qa-evidence.json");
246+
const payload = JSON.parse(fs.readFileSync(evidencePath, "utf8"));
247+
if (payload.profile !== "release") {
248+
throw new Error(`qa-evidence.json profile must be release, got ${JSON.stringify(payload.profile)}`);
249+
}
250+
if (!payload.scorecard || !Array.isArray(payload.scorecard.categoryReports)) {
251+
throw new Error("release qa-evidence.json must include scorecard.categoryReports");
252+
}
253+
if (payload.scorecard.categoryReports.length === 0) {
254+
throw new Error("release qa-evidence.json scorecard has no category reports");
255+
}
256+
257+
const manifest = {
258+
artifactName: process.env.ARTIFACT_NAME,
259+
generatedAt: new Date().toISOString(),
260+
requestedRef: process.env.REQUESTED_REF,
261+
targetSha: process.env.TARGET_SHA,
262+
trustedReason: process.env.TRUSTED_REASON,
263+
evidenceMode: payload.evidenceMode,
264+
qaEvidencePath: "qa-evidence.json",
265+
scorecard: {
266+
categories: payload.scorecard.categories,
267+
features: payload.scorecard.features,
268+
categoryReports: payload.scorecard.categoryReports.length,
269+
},
270+
};
271+
fs.writeFileSync(
272+
path.join(outputDir, "release-qa-profile-evidence-manifest.json"),
273+
`${JSON.stringify(manifest, null, 2)}\n`,
274+
);
275+
NODE
276+
277+
echo "artifact_name=${ARTIFACT_NAME}" >> "$GITHUB_OUTPUT"
278+
echo "target_sha=${TARGET_SHA}" >> "$GITHUB_OUTPUT"
279+
echo "trusted_reason=${TRUSTED_REASON}" >> "$GITHUB_OUTPUT"
280+
echo "qa_evidence_path=qa-evidence.json" >> "$GITHUB_OUTPUT"
281+
{
282+
echo "### Release QA profile evidence"
283+
echo
284+
echo "- Artifact: \`${ARTIFACT_NAME}\`"
285+
echo "- Target SHA: \`${TARGET_SHA}\`"
286+
echo "- Evidence path: \`${OUTPUT_DIR}/qa-evidence.json\`"
287+
echo "- Manifest: \`${OUTPUT_DIR}/release-qa-profile-evidence-manifest.json\`"
288+
} >> "$GITHUB_STEP_SUMMARY"
289+
290+
- name: Upload release QA profile evidence
291+
if: always()
292+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
293+
with:
294+
name: release-qa-profile-evidence-${{ needs.validate_selected_ref.outputs.selected_revision }}
295+
path: ${{ steps.run_profile.outputs.output_dir }}
296+
retention-days: 30
297+
if-no-files-found: error
298+
299+
- name: Fail if release QA profile failed
300+
if: always()
301+
env:
302+
QA_EXIT_CODE: ${{ steps.run_profile.outputs.qa_exit_code }}
303+
shell: bash
304+
run: |
305+
set -euo pipefail
306+
if [[ -z "${QA_EXIT_CODE:-}" ]]; then
307+
echo "Release QA profile did not report an exit code." >&2
308+
exit 1
309+
fi
310+
if [[ "$QA_EXIT_CODE" != "0" ]]; then
311+
echo "Release QA profile failed with exit code ${QA_EXIT_CODE}." >&2
312+
exit "$QA_EXIT_CODE"
313+
fi

0 commit comments

Comments
 (0)