Skip to content

Commit 7e0083c

Browse files
authored
ci: add release QA profile evidence (#95094)
* ci: add release qa profile evidence * ci: simplify release qa profile evidence * ci: reuse qa profile evidence workflow * ci: remove inherited secrets lint comment * ci: pass qa profile evidence secret explicitly * ci: run maturity scorecard in release checks * ci: declare maturity scorecard reusable secret
1 parent 740578b commit 7e0083c

4 files changed

Lines changed: 149 additions & 2 deletions

File tree

.github/workflows/maturity-scorecard.yml

Lines changed: 49 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,40 @@ on:
1212
required: true
1313
default: main
1414
type: string
15+
expected_sha:
16+
description: Optional full SHA that ref must resolve to
17+
required: false
18+
default: ""
19+
type: string
20+
workflow_call:
21+
inputs:
22+
qa_evidence_run_id:
23+
description: Optional workflow run id containing qa-evidence.json
24+
required: false
25+
default: ""
26+
type: string
27+
ref:
28+
description: OpenClaw branch, tag, or SHA containing the maturity score source
29+
required: true
30+
type: string
31+
expected_sha:
32+
description: Optional full SHA that ref must resolve to
33+
required: false
34+
default: ""
35+
type: string
36+
secrets:
37+
OPENAI_API_KEY:
38+
description: OpenAI API key used by live QA profile scenarios
39+
required: true
40+
OPENCLAW_MATURITY_SCORECARD_AGENT_OPENAI_API_KEY:
41+
description: Optional OpenAI API key used by maturity scorecard agent steps
42+
required: false
43+
GH_APP_PRIVATE_KEY:
44+
description: Optional GitHub App private key for generated docs PR creation
45+
required: false
46+
GH_APP_PRIVATE_KEY_FALLBACK:
47+
description: Optional fallback GitHub App private key for generated docs PR creation
48+
required: false
1549

1650
permissions:
1751
actions: read
@@ -43,14 +77,25 @@ jobs:
4377
- name: Validate selected ref
4478
id: validate
4579
env:
80+
EXPECTED_SHA: ${{ inputs.expected_sha }}
4681
INPUT_REF: ${{ inputs.ref }}
4782
shell: bash
4883
run: |
4984
set -euo pipefail
5085
5186
selected_revision="$(git rev-parse HEAD)"
87+
expected_sha="${EXPECTED_SHA,,}"
5288
trusted_reason=""
5389
90+
if [[ -n "${expected_sha// }" && ! "$expected_sha" =~ ^[0-9a-f]{40}$ ]]; then
91+
echo "expected_sha must be a full 40-character SHA; got: ${EXPECTED_SHA}" >&2
92+
exit 1
93+
fi
94+
if [[ -n "${expected_sha// }" && "${selected_revision,,}" != "$expected_sha" ]]; then
95+
echo "Ref '${INPUT_REF}' resolved to ${selected_revision}, expected ${EXPECTED_SHA}." >&2
96+
exit 1
97+
fi
98+
5499
git fetch --no-tags origin +refs/heads/main:refs/remotes/origin/main
55100
56101
if git merge-base --is-ancestor "$selected_revision" refs/remotes/origin/main; then
@@ -88,6 +133,7 @@ jobs:
88133
uses: ./.github/workflows/qa-profile-evidence.yml
89134
with:
90135
ref: ${{ needs.validate_selected_ref.outputs.selected_revision }}
136+
expected_sha: ${{ needs.validate_selected_ref.outputs.selected_revision }}
91137
qa_profile: all
92138
secrets:
93139
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
@@ -321,6 +367,7 @@ jobs:
321367
--strict-inputs
322368
323369
- name: Create generated docs PR app token
370+
if: ${{ github.event_name == 'workflow_dispatch' }}
324371
id: app-token
325372
continue-on-error: true
326373
uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3
@@ -331,7 +378,7 @@ jobs:
331378
permission-pull-requests: write
332379

333380
- name: Create generated docs PR fallback app token
334-
if: ${{ steps.app-token.outcome == 'failure' }}
381+
if: ${{ github.event_name == 'workflow_dispatch' && steps.app-token.outcome == 'failure' }}
335382
id: app-token-fallback
336383
uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3
337384
with:
@@ -341,6 +388,7 @@ jobs:
341388
permission-pull-requests: write
342389

343390
- name: Open generated docs PR
391+
if: ${{ github.event_name == 'workflow_dispatch' }}
344392
env:
345393
GH_TOKEN: ${{ steps.app-token.outputs.token || steps.app-token-fallback.outputs.token }}
346394
QA_EVIDENCE_RUN_ID: ${{ inputs.qa_evidence_run_id }}

.github/workflows/openclaw-release-checks.yml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -767,6 +767,20 @@ jobs:
767767
OPENCLAW_QA_CONVEX_SITE_URL: ${{ secrets.OPENCLAW_QA_CONVEX_SITE_URL }}
768768
OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }}
769769

770+
maturity_scorecard_release_checks:
771+
name: Render maturity scorecard release docs
772+
needs: [resolve_target]
773+
if: contains(fromJSON('["all","qa"]'), needs.resolve_target.outputs.rerun_group)
774+
permissions:
775+
actions: read
776+
contents: read
777+
uses: ./.github/workflows/maturity-scorecard.yml
778+
with:
779+
ref: ${{ needs.resolve_target.outputs.ref }}
780+
expected_sha: ${{ needs.resolve_target.outputs.revision }}
781+
secrets:
782+
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
783+
770784
qa_lab_parity_lane_release_checks:
771785
name: Run QA Lab parity lane (${{ matrix.lane }})
772786
needs: [resolve_target]
@@ -1946,6 +1960,7 @@ jobs:
19461960
- docker_e2e_release_checks
19471961
- package_acceptance_release_checks
19481962
- qa_lab_parity_lane_release_checks
1963+
- maturity_scorecard_release_checks
19491964
- qa_lab_parity_report_release_checks
19501965
- qa_lab_runtime_parity_release_checks
19511966
- runtime_tool_coverage_release_checks
@@ -2031,6 +2046,7 @@ jobs:
20312046
"docker_e2e_release_checks=${{ needs.docker_e2e_release_checks.result }}" \
20322047
"package_acceptance_release_checks=${{ needs.package_acceptance_release_checks.result }}" \
20332048
"qa_lab_parity_lane_release_checks=${{ needs.qa_lab_parity_lane_release_checks.result }}" \
2049+
"maturity_scorecard_release_checks=${{ needs.maturity_scorecard_release_checks.result }}" \
20342050
"qa_lab_parity_report_release_checks=${{ needs.qa_lab_parity_report_release_checks.result }}" \
20352051
"qa_lab_runtime_parity_release_checks=${{ needs.qa_lab_runtime_parity_release_checks.result }}" \
20362052
"runtime_tool_coverage_release_checks=${{ needs.runtime_tool_coverage_release_checks.result }}" \

docs/ci.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -198,7 +198,7 @@ Every lane uploads GitHub artifacts. When `CLAWGRIT_REPORTS_TOKEN` is configured
198198

199199
## Full Release Validation
200200

201-
`Full Release Validation` is the manual umbrella workflow for "run everything before release." It accepts a branch, tag, or full commit SHA, dispatches the manual `CI` workflow with that target, dispatches `Plugin Prerelease` for release-only plugin/package/static/Docker proof, and dispatches `OpenClaw Release Checks` for install smoke, package acceptance, cross-OS package checks, QA Lab parity, Matrix, and Telegram lanes. Stable and full profiles always include exhaustive live/E2E and Docker release-path soak coverage; the beta profile can opt in with `run_release_soak=true`. The canonical package Telegram E2E runs inside Package Acceptance, so a full candidate does not start a duplicate live poller. After publishing, pass `release_package_spec` to reuse the shipped npm package across release checks, Package Acceptance, Docker, cross-OS, and Telegram without rebuilding. Use `npm_telegram_package_spec` only for a focused published-package Telegram rerun. The Codex plugin live package lane uses the same selected state by default: published `release_package_spec=openclaw@<tag>` derives `codex_plugin_spec=npm:@openclaw/codex@<tag>`, while SHA/artifact runs pack `extensions/codex` from the selected ref. Set `codex_plugin_spec` explicitly for custom plugin sources such as `npm:`, `npm-pack:`, or `git:` specs.
201+
`Full Release Validation` is the manual umbrella workflow for "run everything before release." It accepts a branch, tag, or full commit SHA, dispatches the manual `CI` workflow with that target, dispatches `Plugin Prerelease` for release-only plugin/package/static/Docker proof, and dispatches `OpenClaw Release Checks` for install smoke, package acceptance, cross-OS package checks, maturity scorecard rendering from QA profile evidence, QA Lab parity, Matrix, and Telegram lanes. Stable and full profiles always include exhaustive live/E2E and Docker release-path soak coverage; the beta profile can opt in with `run_release_soak=true`. The canonical package Telegram E2E runs inside Package Acceptance, so a full candidate does not start a duplicate live poller. After publishing, pass `release_package_spec` to reuse the shipped npm package across release checks, Package Acceptance, Docker, cross-OS, and Telegram without rebuilding. Use `npm_telegram_package_spec` only for a focused published-package Telegram rerun. The Codex plugin live package lane uses the same selected state by default: published `release_package_spec=openclaw@<tag>` derives `codex_plugin_spec=npm:@openclaw/codex@<tag>`, while SHA/artifact runs pack `extensions/codex` from the selected ref. Set `codex_plugin_spec` explicitly for custom plugin sources such as `npm:`, `npm-pack:`, or `git:` specs.
202202

203203
See [Full release validation](/reference/full-release-validation) for the
204204
stage matrix, exact workflow job names, profile differences, artifacts, and

test/scripts/ci-workflow-guards.test.ts

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,10 @@ function readQaProfileEvidenceWorkflow() {
2929
return parse(readFileSync(".github/workflows/qa-profile-evidence.yml", "utf8"));
3030
}
3131

32+
function readReleaseChecksWorkflow() {
33+
return parse(readFileSync(".github/workflows/openclaw-release-checks.yml", "utf8"));
34+
}
35+
3236
function readCriticalQualityWorkflow() {
3337
return readFileSync(".github/workflows/codeql-critical-quality.yml", "utf8");
3438
}
@@ -608,6 +612,34 @@ describe("ci workflow guards", () => {
608612
const publishJob = maturityWorkflow.jobs.publish;
609613
const qaRunJob = qaEvidenceWorkflow.jobs.run_qa_profile;
610614

615+
expect(maturityWorkflow.on.workflow_call.inputs).toMatchObject({
616+
qa_evidence_run_id: {
617+
description: "Optional workflow run id containing qa-evidence.json",
618+
required: false,
619+
default: "",
620+
type: "string",
621+
},
622+
ref: {
623+
description: "OpenClaw branch, tag, or SHA containing the maturity score source",
624+
required: true,
625+
type: "string",
626+
},
627+
expected_sha: {
628+
description: "Optional full SHA that ref must resolve to",
629+
required: false,
630+
default: "",
631+
type: "string",
632+
},
633+
});
634+
expect(maturityWorkflow.on.workflow_call.secrets.OPENAI_API_KEY.required).toBe(true);
635+
expect(
636+
maturityWorkflow.on.workflow_call.secrets.OPENCLAW_MATURITY_SCORECARD_AGENT_OPENAI_API_KEY
637+
.required,
638+
).toBe(false);
639+
expect(maturityWorkflow.on.workflow_call.secrets.GH_APP_PRIVATE_KEY.required).toBe(false);
640+
expect(maturityWorkflow.on.workflow_call.secrets.GH_APP_PRIVATE_KEY_FALLBACK.required).toBe(
641+
false,
642+
);
611643
expect(qaEvidenceWorkflow.on.workflow_dispatch.inputs).not.toHaveProperty("fail_on_qa_failure");
612644
expect(qaEvidenceWorkflow.on.workflow_call.inputs).not.toHaveProperty("fail_on_qa_failure");
613645
expect(qaEvidenceWorkflow.on.workflow_dispatch.inputs.qa_profile).not.toHaveProperty("options");
@@ -627,10 +659,18 @@ describe("ci workflow guards", () => {
627659
expect(generateJob.uses).toBe("./.github/workflows/qa-profile-evidence.yml");
628660
expect(generateJob.with).toMatchObject({
629661
ref: "${{ needs.validate_selected_ref.outputs.selected_revision }}",
662+
expected_sha: "${{ needs.validate_selected_ref.outputs.selected_revision }}",
630663
qa_profile: "all",
631664
});
632665
expect(generateJob.with).not.toHaveProperty("fail_on_qa_failure");
633666

667+
const validateRefStep = maturityWorkflow.jobs.validate_selected_ref.steps.find(
668+
(step) => step.name === "Validate selected ref",
669+
);
670+
expect(validateRefStep.env.EXPECTED_SHA).toBe("${{ inputs.expected_sha }}");
671+
expect(validateRefStep.run).toContain("expected_sha must be a full 40-character SHA");
672+
expect(validateRefStep.run).toContain('"${selected_revision,,}" != "$expected_sha"');
673+
634674
const generatedDownloadStep = publishJob.steps.find(
635675
(step) => step.name === "Download generated QA evidence artifact",
636676
);
@@ -671,6 +711,49 @@ describe("ci workflow guards", () => {
671711

672712
const qaFailStep = qaRunJob.steps.find((step) => step.name === "Fail if QA profile failed");
673713
expect(qaFailStep.if).toBe("always()");
714+
715+
const createTokenStep = publishJob.steps.find(
716+
(step) => step.name === "Create generated docs PR app token",
717+
);
718+
const createFallbackTokenStep = publishJob.steps.find(
719+
(step) => step.name === "Create generated docs PR fallback app token",
720+
);
721+
const openDocsPrStep = publishJob.steps.find((step) => step.name === "Open generated docs PR");
722+
expect(createTokenStep.if).toBe("${{ github.event_name == 'workflow_dispatch' }}");
723+
expect(createFallbackTokenStep.if).toBe(
724+
"${{ github.event_name == 'workflow_dispatch' && steps.app-token.outcome == 'failure' }}",
725+
);
726+
expect(openDocsPrStep.if).toBe("${{ github.event_name == 'workflow_dispatch' }}");
727+
});
728+
729+
it("runs maturity scorecard from release checks", () => {
730+
const releaseWorkflow = readReleaseChecksWorkflow();
731+
const job = releaseWorkflow.jobs.maturity_scorecard_release_checks;
732+
const summaryJob = releaseWorkflow.jobs.summary;
733+
const verifyStep = summaryJob.steps.find(
734+
(step) => step.name === "Verify release check results",
735+
);
736+
737+
expect(releaseWorkflow.jobs).not.toHaveProperty("qa_profile_release_evidence_release_checks");
738+
expect(job.name).toBe("Render maturity scorecard release docs");
739+
expect(job.if).toBe(
740+
'contains(fromJSON(\'["all","qa"]\'), needs.resolve_target.outputs.rerun_group)',
741+
);
742+
expect(job.permissions).toMatchObject({
743+
actions: "read",
744+
contents: "read",
745+
});
746+
expect(job.uses).toBe("./.github/workflows/maturity-scorecard.yml");
747+
expect(job.with).toMatchObject({
748+
ref: "${{ needs.resolve_target.outputs.ref }}",
749+
expected_sha: "${{ needs.resolve_target.outputs.revision }}",
750+
});
751+
expect(job.with).not.toHaveProperty("qa_profile");
752+
expect(summaryJob.needs).toContain("maturity_scorecard_release_checks");
753+
expect(verifyStep.run).toContain(
754+
'"maturity_scorecard_release_checks=${{ needs.maturity_scorecard_release_checks.result }}"',
755+
);
756+
expect(verifyStep.run).not.toContain("qa_profile_release_evidence_release_checks");
674757
});
675758

676759
it("keeps workflow guards in fast CI-routing checks", () => {

0 commit comments

Comments
 (0)