Skip to content

Commit e7e8b67

Browse files
refactor(e2e): centralize workflow plan output (#6970)
## Summary Moves E2E plan validation, GitHub output emission, and execution-plan summary rendering from the matrix-generation workflow shell into the typed planner. Controller-dispatch identity validation intentionally stays in the trusted inline workflow shell. ## Related Issue Fixes #6955 ## Changes - Add `--ci-output` to the E2E workflow planner so it validates selectors, inference mode, plan schema, and Hermes selection before writing the four GitHub outputs and the equivalent job summary. - Reduce the matrix-generation workflow step to a single planner invocation while the workflow-boundary tests keep asserting the wiring. - Keep controller-dispatch identity validation in the inline workflow shell. The `generate-matrix` job checks out the untrusted PR commit before the gate runs, so resolving the gate from a repo file would execute PR-controlled code with `GITHUB_TOKEN` and could bypass its own identity checks. - Harden planner schema validation: reject `.`/`..` segments in test-matrix file paths, and enforce id uniqueness across `matrix` and `testMatrix` combined. - Cover the `--ci-output` CLI handoff with a subprocess regression driving the real CLI through `JOBS`/`TARGETS`, plus a mixed-selector case that exits nonzero without writing outputs. - Drop the report-to-pr planner-schema test and its fake-`npx` helper. It stubbed planner stdout so the workflow shell could validate it with `jq`; the planner now validates its own output, and the equivalent cases live in `workflow-plan.test.ts`. ## Type of Change - [x] Code change (feature, bug fix, or refactor) - [ ] Code change with doc updates - [ ] Doc only (prose changes, no code sample modifications) - [ ] Doc only (includes code sample changes) ## Quality Gates - [x] Tests added or updated for changed behavior - [ ] Existing tests cover changed behavior — justification: - [ ] Tests not applicable — justification: - [ ] Docs updated for user-facing behavior changes - [x] Docs not applicable — justification: internal GitHub Actions workflow and E2E planner refactor; no user-facing behavior changed. - [x] Sensitive paths changed (security, policy, credentials, preflight, onboarding, inference, runner, sandbox, or messaging) - [ ] Sensitive-path review completed or maintainer-approved waiver recorded — reviewer/approval link/justification: - [ ] Non-success, skipped, or missing CI check accepted by maintainer — check name, approval link, and follow-up issue: `E2E / PR Gate` — fork PRs cannot dispatch credentialed E2E, so the gate times out waiting for the trusted verdict; needs a maintainer dispatch or an approved skip. ## Verification - [x] PR description includes a `Signed-off-by:` line and every commit appears as `Verified` in GitHub - [x] Normal `pre-commit`, `commit-msg`, and `pre-push` hooks passed, or `npm run check:diff` passed when hooks were skipped or unavailable - [x] Targeted behavior tests pass for the current change set, or tests are marked not applicable above — `npx vitest run --project e2e-support test/e2e/support/workflow-plan.test.ts test/e2e/support/e2e-workflow.test.ts test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts` (84 passed); `npm run typecheck:cli` passed. - [ ] Applicable broad gate passed — `npm test` for broad runtime/test-harness changes; `npm run check` for repo-wide validation/coverage changes — command/result: - [x] Quality Gates section completed with required justifications or waivers - [x] No secrets, API keys, or credentials committed - [ ] `npm run docs` builds without warnings (doc changes only) - [ ] Doc pages follow the [style guide](https://github.com/NVIDIA/NemoClaw/blob/main/docs/CONTRIBUTING.md) (doc changes only) - [ ] New doc pages include SPDX header and frontmatter (new pages only) --- Signed-off-by: harjoth <[email protected]> Signed-off-by: harjoth <[email protected]> Signed-off-by: Julie Yaunches <[email protected]> Co-authored-by: Julie Yaunches <[email protected]>
1 parent 54fd778 commit e7e8b67

8 files changed

Lines changed: 388 additions & 185 deletions

‎.github/workflows/e2e.yaml‎

Lines changed: 3 additions & 95 deletions
Original file line numberDiff line numberDiff line change
@@ -173,109 +173,17 @@ jobs:
173173
TARGETS: ${{ inputs.targets }}
174174
run: |
175175
set -euo pipefail
176-
case "${INFERENCE_MODE}" in
177-
mock|internal-nvidia|public-nvidia) ;;
178-
*) echo "::error::Invalid inference_mode: ${INFERENCE_MODE}" >&2; exit 1 ;;
179-
esac
180-
for selector_name in JOBS TARGETS; do
181-
selector_value="${!selector_name}"
182-
if [ -n "${selector_value}" ] && [[ ! "${selector_value}" =~ ^[A-Za-z0-9_-]+(,[A-Za-z0-9_-]+)*$ ]]; then
183-
echo "::error::Invalid ${selector_name,,} input; use comma-separated ids" >&2
184-
exit 1
185-
fi
186-
done
187-
188-
planner_args=()
189-
if [ -n "${JOBS}" ]; then
190-
planner_args+=(--jobs "${JOBS}")
191-
fi
192-
if [ -n "${TARGETS}" ]; then
193-
planner_args+=(--targets "${TARGETS}")
194-
fi
195-
plan="$(npx tsx tools/e2e/workflow-plan.mts "${planner_args[@]}")"
196-
plan_filter='
197-
type == "object" and
198-
(keys | sort) == ["explicitOnlyJobs", "hermesSelected", "matrix", "testMatrix"] and
199-
(.matrix | type) == "array" and
200-
all(.matrix[];
201-
type == "object" and
202-
(keys | sort) == ["expectedStateId", "id", "install", "label", "onboarding", "pendingRuntimeSuites", "platform", "requiredSecrets", "runner", "runtime", "suites", "supportReasons", "supported"] and
203-
(.id | type == "string" and test("^[a-z0-9]+(?:-[a-z0-9]+)*$")) and
204-
(.runner | type == "string" and test("^[A-Za-z0-9_-]+$")) and
205-
(.label | type) == "string" and
206-
(.platform | type) == "string" and
207-
(.install | type) == "string" and
208-
(.runtime | type) == "string" and
209-
(.onboarding | type) == "string" and
210-
(.expectedStateId | type) == "string" and
211-
(.supported | type) == "boolean" and
212-
all(.suites[], .requiredSecrets[], .supportReasons[], .pendingRuntimeSuites[]; type == "string")
213-
) and
214-
(([.matrix[].id] | length) == ([.matrix[].id] | unique | length)) and
215-
(.testMatrix | type) == "array" and
216-
all(.testMatrix[];
217-
type == "object" and
218-
(keys | sort) == ["file", "id", "project"] and
219-
(.id | test("^[a-z0-9]+(?:-[a-z0-9]+)*$")) and
220-
(.file | test("^test/(?:[A-Za-z0-9._-]+/)*[A-Za-z0-9._-]+[.]test[.](?:js|ts)$")) and
221-
((.project == "e2e-live" and (.file | startswith("test/e2e/live/"))) or
222-
(.project == "integration" and (.file | startswith("test/")) and
223-
((.file | startswith("test/e2e/")) | not)))
224-
) and
225-
(([.testMatrix[].id] | length) == ([.testMatrix[].id] | unique | length)) and
226-
(.hermesSelected | type) == "boolean" and
227-
(.explicitOnlyJobs | type) == "array" and
228-
all(.explicitOnlyJobs[]; type == "string" and test("^[A-Za-z0-9_-]+$")) and
229-
((.explicitOnlyJobs | length) == (.explicitOnlyJobs | unique | length))
230-
'
231-
if ! jq -e "${plan_filter}" <<< "${plan}" >/dev/null; then
232-
echo "::error::E2E planner returned an invalid output schema" >&2
233-
exit 1
234-
fi
235-
236-
expected_hermes_selected=false
237-
if { [ -z "${JOBS}" ] && [ -z "${TARGETS}" ]; } \
238-
|| [[ ",${JOBS}," == *",hermes-e2e,"* ]] \
239-
|| [[ ",${TARGETS}," == *",hermes-e2e,"* ]]; then
240-
expected_hermes_selected=true
241-
fi
242-
hermes_selected="$(jq -r '.hermesSelected' <<< "${plan}")"
243-
if [ "${hermes_selected}" != "${expected_hermes_selected}" ]; then
244-
echo "::error::E2E planner changed the trusted Hermes selection" >&2
245-
exit 1
246-
fi
247-
248-
matrix="$(jq -c '.matrix' <<< "${plan}")"
249-
test_matrix="$(jq -c '.testMatrix' <<< "${plan}")"
250-
explicit_only_jobs_csv="$(jq -r '.explicitOnlyJobs | join(",")' <<< "${plan}")"
176+
npx tsx tools/e2e/workflow-plan.mts --ci-output
251177
if [ -n "${CHECKOUT_SHA}" ]; then
178+
matrix="$(sed -n 's/^matrix=//p' "${GITHUB_OUTPUT}")"
252179
expected_controller_matrix="$(jq -c 'map({id, runner}) | sort_by(.id)' <<< "${CONTROLLER_MATRIX}")"
253180
actual_controller_matrix="$(jq -c 'map({id, runner}) | sort_by(.id)' <<< "${matrix}")"
254181
if [ "${actual_controller_matrix}" != "${expected_controller_matrix}" ]; then
182+
: > "${GITHUB_OUTPUT}"
255183
echo "::error::E2E planner matrix does not match controller-selected targets" >&2
256184
exit 1
257185
fi
258186
fi
259-
echo "matrix=${matrix}" >> "$GITHUB_OUTPUT"
260-
echo "test_matrix=${test_matrix}" >> "$GITHUB_OUTPUT"
261-
echo "hermes_selected=${hermes_selected}" >> "$GITHUB_OUTPUT"
262-
echo "explicit_only_jobs=${explicit_only_jobs_csv}" >> "$GITHUB_OUTPUT"
263-
264-
MATRIX_JSON="${matrix}" TEST_MATRIX_JSON="${test_matrix}" python3 - <<'PY' >> "$GITHUB_STEP_SUMMARY"
265-
import json
266-
import os
267-
268-
typed_rows = json.loads(os.environ["MATRIX_JSON"])
269-
test_rows = json.loads(os.environ["TEST_MATRIX_JSON"])
270-
print("## E2E Execution Plan")
271-
print()
272-
print("| Test | Execution | Runner |")
273-
print("| --- | --- | --- |")
274-
for row in typed_rows:
275-
print(f"| `{row['id']}` | typed registry | `{row['runner']}` |")
276-
for row in test_rows:
277-
print(f"| `{row['id']}` | shared E2E job | `ubuntu-latest` |")
278-
PY
279187
280188
live:
281189
needs: generate-matrix

‎test/e2e/support/e2e-operations-workflow-boundary.test.ts‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,33 @@ describe("E2E operations workflow boundary", () => {
102102
);
103103
});
104104

105+
it("binds controller dispatch to the exact checkout, plan, and correlation identity (#6955)", () => {
106+
const workflow = readE2eOperationsWorkflow();
107+
const validation = workflow.jobs["generate-matrix"].steps!.find(
108+
(step) => step.name === "Validate controller dispatch",
109+
)!;
110+
validation.run = validation
111+
.run!.replace('[[ "$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$ ]]', '[[ -n "$CHECKOUT_SHA" ]]')
112+
.replace('[[ "$PLAN_HASH" =~ ^[a-f0-9]{64}$ ]]', '[[ -n "$PLAN_HASH" ]]')
113+
.replace(
114+
'[[ "$CORRELATION_ID" =~ ^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$ ]]',
115+
'[[ -n "$CORRELATION_ID" ]]',
116+
)
117+
.replace(
118+
`[[ "$(jq -r '.head.sha' <<< "$pull_json")" == "$CHECKOUT_SHA" ]]`,
119+
`[[ -n "$(jq -r '.head.sha' <<< "$pull_json")" ]]`,
120+
);
121+
122+
expect(validateE2eOperationsWorkflow(workflow)).toEqual(
123+
expect.arrayContaining([
124+
'Controller validation must retain "$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$',
125+
'Controller validation must retain "$PLAN_HASH" =~ ^[a-f0-9]{64}$',
126+
'Controller validation must retain "$CORRELATION_ID" =~ ^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$',
127+
`Controller validation must retain [[ "$(jq -r '.head.sha' <<< "$pull_json")" == "$CHECKOUT_SHA" ]]`,
128+
]),
129+
);
130+
});
131+
105132
it("keeps every planned job wired to bound evidence", () => {
106133
const workflow = readE2eOperationsWorkflow();
107134
const job = workflow.jobs["cloud-onboard"];

‎test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts‎

Lines changed: 5 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -94,13 +94,14 @@ function executeGenerateMatrixWithPlannerOutput(
9494
fakeNpx,
9595
[
9696
"#!/usr/bin/env bash",
97-
"expected=(tsx tools/e2e/workflow-plan.mts)",
98-
'[[ -z "${JOBS:-}" ]] || expected+=(--jobs "$JOBS")',
99-
'[[ -z "${TARGETS:-}" ]] || expected+=(--targets "$TARGETS")',
97+
"expected=(tsx tools/e2e/workflow-plan.mts --ci-output)",
10098
'actual=("$@")',
10199
'[[ "${#actual[@]}" -eq "${#expected[@]}" ]] || exit 97',
102100
'for index in "${!expected[@]}"; do [[ "${actual[$index]}" == "${expected[$index]}" ]] || exit 97; done',
103-
"printf '%s\\n' \"${FAKE_E2E_PLAN}\"",
101+
'printf "matrix=%s\\n" "$(jq -c .matrix <<< "${FAKE_E2E_PLAN}")" >> "${GITHUB_OUTPUT}"',
102+
'printf "test_matrix=%s\\n" "$(jq -c .testMatrix <<< "${FAKE_E2E_PLAN}")" >> "${GITHUB_OUTPUT}"',
103+
'printf "hermes_selected=%s\\n" "$(jq -r .hermesSelected <<< "${FAKE_E2E_PLAN}")" >> "${GITHUB_OUTPUT}"',
104+
'printf "explicit_only_jobs=%s\\n" "$(jq -r ".explicitOnlyJobs | join(\\\",\\\")" <<< "${FAKE_E2E_PLAN}")" >> "${GITHUB_OUTPUT}"',
104105
"",
105106
].join("\n"),
106107
{ mode: 0o755 },
@@ -867,33 +868,6 @@ it("carries the generated planner matrix through the workflow output and PR repo
867868
}
868869
});
869870

870-
it("fails closed when planner output violates the workflow schema", () => {
871-
const [selected] = discoverCredentialFreeTests();
872-
expect(selected).toBeDefined();
873-
const validPlan = buildE2eWorkflowPlan();
874-
const [registryRow] = validPlan.matrix;
875-
expect(registryRow).toBeDefined();
876-
const { explicitOnlyJobs: _omitted, ...missingField } = validPlan;
877-
const malformedPlans = [
878-
["missing required field", missingField],
879-
["duplicate matrix id", { ...validPlan, matrix: [...validPlan.matrix, { ...registryRow }] }],
880-
["invalid test ID", { ...validPlan, testMatrix: [{ ...selected, id: "invalid_id" }] }],
881-
["nonboolean selection", { ...validPlan, hermesSelected: "false" }],
882-
] as const;
883-
884-
for (const [label, plan] of malformedPlans) {
885-
const generated = executeGenerateMatrixWithPlannerOutput(plan);
886-
expect(
887-
generated.result.status,
888-
`${label}: ${generated.result.stderr || generated.result.stdout}`,
889-
).toBe(1);
890-
expect(generated.result.stderr).toContain(
891-
"::error::E2E planner returned an invalid output schema",
892-
);
893-
expect(generated.workflowOutput).toBe("");
894-
}
895-
});
896-
897871
it("builds controller target matrices only from trusted runner mappings (#7031)", () => {
898872
const target = "ubuntu-repo-cloud-langchain-deepagents-code";
899873

‎test/e2e/support/e2e-workflow.test.ts‎

Lines changed: 13 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import { buildE2eWorkflowPlan } from "../../../tools/e2e/workflow-plan.mts";
1919
import { readWorkflow, removeJobNeed } from "../../helpers/e2e-workflow-contract";
2020
import { testTimeoutOptions } from "../../helpers/timeouts";
2121
import { assertChannelsStopStartSandboxName } from "../live/channels-stop-start-safety.ts";
22+
import { requireFixture } from "./require-fixture";
2223

2324
describe("e2e workflow boundary", () => {
2425
it("guards channels-stop-start destructive cleanup to test-owned sandboxes", () => {
@@ -83,7 +84,7 @@ describe("e2e workflow boundary", () => {
8384
}
8485
});
8586

86-
it("requires unknown inference modes to be rejected before planning", () => {
87+
it("requires matrix generation to use the planner CI-output mode", () => {
8788
const workflow = readWorkflow() as {
8889
jobs: Record<string, { steps?: Array<{ name?: string; run?: string }> }>;
8990
};
@@ -95,13 +96,13 @@ describe("e2e workflow boundary", () => {
9596
(() => {
9697
throw new Error("workflow missing Generate E2E target matrix script");
9798
})();
98-
generate!.run = generateRun.replace(
99-
"Invalid inference_mode: ${INFERENCE_MODE}",
100-
"Unsupported inference mode",
101-
);
99+
requireFixture(generateRun.includes("--ci-output"), "planner fixture missing --ci-output");
100+
const invalidRun = generateRun.replace("--ci-output", "--plain-output");
101+
requireFixture(invalidRun !== generateRun, "planner fixture mutation did not apply");
102+
generate!.run = invalidRun;
102103

103104
expect(validateE2eWorkflow(workflow)).toContain(
104-
"step 'Generate E2E target matrix' run script must include Invalid inference_mode: ${INFERENCE_MODE}",
105+
"step 'Generate E2E target matrix' run script must include --ci-output",
105106
);
106107
});
107108

@@ -1180,27 +1181,23 @@ jobs:
11801181
}
11811182
});
11821183

1183-
it("rejects raw jobs selector echo from matrix generation", () => {
1184+
it("rejects matrix generation that bypasses the planner CI-output mode", () => {
11841185
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-workflow-"));
11851186
const workflowPath = path.join(tmp, "workflow.yaml");
11861187
const workflow = fs.readFileSync(
11871188
path.join(process.cwd(), ".github/workflows/e2e.yaml"),
11881189
"utf8",
11891190
);
1190-
fs.writeFileSync(
1191-
workflowPath,
1192-
workflow.replace(
1193-
'echo "::error::Invalid ${selector_name,,} input; use comma-separated ids" >&2',
1194-
'echo "::error::Invalid jobs input: ${JOBS}" >&2',
1195-
),
1196-
);
1191+
requireFixture(workflow.includes("--ci-output"), "workflow fixture missing --ci-output");
1192+
const invalidWorkflow = workflow.replace("--ci-output", "--plain-output");
1193+
requireFixture(invalidWorkflow !== workflow, "workflow fixture mutation did not apply");
1194+
fs.writeFileSync(workflowPath, invalidWorkflow);
11971195

11981196
try {
11991197
const errors = validateE2eWorkflowBoundary(workflowPath);
12001198
expect(errors).toEqual(
12011199
expect.arrayContaining([
1202-
"step 'Generate E2E target matrix' run script must include Invalid ${selector_name,,} input; use comma-separated ids",
1203-
"step 'Generate E2E target matrix' run script must not include Invalid jobs input: ${JOBS}",
1200+
"step 'Generate E2E target matrix' run script must include --ci-output",
12041201
]),
12051202
);
12061203
} finally {

0 commit comments

Comments
 (0)