11# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
22# SPDX-License-Identifier: Apache-2.0
33#
4- # Nightly full E2E: install → onboard → live inference against NVIDIA Endpoint API.
4+ # Nightly E2E tests:
5+ #
6+ # cloud-e2e Cloud inference (NVIDIA Endpoint API) on ubuntu-latest.
7+ # cloud-experimental-e2e Experimental cloud inference test.
8+ # gpu-e2e Local Ollama inference on a GPU self-hosted runner.
9+ # Controlled by the GPU_E2E_ENABLED repository variable.
10+ # Set vars.GPU_E2E_ENABLED to "true" in repo settings to enable.
11+ # notify-on-failure Auto-creates a GitHub issue when any E2E job fails.
12+ #
513# Runs directly on the runner (not inside Docker) because OpenShell bootstraps
614# a K3s cluster inside a privileged Docker container — nesting would break networking.
715#
8- # Requires NVIDIA_API_KEY repository secret.
16+ # Requires NVIDIA_API_KEY repository secret (for cloud-e2e and cloud-experimental-e2e) .
917# Only runs on schedule and manual dispatch — never on PRs (secret protection).
1018
1119name : nightly-e2e
@@ -23,15 +31,15 @@ concurrency:
2331 cancel-in-progress : true
2432
2533jobs :
26- full -e2e :
34+ cloud -e2e :
2735 if : github.repository == 'NVIDIA/NemoClaw'
2836 runs-on : ubuntu-latest
2937 timeout-minutes : 45
3038 steps :
3139 - name : Checkout
3240 uses : actions/checkout@v6
3341
34- - name : Run full E2E test
42+ - name : Run cloud E2E test
3543 env :
3644 NVIDIA_API_KEY : ${{ secrets.NVIDIA_API_KEY }}
3745 NEMOCLAW_NON_INTERACTIVE : " 1"
7684 name : install-log-cloud-experimental
7785 path : /tmp/nemoclaw-e2e-install.log
7886 if-no-files-found : ignore
87+
88+ # ── GPU E2E (Ollama local inference) ──────────────────────────
89+ # Enable by setting repository variable GPU_E2E_ENABLED=true
90+ # (Settings → Secrets and variables → Actions → Variables)
91+ #
92+ # Runner labels: using 'self-hosted' for now. Refine to
93+ # [self-hosted, linux, x64, gpu] once NVIDIA runner labels are confirmed.
94+ gpu-e2e :
95+ if : github.repository == 'NVIDIA/NemoClaw' && vars.GPU_E2E_ENABLED == 'true'
96+ runs-on : self-hosted
97+ timeout-minutes : 60
98+ env :
99+ NEMOCLAW_NON_INTERACTIVE : " 1"
100+ NEMOCLAW_SANDBOX_NAME : " e2e-gpu-ollama"
101+ NEMOCLAW_RECREATE_SANDBOX : " 1"
102+ NEMOCLAW_PROVIDER : " ollama"
103+ steps :
104+ - name : Checkout
105+ uses : actions/checkout@v6
106+
107+ - name : Verify GPU availability
108+ run : |
109+ echo "=== GPU Info ==="
110+ nvidia-smi
111+ echo ""
112+ echo "=== VRAM ==="
113+ nvidia-smi --query-gpu=name,memory.total --format=csv,noheader
114+ echo ""
115+ echo "=== Docker ==="
116+ docker info --format '{{.ServerVersion}}'
117+
118+ - name : Run GPU E2E test (Ollama local inference)
119+ run : bash test/e2e/test-gpu-e2e.sh
120+
121+ - name : Upload install log on failure
122+ if : failure()
123+ uses : actions/upload-artifact@v4
124+ with :
125+ name : gpu-e2e-install-log
126+ path : /tmp/nemoclaw-gpu-e2e-install.log
127+ if-no-files-found : ignore
128+
129+ - name : Upload test log on failure
130+ if : failure()
131+ uses : actions/upload-artifact@v4
132+ with :
133+ name : gpu-e2e-test-log
134+ path : /tmp/nemoclaw-gpu-e2e-test.log
135+ if-no-files-found : ignore
136+
137+ notify-on-failure :
138+ runs-on : ubuntu-latest
139+ needs : [cloud-e2e, cloud-experimental-e2e, gpu-e2e]
140+ if : ${{ always() && (needs.cloud-e2e.result == 'failure' || needs.cloud-experimental-e2e.result == 'failure' || needs.gpu-e2e.result == 'failure') }}
141+ permissions :
142+ issues : write
143+ steps :
144+ - name : Create or update failure issue
145+ uses : actions/github-script@v7
146+ with :
147+ script : |
148+ const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
149+ const title = 'Nightly E2E failed';
150+
151+ const { data: existing } = await github.rest.issues.listForRepo({
152+ owner: context.repo.owner,
153+ repo: context.repo.repo,
154+ state: 'open',
155+ labels: 'CI/CD',
156+ per_page: 100,
157+ });
158+ const match = existing.find(i => !i.pull_request && i.title.startsWith(title));
159+
160+ if (match) {
161+ await github.rest.issues.createComment({
162+ owner: context.repo.owner,
163+ repo: context.repo.repo,
164+ issue_number: match.number,
165+ body: `Failed again on ${new Date().toISOString().split('T')[0]}.\n\n**Run:** ${runUrl}\n**Artifacts:** Check the run artifacts for install/test logs (artifact names vary by job).`,
166+ });
167+ } else {
168+ await github.rest.issues.create({
169+ owner: context.repo.owner,
170+ repo: context.repo.repo,
171+ title: `${title} — ${new Date().toISOString().split('T')[0]}`,
172+ body: `The nightly E2E pipeline failed.\n\n**Run:** ${runUrl}\n**Artifacts:** Check the run artifacts for install/test logs (artifact names vary by job).`,
173+ labels: ['bug', 'CI/CD'],
174+ });
175+ }
0 commit comments