Skip to content

Commit 70fabaa

Browse files
committed
fix: improve gateway lifecycle recovery
1 parent 0acb2e4 commit 70fabaa

7 files changed

Lines changed: 1376 additions & 144 deletions

File tree

‎bin/lib/onboard.js‎

Lines changed: 85 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,38 @@ function hasStaleGateway(gwInfoOutput) {
185185
return typeof gwInfoOutput === "string" && gwInfoOutput.length > 0 && gwInfoOutput.includes(GATEWAY_NAME);
186186
}
187187

188+
function stripAnsi(value = "") {
189+
let cleaned = "";
190+
for (let i = 0; i < value.length; i += 1) {
191+
if (value.charCodeAt(i) === 27 && value[i + 1] === "[") {
192+
i += 2;
193+
while (i < value.length && /[0-9;]/.test(value[i])) {
194+
i += 1;
195+
}
196+
if (value[i] === "m") {
197+
continue;
198+
}
199+
}
200+
cleaned += value[i] || "";
201+
}
202+
return cleaned;
203+
}
204+
205+
function getActiveGatewayName(statusOutput = "") {
206+
if (typeof statusOutput !== "string" || statusOutput.length === 0) {
207+
return "";
208+
}
209+
const match = stripAnsi(statusOutput)
210+
.match(/^\s*Gateway:\s+(.+?)\s*$/m);
211+
return match ? match[1].trim() : "";
212+
}
213+
214+
function isGatewayHealthy(statusOutput = "", gwInfoOutput = "") {
215+
const connected = typeof statusOutput === "string" && statusOutput.includes("Connected");
216+
const activeGateway = getActiveGatewayName(statusOutput);
217+
return connected && activeGateway === GATEWAY_NAME && hasStaleGateway(gwInfoOutput);
218+
}
219+
188220
function streamSandboxCreate(command, env = process.env, options = {}) {
189221
const child = spawn("bash", ["-lc", command], {
190222
cwd: ROOT,
@@ -1237,8 +1269,16 @@ async function preflight() {
12371269
// A previous onboard run may have left the gateway container and port
12381270
// forward running. If a NemoClaw-owned gateway is still present, tear
12391271
// it down so the port check below doesn't fail on our own leftovers.
1272+
const gatewayStatus = runCaptureOpenshell(["status"], { ignoreError: true });
12401273
const gwInfo = runCaptureOpenshell(["gateway", "info", "-g", GATEWAY_NAME], { ignoreError: true });
1241-
if (hasStaleGateway(gwInfo)) {
1274+
const healthyGateway = isGatewayHealthy(gatewayStatus, gwInfo);
1275+
if (healthyGateway) {
1276+
console.log(" Reusing existing NemoClaw gateway...");
1277+
runOpenshell(["forward", "stop", "18789"], { ignoreError: true });
1278+
runOpenshell(["gateway", "select", GATEWAY_NAME], { ignoreError: true });
1279+
process.env.OPENSHELL_GATEWAY = GATEWAY_NAME;
1280+
console.log(" ✓ Existing gateway selected");
1281+
} else if (hasStaleGateway(gwInfo)) {
12421282
console.log(" Cleaning up previous NemoClaw session...");
12431283
runOpenshell(["forward", "stop", "18789"], { ignoreError: true });
12441284
runOpenshell(["gateway", "destroy", "-g", GATEWAY_NAME], { ignoreError: true });
@@ -1251,6 +1291,10 @@ async function preflight() {
12511291
{ port: 18789, label: "NemoClaw dashboard" },
12521292
];
12531293
for (const { port, label } of requiredPorts) {
1294+
if (port === 8080 && healthyGateway) {
1295+
console.log(` ✓ Port ${port} already in use by active NemoClaw gateway (${label})`);
1296+
continue;
1297+
}
12541298
const portCheck = await checkPortAvailable(port);
12551299
if (!portCheck.ok) {
12561300
console.error("");
@@ -1305,11 +1349,21 @@ function destroyGateway() {
13051349

13061350
// ── Step 2: Gateway ──────────────────────────────────────────────
13071351

1308-
async function startGateway(_gpu) {
1352+
async function startGatewayWithOptions(_gpu, { exitOnFailure = true } = {}) {
13091353
step(3, 7, "Starting OpenShell gateway");
13101354

1311-
// Clean up any previous gateway and its Docker volumes
1312-
destroyGateway();
1355+
const gatewayStatus = runCaptureOpenshell(["status"], { ignoreError: true });
1356+
const gwInfo = runCaptureOpenshell(["gateway", "info", "-g", GATEWAY_NAME], { ignoreError: true });
1357+
if (isGatewayHealthy(gatewayStatus, gwInfo)) {
1358+
console.log(" ✓ Reusing existing gateway");
1359+
runOpenshell(["gateway", "select", GATEWAY_NAME], { ignoreError: true });
1360+
process.env.OPENSHELL_GATEWAY = GATEWAY_NAME;
1361+
return;
1362+
}
1363+
1364+
if (hasStaleGateway(gwInfo)) {
1365+
runOpenshell(["gateway", "destroy", "-g", GATEWAY_NAME], { ignoreError: true });
1366+
}
13131367

13141368
const gwArgs = ["--name", GATEWAY_NAME];
13151369
// Do NOT pass --gpu here. On DGX Spark (and most GPU hosts), inference is
@@ -1332,22 +1386,29 @@ async function startGateway(_gpu) {
13321386
if (startResult.status !== 0) {
13331387
console.error(" Gateway failed to start. Cleaning up stale state...");
13341388
destroyGateway();
1335-
console.error(" Stale state removed. Please rerun: nemoclaw onboard");
1336-
process.exit(1);
1389+
if (exitOnFailure) {
1390+
console.error(" Stale state removed. Please rerun: nemoclaw onboard");
1391+
process.exit(1);
1392+
}
1393+
throw new Error("Gateway failed to start");
13371394
}
13381395

13391396
// Verify health
13401397
for (let i = 0; i < 5; i++) {
13411398
const status = runCaptureOpenshell(["status"], { ignoreError: true });
1342-
if (status.includes("Connected")) {
1399+
const gwInfo = runCaptureOpenshell(["gateway", "info", "-g", GATEWAY_NAME], { ignoreError: true });
1400+
if (isGatewayHealthy(status, gwInfo)) {
13431401
console.log(" ✓ Gateway is healthy");
13441402
break;
13451403
}
13461404
if (i === 4) {
13471405
console.error(" Gateway health check failed. Cleaning up stale state...");
13481406
destroyGateway();
1349-
console.error(" Stale state removed. Please rerun: nemoclaw onboard");
1350-
process.exit(1);
1407+
if (exitOnFailure) {
1408+
console.error(" Stale state removed. Please rerun: nemoclaw onboard");
1409+
process.exit(1);
1410+
}
1411+
throw new Error("Gateway failed to start");
13511412
}
13521413
sleep(2);
13531414
}
@@ -1364,6 +1425,14 @@ async function startGateway(_gpu) {
13641425
process.env.OPENSHELL_GATEWAY = GATEWAY_NAME;
13651426
}
13661427

1428+
async function startGateway(_gpu) {
1429+
return startGatewayWithOptions(_gpu, { exitOnFailure: true });
1430+
}
1431+
1432+
async function startGatewayForRecovery(_gpu) {
1433+
return startGatewayWithOptions(_gpu, { exitOnFailure: false });
1434+
}
1435+
13671436
// ── Step 3: Sandbox ──────────────────────────────────────────────
13681437

13691438
async function createSandbox(gpu, model, provider, preferredInferenceApi = null) {
@@ -1415,6 +1484,8 @@ async function createSandbox(gpu, model, provider, preferredInferenceApi = null)
14151484
run(`cp -r "${path.join(ROOT, "nemoclaw-blueprint")}" "${buildCtx}/nemoclaw-blueprint"`);
14161485
run(`cp -r "${path.join(ROOT, "scripts")}" "${buildCtx}/scripts"`);
14171486
run(`rm -rf "${buildCtx}/nemoclaw/node_modules"`, { ignoreError: true });
1487+
run(`rm -rf "${buildCtx}/nemoclaw-blueprint/.venv" "${buildCtx}/nemoclaw-blueprint/.pytest_cache"`, { ignoreError: true });
1488+
run(`find "${buildCtx}/nemoclaw-blueprint" -type d -name __pycache__ -prune -exec rm -rf {} +`, { ignoreError: true });
14181489

14191490
// Create sandbox (use -- echo to avoid dropping into interactive shell)
14201491
// Pass the base policy so sandbox starts in proxy mode (required for policy updates later)
@@ -1551,9 +1622,7 @@ async function setupNim(gpu) {
15511622
const options = [];
15521623
options.push({
15531624
key: "build",
1554-
label:
1555-
"NVIDIA Endpoints" +
1556-
(!ollamaRunning && !(EXPERIMENTAL && vllmRunning) ? " (recommended)" : ""),
1625+
label: "NVIDIA Endpoints",
15571626
});
15581627
options.push({ key: "openai", label: "OpenAI" });
15591628
options.push({ key: "custom", label: "Other OpenAI-compatible endpoint" });
@@ -2297,12 +2366,16 @@ module.exports = {
22972366
getInstalledOpenshellVersion,
22982367
getStableGatewayImageRef,
22992368
hasStaleGateway,
2369+
isGatewayHealthy,
23002370
isSandboxReady,
23012371
onboard,
2372+
preflight,
23022373
pruneStaleSandboxEntry,
23032374
runCaptureOpenshell,
23042375
setupInference,
23052376
setupNim,
2377+
startGateway,
2378+
startGatewayForRecovery,
23062379
writeSandboxConfigSyncFile,
23072380
patchStagedDockerfile,
23082381
};

0 commit comments

Comments
 (0)