Skip to content

Commit d47a7e9

Browse files
committed
refactor: reuse shared gateway startup recovery
1 parent 24878fc commit d47a7e9

3 files changed

Lines changed: 51 additions & 27 deletions

File tree

‎bin/lib/onboard.js‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1313,7 +1313,7 @@ async function preflight() {
13131313

13141314
// ── Step 2: Gateway ──────────────────────────────────────────────
13151315

1316-
async function startGateway(_gpu) {
1316+
async function startGatewayWithOptions(_gpu, { exitOnFailure = true } = {}) {
13171317
step(3, 7, "Starting OpenShell gateway");
13181318

13191319
const gatewayStatus = runCaptureOpenshell(["status"], { ignoreError: true });
@@ -1357,7 +1357,10 @@ async function startGateway(_gpu) {
13571357
}
13581358
if (i === 4) {
13591359
console.error(" Gateway failed to start. Run: openshell gateway info");
1360-
process.exit(1);
1360+
if (exitOnFailure) {
1361+
process.exit(1);
1362+
}
1363+
throw new Error("Gateway failed to start");
13611364
}
13621365
sleep(2);
13631366
}
@@ -1374,6 +1377,14 @@ async function startGateway(_gpu) {
13741377
process.env.OPENSHELL_GATEWAY = GATEWAY_NAME;
13751378
}
13761379

1380+
async function startGateway(_gpu) {
1381+
return startGatewayWithOptions(_gpu, { exitOnFailure: true });
1382+
}
1383+
1384+
async function startGatewayForRecovery(_gpu) {
1385+
return startGatewayWithOptions(_gpu, { exitOnFailure: false });
1386+
}
1387+
13771388
// ── Step 3: Sandbox ──────────────────────────────────────────────
13781389

13791390
async function createSandbox(gpu, model, provider, preferredInferenceApi = null) {
@@ -2316,6 +2327,7 @@ module.exports = {
23162327
setupInference,
23172328
setupNim,
23182329
startGateway,
2330+
startGatewayForRecovery,
23192331
writeSandboxConfigSyncFile,
23202332
patchStagedDockerfile,
23212333
};

‎bin/nemoclaw.js‎

Lines changed: 29 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ const YW = _useColor ? "\x1b[1;33m" : "";
2222

2323
const { ROOT, SCRIPTS, run, runCapture: _runCapture, runInteractive, shellQuote, validateName } = require("./lib/runner");
2424
const { resolveOpenshell } = require("./lib/resolve-openshell");
25+
const { startGatewayForRecovery } = require("./lib/onboard");
2526
const {
2627
ensureApiKey,
2728
ensureGithubToken,
@@ -113,7 +114,7 @@ function getNamedGatewayLifecycleState() {
113114
return { state: "missing_named", status: status.output, gatewayInfo: gatewayInfo.output };
114115
}
115116

116-
function recoverNamedGatewayRuntime() {
117+
async function recoverNamedGatewayRuntime() {
117118
const before = getNamedGatewayLifecycleState();
118119
if (before.state === "healthy_named") {
119120
return { recovered: true, before, after: before, attempted: false };
@@ -126,12 +127,23 @@ function recoverNamedGatewayRuntime() {
126127
return { recovered: true, before, after, attempted: true, via: "select" };
127128
}
128129

129-
runOpenshell(["gateway", "start", "--name", "nemoclaw"], { ignoreError: true });
130-
runOpenshell(["gateway", "select", "nemoclaw"], { ignoreError: true });
131-
after = getNamedGatewayLifecycleState();
132-
if (after.state === "healthy_named") {
133-
process.env.OPENSHELL_GATEWAY = "nemoclaw";
134-
return { recovered: true, before, after, attempted: true, via: "start" };
130+
const shouldStartGateway = [before.state, after.state].some((state) =>
131+
["named_unhealthy", "named_unreachable", "connected_other"].includes(state)
132+
);
133+
134+
if (shouldStartGateway) {
135+
try {
136+
await startGatewayForRecovery();
137+
} catch {
138+
// Fall through to the lifecycle re-check below so we preserve the
139+
// existing recovery result shape and emit the correct classification.
140+
}
141+
runOpenshell(["gateway", "select", "nemoclaw"], { ignoreError: true });
142+
after = getNamedGatewayLifecycleState();
143+
if (after.state === "healthy_named") {
144+
process.env.OPENSHELL_GATEWAY = "nemoclaw";
145+
return { recovered: true, before, after, attempted: true, via: "start" };
146+
}
135147
}
136148

137149
return { recovered: false, before, after, attempted: true };
@@ -183,7 +195,7 @@ function printGatewayLifecycleHint(output = "", sandboxName = "", writer = conso
183195
}
184196
}
185197

186-
function getReconciledSandboxGatewayState(sandboxName) {
198+
async function getReconciledSandboxGatewayState(sandboxName) {
187199
let lookup = getSandboxGatewayState(sandboxName);
188200
if (lookup.state === "present") {
189201
return lookup;
@@ -193,7 +205,7 @@ function getReconciledSandboxGatewayState(sandboxName) {
193205
}
194206

195207
if (lookup.state === "gateway_error") {
196-
const recovery = recoverNamedGatewayRuntime();
208+
const recovery = await recoverNamedGatewayRuntime();
197209
if (recovery.recovered) {
198210
const retried = getSandboxGatewayState(sandboxName);
199211
if (retried.state === "present" || retried.state === "missing") {
@@ -235,8 +247,8 @@ function getReconciledSandboxGatewayState(sandboxName) {
235247
return lookup;
236248
}
237249

238-
function ensureLiveSandboxOrExit(sandboxName) {
239-
const lookup = getReconciledSandboxGatewayState(sandboxName);
250+
async function ensureLiveSandboxOrExit(sandboxName) {
251+
const lookup = await getReconciledSandboxGatewayState(sandboxName);
240252
if (lookup.state === "present") {
241253
return lookup;
242254
}
@@ -539,8 +551,8 @@ function listSandboxes() {
539551

540552
// ── Sandbox-scoped actions ───────────────────────────────────────
541553

542-
function sandboxConnect(sandboxName) {
543-
ensureLiveSandboxOrExit(sandboxName);
554+
async function sandboxConnect(sandboxName) {
555+
await ensureLiveSandboxOrExit(sandboxName);
544556
// Ensure port forward is alive before connecting
545557
runOpenshell(["forward", "start", "--background", "18789", sandboxName], { ignoreError: true });
546558
const result = spawnSync(getOpenshellBinary(), ["sandbox", "connect", sandboxName], {
@@ -551,7 +563,7 @@ function sandboxConnect(sandboxName) {
551563
exitWithSpawnResult(result);
552564
}
553565

554-
function sandboxStatus(sandboxName) {
566+
async function sandboxStatus(sandboxName) {
555567
const sb = registry.getSandbox(sandboxName);
556568
const live = parseGatewayInference(
557569
_runCapture("openshell inference get 2>/dev/null", { ignoreError: true })
@@ -565,7 +577,7 @@ function sandboxStatus(sandboxName) {
565577
console.log(` Policies: ${(sb.policies || []).join(", ") || "none"}`);
566578
}
567579

568-
const lookup = getReconciledSandboxGatewayState(sandboxName);
580+
const lookup = await getReconciledSandboxGatewayState(sandboxName);
569581
if (lookup.state === "present") {
570582
console.log("");
571583
if (lookup.recoveredGateway) {
@@ -778,8 +790,8 @@ const [cmd, ...args] = process.argv.slice(2);
778790
const actionArgs = args.slice(1);
779791

780792
switch (action) {
781-
case "connect": sandboxConnect(cmd); break;
782-
case "status": sandboxStatus(cmd); break;
793+
case "connect": await sandboxConnect(cmd); break;
794+
case "status": await sandboxStatus(cmd); break;
783795
case "logs": sandboxLogs(cmd, actionArgs.includes("--follow")); break;
784796
case "policy-add": await sandboxPolicyAdd(cmd); break;
785797
case "policy-list": sandboxPolicyList(cmd); break;

‎test/cli.test.js‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,11 @@ function run(args) {
1313
return runWithEnv(args);
1414
}
1515

16-
function runWithEnv(args, env = {}) {
16+
function runWithEnv(args, env = {}, timeout = 10000) {
1717
try {
1818
const out = execSync(`node "${CLI}" ${args}`, {
1919
encoding: "utf-8",
20-
timeout: 10000,
20+
timeout,
2121
env: { ...process.env, HOME: "/tmp/nemoclaw-cli-test-" + Date.now(), ...env },
2222
});
2323
return { code: 0, out };
@@ -181,14 +181,14 @@ describe("CLI dispatch", () => {
181181
const r = runWithEnv("alpha status", {
182182
HOME: home,
183183
PATH: `${localBin}:${process.env.PATH || ""}`,
184-
});
184+
}, 25000);
185185

186186
expect(r.code).toBe(0);
187187
expect(r.out.includes("Could not verify sandbox 'alpha'")).toBeTruthy();
188188
expect(r.out.includes("gateway identity drift after restart")).toBeTruthy();
189189
const saved = JSON.parse(fs.readFileSync(path.join(registryDir, "sandboxes.json"), "utf8"));
190190
expect(saved.sandboxes.alpha).toBeTruthy();
191-
});
191+
}, 25000);
192192

193193
it("recovers status after gateway runtime is reattached", () => {
194194
const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-recover-status-"));
@@ -302,7 +302,7 @@ describe("CLI dispatch", () => {
302302
const statusResult = runWithEnv("alpha status", {
303303
HOME: home,
304304
PATH: `${localBin}:${process.env.PATH || ""}`,
305-
});
305+
}, 25000);
306306
expect(statusResult.code).toBe(0);
307307
expect(statusResult.out.includes("gateway trust material rotated after restart")).toBeTruthy();
308308
expect(statusResult.out.includes("cannot be reattached safely")).toBeTruthy();
@@ -375,7 +375,7 @@ describe("CLI dispatch", () => {
375375
const statusResult = runWithEnv("alpha status", {
376376
HOME: home,
377377
PATH: `${localBin}:${process.env.PATH || ""}`,
378-
});
378+
}, 25000);
379379
expect(statusResult.code).toBe(0);
380380
expect(statusResult.out.includes("gateway is still refusing connections after restart")).toBeTruthy();
381381
expect(statusResult.out.includes("Retry `openshell gateway start --name nemoclaw`")).toBeTruthy();
@@ -387,7 +387,7 @@ describe("CLI dispatch", () => {
387387
expect(connectResult.code).toBe(1);
388388
expect(connectResult.out.includes("gateway is still refusing connections after restart")).toBeTruthy();
389389
expect(connectResult.out.includes("If the gateway never becomes healthy")).toBeTruthy();
390-
});
390+
}, 25000);
391391

392392
it("explains when the named gateway is no longer configured after restart or rebuild", () => {
393393
const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-gateway-missing-"));
@@ -446,5 +446,5 @@ describe("CLI dispatch", () => {
446446
expect(statusResult.code).toBe(0);
447447
expect(statusResult.out.includes("gateway is no longer configured after restart/rebuild")).toBeTruthy();
448448
expect(statusResult.out.includes("Start the gateway again")).toBeTruthy();
449-
});
449+
}, 25000);
450450
});

0 commit comments

Comments
 (0)