Skip to content

Commit 82fc0cd

Browse files
committed
feat(sdk): route TypeScript SDK exec and forward to the sandbox owner replica
Ask GetSandbox for the owner replica and send it as x-openshell-replica on ExecSandbox, ExecSandboxInteractive and ForwardTcp. Retry once without the hint when the per-replica route has no healthy endpoint, holding stream input until response headers arrive so the retry only resends the first frame. Part of NVIDIA#4223 Signed-off-by: divesh <[email protected]>
1 parent 42671c2 commit 82fc0cd

3 files changed

Lines changed: 289 additions & 21 deletions

File tree

‎sdk/typescript/src/client.ts‎

Lines changed: 62 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ import {
4040
import type { EffectiveSetting, GetSandboxConfigResponse, SandboxPolicy, SettingValue } from './gen/sandbox_pb.js';
4141
import { PolicySource, type SandboxPolicySchema, SettingScope, type SettingValueSchema } from './gen/sandbox_pb.js';
4242
import { ProviderClient } from './providers.js';
43+
import { heldUntilReady, OWNER_REPLICA_HEADER, OWNER_REQUEST_HEADER, routedStream } from './routing.js';
4344
import { validateSshResponse } from './ssh-validate.js';
4445
import { buildTransport, type ConnectOptions } from './transport.js';
4546

@@ -1282,6 +1283,29 @@ export class SandboxClient {
12821283
}
12831284
}
12841285

1286+
// get() that also asks the gateway which replica owns the supervisor session.
1287+
private async getWithOwner(
1288+
name: string,
1289+
options?: SandboxCallOptions | null,
1290+
): Promise<{ ref: SandboxRef; owner?: string }> {
1291+
let owner: string | undefined;
1292+
try {
1293+
const resp = await this.grpc.getSandbox(
1294+
{ ...namedTarget(name, options) },
1295+
{
1296+
...requestCallOptions(options),
1297+
headers: { [OWNER_REQUEST_HEADER]: '1' },
1298+
onHeader: (header) => {
1299+
owner = header.get(OWNER_REPLICA_HEADER) || undefined;
1300+
},
1301+
},
1302+
);
1303+
return { ref: sandboxRef(resp.sandbox), owner };
1304+
} catch (e) {
1305+
throw fromConnect(e);
1306+
}
1307+
}
1308+
12851309
list(options?: ListOptions | null): Pager<SandboxRef> {
12861310
return new Pager(async (pageToken) => {
12871311
try {
@@ -1385,22 +1409,22 @@ export class SandboxClient {
13851409
): AsyncGenerator<ExecStreamEvent, void, void> {
13861410
try {
13871411
// Preserve the existing preflight so lookup failures surface before the stream starts.
1388-
await this.get(name, {
1412+
const { owner } = await this.getWithOwner(name, {
13891413
workspace: options?.workspace,
13901414
...(options?.signal ? { signal: options.signal } : {}),
13911415
});
1392-
const stream = this.grpc.execSandbox(
1393-
{
1394-
...sandboxTarget(name, options),
1395-
command,
1396-
workdir: options?.workdir ?? '',
1397-
environment: options?.environment ?? {},
1398-
executionTimeout: durationFromSeconds(options?.timeoutSecs ?? 0),
1399-
stdin: options?.stdin ? new Uint8Array(options.stdin) : new Uint8Array(),
1400-
tty: false,
1401-
noLoginShell: options?.noLoginShell ?? false,
1402-
},
1403-
{ signal: options?.signal },
1416+
const request = {
1417+
...sandboxTarget(name, options),
1418+
command,
1419+
workdir: options?.workdir ?? '',
1420+
environment: options?.environment ?? {},
1421+
executionTimeout: durationFromSeconds(options?.timeoutSecs ?? 0),
1422+
stdin: options?.stdin ? new Uint8Array(options.stdin) : new Uint8Array(),
1423+
tty: false,
1424+
noLoginShell: options?.noLoginShell ?? false,
1425+
};
1426+
const stream = routedStream(owner, ({ headers, onHeader }) =>
1427+
this.grpc.execSandbox(request, { signal: options?.signal, headers, onHeader }),
14041428
);
14051429

14061430
let sawExit = false;
@@ -1541,14 +1565,18 @@ export class SandboxClient {
15411565
command: string[],
15421566
options?: ExecInteractiveOptions | null,
15431567
): Promise<ExecInteractiveSessionControl> {
1568+
let owner: string | undefined;
15441569
try {
1545-
await this.get(name, { workspace: options?.workspace, ...(options?.signal ? { signal: options.signal } : {}) });
1570+
({ owner } = await this.getWithOwner(name, {
1571+
workspace: options?.workspace,
1572+
...(options?.signal ? { signal: options.signal } : {}),
1573+
}));
15461574
} catch (e) {
15471575
throw e instanceof SdkError ? e : fromConnect(e);
15481576
}
15491577

15501578
const input = new Pushable<MessageInitShape<typeof ExecSandboxInputSchema>>();
1551-
input.push({
1579+
const start: MessageInitShape<typeof ExecSandboxInputSchema> = {
15521580
payload: {
15531581
case: 'start',
15541582
value: {
@@ -1564,7 +1592,7 @@ export class SandboxClient {
15641592
noLoginShell: options?.noLoginShell ?? false,
15651593
},
15661594
},
1567-
});
1595+
};
15681596

15691597
const controller = new AbortController();
15701598
const signal = options?.signal ? AbortSignal.any([options.signal, controller.signal]) : controller.signal;
@@ -1618,7 +1646,9 @@ export class SandboxClient {
16181646
if (signal.aborted) throw new SdkError('canceled', 'exec cancelled');
16191647
// Start and observe the transport immediately, independently of output
16201648
// consumption. Backpressure bounds the queue to 16 chunks of 64 KiB.
1621-
const stream = grpc.execSandboxInteractive(input, { signal });
1649+
const stream = routedStream(owner, ({ headers, onHeader, ready }) =>
1650+
grpc.execSandboxInteractive(heldUntilReady(start, input, ready), { signal, headers, onHeader }),
1651+
);
16221652
for await (const event of stream) {
16231653
if (signal.aborted) throw new SdkError('canceled', 'exec cancelled');
16241654
if (exitCode !== undefined) {
@@ -1713,12 +1743,18 @@ export class SandboxClient {
17131743
const localPort = opts.localPort ?? 0;
17141744

17151745
let sandboxId: string;
1746+
let owner: string | undefined;
17161747
try {
1717-
const ref = await this.get(name, { workspace: opts.workspace, ...(opts.signal ? { signal: opts.signal } : {}) });
1748+
const found = await this.getWithOwner(name, {
1749+
workspace: opts.workspace,
1750+
...(opts.signal ? { signal: opts.signal } : {}),
1751+
});
1752+
const ref = found.ref;
17181753
if (ref.phase !== 'ready') {
17191754
throw new SdkError('connect', `sandbox '${name}' is not ready (phase: ${ref.phase})`);
17201755
}
17211756
sandboxId = ref.id;
1757+
owner = found.owner;
17221758
} catch (e) {
17231759
throw e instanceof SdkError ? e : fromConnect(e);
17241760
}
@@ -1744,6 +1780,7 @@ export class SandboxClient {
17441780
opts.workspace,
17451781
targetHost,
17461782
targetPort,
1783+
owner,
17471784
controller.signal,
17481785
)
17491786
.catch((error: unknown) => {
@@ -1828,6 +1865,7 @@ export class SandboxClient {
18281865
workspace: string | undefined,
18291866
targetHost: string,
18301867
targetPort: number,
1868+
owner: string | undefined,
18311869
signal: AbortSignal,
18321870
): Promise<void> {
18331871
let token: string | undefined;
@@ -1840,7 +1878,7 @@ export class SandboxClient {
18401878
// tears down this one socket via the catch below.
18411879
validateSshResponse(session, sandboxId);
18421880
token = session.token;
1843-
input.push({
1881+
const init: MessageInitShape<typeof TcpForwardFrameSchema> = {
18441882
payload: {
18451883
case: 'init',
18461884
value: {
@@ -1853,7 +1891,7 @@ export class SandboxClient {
18531891
authorizationToken: token,
18541892
},
18551893
},
1856-
});
1894+
};
18571895

18581896
socket.on('data', (chunk: Buffer) => {
18591897
for (let off = 0; off < chunk.length; off += FORWARD_CHUNK) {
@@ -1875,7 +1913,10 @@ export class SandboxClient {
18751913
signal.addEventListener('abort', onAbort, { once: true });
18761914

18771915
try {
1878-
for await (const frame of this.grpc.forwardTcp(input, { signal })) {
1916+
const frames = routedStream(owner, ({ headers, onHeader, ready }) =>
1917+
this.grpc.forwardTcp(heldUntilReady(init, input, ready), { signal, headers, onHeader }),
1918+
);
1919+
for await (const frame of frames) {
18791920
if (frame.payload.case !== 'data') continue;
18801921
const data = frame.payload.value;
18811922
if (data.length === 0) continue;

‎sdk/typescript/src/routing.test.ts‎

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
// Replica routing over real gRPC/HTTP2, with a front proxy that answers routed
5+
// calls the way Envoy does when the owner's route has no ready endpoints.
6+
7+
import * as http2 from 'node:http2';
8+
import type { AddressInfo } from 'node:net';
9+
import * as net from 'node:net';
10+
import { Code, ConnectError, type ServiceImpl } from '@connectrpc/connect';
11+
import { connectNodeAdapter, createGrpcTransport } from '@connectrpc/connect-node';
12+
import { afterEach, describe, expect, it } from 'vitest';
13+
import { SandboxClient } from './client.js';
14+
import { OpenShell, SandboxPhase } from './gen/openshell_pb.js';
15+
import { retryUnrouted } from './routing.js';
16+
17+
const enc = (s: string) => new TextEncoder().encode(s);
18+
const closers: Array<() => void> = [];
19+
afterEach(() => {
20+
for (const close of closers.splice(0)) close();
21+
});
22+
23+
interface Gateway {
24+
sandbox: SandboxClient;
25+
wantedOwner: string[];
26+
// Replica header seen by the gateway on each streaming call ('' when unrouted).
27+
routes: string[];
28+
rejectedByProxy: number;
29+
interactiveFrames: string[];
30+
}
31+
32+
async function gateway(opts: { owner?: string; rejectRouted?: boolean; execError?: ConnectError }): Promise<Gateway> {
33+
const g: Gateway = {
34+
sandbox: undefined as never,
35+
wantedOwner: [],
36+
routes: [],
37+
rejectedByProxy: 0,
38+
interactiveFrames: [],
39+
};
40+
const route = (header: Headers): void => {
41+
g.routes.push(header.get('x-openshell-replica') ?? '');
42+
};
43+
const impl: Partial<ServiceImpl<typeof OpenShell>> = {
44+
getSandbox: (_req, ctx) => {
45+
g.wantedOwner.push(ctx.requestHeader.get('x-openshell-want-owner') ?? '');
46+
if (opts.owner) ctx.responseHeader.set('x-openshell-owner', opts.owner);
47+
return { sandbox: { metadata: { id: 'sb-id', name: 'sb' }, status: { phase: SandboxPhase.READY } } };
48+
},
49+
execSandbox: async function* (_req, ctx) {
50+
route(ctx.requestHeader);
51+
if (opts.execError) throw opts.execError;
52+
yield { payload: { case: 'stdout', value: { data: enc('hi') } } };
53+
yield { payload: { case: 'exit', value: { exitCode: 0 } } };
54+
},
55+
// The first (empty) message flushes response headers, like the gateway does.
56+
execSandboxInteractive: async function* (requests, ctx) {
57+
route(ctx.requestHeader);
58+
yield { payload: { case: 'stdout', value: { data: new Uint8Array() } } };
59+
for await (const frame of requests) {
60+
g.interactiveFrames.push(frame.payload.case ?? '');
61+
if (frame.payload.case === 'stdin') {
62+
yield { payload: { case: 'stdout', value: { data: frame.payload.value } } };
63+
yield { payload: { case: 'exit', value: { exitCode: 0 } } };
64+
return;
65+
}
66+
}
67+
},
68+
createSshSession: () => ({
69+
sandboxId: 'sb-id',
70+
token: 'tok',
71+
gatewayHost: 'gw',
72+
gatewayPort: 443,
73+
gatewayScheme: 'https',
74+
}),
75+
revokeSshSession: () => ({ outcome: 1 }),
76+
forwardTcp: async function* (requests, ctx) {
77+
route(ctx.requestHeader);
78+
yield { payload: { case: 'data', value: new Uint8Array() } };
79+
for await (const frame of requests) {
80+
if (frame.payload.case === 'data') yield { payload: { case: 'data', value: frame.payload.value } };
81+
}
82+
},
83+
};
84+
const adapter = connectNodeAdapter({ routes: (router) => router.service(OpenShell, impl) });
85+
const server = http2.createServer((req, res) => {
86+
if (opts.rejectRouted && req.headers['x-openshell-replica']) {
87+
g.rejectedByProxy++;
88+
res.writeHead(200, { 'content-type': 'application/grpc', 'grpc-status': '14' });
89+
res.end();
90+
return;
91+
}
92+
adapter(req, res);
93+
});
94+
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve));
95+
closers.push(() => server.close());
96+
const { port } = server.address() as AddressInfo;
97+
g.sandbox = new SandboxClient(createGrpcTransport({ baseUrl: `http://127.0.0.1:${port}` }));
98+
return g;
99+
}
100+
101+
// Run exec, an interactive exec that writes stdin right away, and one forwarded connection.
102+
async function runAll(g: Gateway): Promise<string[]> {
103+
const exec = await g.sandbox.exec('sb', ['true']);
104+
105+
const session = await g.sandbox.execInteractive('sb', ['cat']);
106+
session.write(Buffer.from('typed'));
107+
let typed = '';
108+
for await (const event of session.output) if (!('type' in event)) typed += event.data.toString();
109+
110+
const handle = await g.sandbox.forward('sb', { targetPort: 9000 });
111+
const echoed = await new Promise<string>((resolve, reject) => {
112+
const socket = net.connect(handle.localPort, handle.localHost, () => socket.write('ping'));
113+
socket.on('data', (d) => {
114+
resolve(d.toString());
115+
socket.end();
116+
});
117+
socket.on('error', reject);
118+
});
119+
await handle.close();
120+
return [exec.stdout.toString(), typed, echoed];
121+
}
122+
123+
describe('replica routing', () => {
124+
it('asks for the owner and routes exec, interactive exec and forward to it', async () => {
125+
const g = await gateway({ owner: 'openshell-1' });
126+
expect(await runAll(g)).toEqual(['hi', 'typed', 'ping']);
127+
expect(g.wantedOwner).toEqual(['1', '1', '1']);
128+
expect(g.routes).toEqual(['openshell-1', 'openshell-1', 'openshell-1']);
129+
});
130+
131+
it('sends no hint when the gateway reports no owner', async () => {
132+
const g = await gateway({});
133+
expect(await runAll(g)).toEqual(['hi', 'typed', 'ping']);
134+
expect(g.routes).toEqual(['', '', '']);
135+
});
136+
137+
it('retries once unrouted when the proxy rejects the owner route, without losing input', async () => {
138+
const g = await gateway({ owner: 'openshell-1', rejectRouted: true });
139+
expect(await runAll(g)).toEqual(['hi', 'typed', 'ping']);
140+
expect(g.rejectedByProxy).toBe(3);
141+
expect(g.routes).toEqual(['', '', '']);
142+
expect(g.interactiveFrames).toEqual(['start', 'stdin']);
143+
});
144+
145+
it('does not retry a gateway error', async () => {
146+
const g = await gateway({
147+
owner: 'openshell-1',
148+
execError: new ConnectError('supervisor not connected', Code.Unavailable),
149+
});
150+
await expect(g.sandbox.exec('sb', ['true'])).rejects.toThrow('supervisor not connected');
151+
expect(g.routes).toEqual(['openshell-1']);
152+
});
153+
154+
it('matches only the proxy no-endpoints answer', () => {
155+
expect(retryUnrouted(new ConnectError('', Code.Unavailable))).toBe(true);
156+
expect(retryUnrouted(new ConnectError('no healthy upstream', Code.Unavailable))).toBe(true);
157+
expect(retryUnrouted(new ConnectError('upstream connect error: connection failure', Code.Unavailable))).toBe(true);
158+
expect(retryUnrouted(new ConnectError('supervisor not connected', Code.Unavailable))).toBe(false);
159+
expect(retryUnrouted(new ConnectError('', Code.NotFound))).toBe(false);
160+
expect(retryUnrouted(new Error(''))).toBe(false);
161+
});
162+
});

0 commit comments

Comments
 (0)