Skip to content

Commit e57548c

Browse files
committed
feat(sdk): route Python SDK exec to the sandbox owner replica
Ask GetSandbox for the owner replica, cache it for 30 seconds, and send it as x-openshell-replica on ExecSandbox. Retry once without the hint when the per-replica route has no healthy endpoint, forgetting the cached owner. Part of NVIDIA#4223 Signed-off-by: divesh <[email protected]>
1 parent 50a3237 commit e57548c

2 files changed

Lines changed: 204 additions & 4 deletions

File tree

‎python/openshell/sandbox.py‎

Lines changed: 97 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@
2929
openshell_pb2,
3030
openshell_pb2_grpc,
3131
)
32-
from .errors import _error_mapping_channel
32+
from .errors import _error_mapping_channel, from_grpc_error
3333
from .mutations import DeletionOutcome, DeletionResult
3434

3535
_ClientCallDetailsBase = namedtuple(
@@ -40,6 +40,9 @@
4040
_OAUTH_MAX_RESPONSE_BYTES = 1 << 20
4141
_PAGER_MAX_CONSUMED_TOKENS = 10_000
4242
_PAGER_MAX_CONSUMED_TOKEN_BYTES = 1 << 20
43+
_OWNER_REPLICA_HEADER = "x-openshell-owner"
44+
_OWNER_REQUEST_HEADER = "x-openshell-want-owner"
45+
_ROUTE_REPLICA_HEADER = "x-openshell-replica"
4346
T = TypeVar("T")
4447

4548

@@ -99,6 +102,50 @@ def all(self) -> builtins.list[T]:
99102
return [item for page in self for item in page.items]
100103

101104

105+
class _OwnerCache:
106+
"""Owner replicas the gateway last reported, keyed by workspace and name.
107+
108+
An empty owner means the gateway reported none. A stale owner is safe: a
109+
live non-owner replica relays the call.
110+
"""
111+
112+
_TTL_SECONDS = 30.0
113+
_MAX_ENTRIES = 1024
114+
115+
def __init__(self) -> None:
116+
self._lock = threading.Lock()
117+
self._entries: dict[tuple[str, str], tuple[float, str]] = {}
118+
119+
def get(self, workspace: str, name: str) -> str | None:
120+
with self._lock:
121+
entry = self._entries.get((workspace, name))
122+
if entry is None or time.monotonic() - entry[0] >= self._TTL_SECONDS:
123+
return None
124+
return entry[1]
125+
126+
def put(self, workspace: str, name: str, owner: str) -> None:
127+
with self._lock:
128+
if len(self._entries) >= self._MAX_ENTRIES:
129+
self._entries.clear()
130+
self._entries[(workspace, name)] = (time.monotonic(), owner)
131+
132+
def forget(self, workspace: str, name: str) -> None:
133+
with self._lock:
134+
self._entries.pop((workspace, name), None)
135+
136+
137+
def _retry_unrouted(error: grpc.RpcError) -> bool:
138+
"""Envoy Gateway answers a replica route with no ready endpoints with
139+
UNAVAILABLE and no message; gateway errors always carry one."""
140+
status = from_grpc_error(error)
141+
details = status.details() or ""
142+
return status.code() == grpc.StatusCode.UNAVAILABLE and (
143+
details == ""
144+
or "no healthy upstream" in details
145+
or "connection failure" in details
146+
)
147+
148+
102149
def _workspace_scope(workspace: str) -> datamodel_pb2.WorkspaceSelector:
103150
if not workspace:
104151
raise ValueError("workspace must be non-empty")
@@ -730,6 +777,7 @@ def __init__(
730777
self._stub = openshell_pb2_grpc.OpenShellStub(
731778
_error_mapping_channel(self._channel)
732779
)
780+
self._owners = _OwnerCache()
733781

734782
@classmethod
735783
def from_active_cluster(
@@ -1280,7 +1328,7 @@ def exec_stream(
12801328
grpc_deadline = self._timeout
12811329
if timeout_seconds and timeout_seconds + 10 > grpc_deadline:
12821330
grpc_deadline = timeout_seconds + 10
1283-
stream = self._stub.ExecSandbox(request, timeout=grpc_deadline)
1331+
stream = self._open_exec(request, workspace, grpc_deadline)
12841332

12851333
stdout_parts: list[bytes] = []
12861334
stderr_parts: list[bytes] = []
@@ -1308,6 +1356,53 @@ def exec_stream(
13081356
stderr=b"".join(stderr_parts).decode("utf-8", errors="replace"),
13091357
)
13101358

1359+
def _owner_of(self, workspace: str, sandbox: str) -> str:
1360+
"""Best effort: a failed lookup leaves the call unrouted."""
1361+
owner = self._owners.get(workspace, sandbox)
1362+
if owner is not None:
1363+
return owner
1364+
try:
1365+
_, call = self._stub.GetSandbox.with_call(
1366+
openshell_pb2.GetSandboxRequest(
1367+
workspace_scope=_workspace_scope(workspace),
1368+
name=sandbox,
1369+
),
1370+
timeout=self._timeout,
1371+
metadata=((_OWNER_REQUEST_HEADER, "1"),),
1372+
)
1373+
except grpc.RpcError:
1374+
return ""
1375+
owner = dict(call.initial_metadata() or ()).get(_OWNER_REPLICA_HEADER, "")
1376+
self._owners.put(workspace, sandbox, owner)
1377+
return owner
1378+
1379+
def _open_exec(
1380+
self,
1381+
request: openshell_pb2.ExecSandboxRequest,
1382+
workspace: str,
1383+
timeout: float,
1384+
) -> Iterator[openshell_pb2.ExecSandboxEvent]:
1385+
"""Open ExecSandbox on the owner replica, retrying once unrouted."""
1386+
owner = self._owner_of(workspace, request.sandbox)
1387+
if owner:
1388+
events = self._stub.ExecSandbox(
1389+
request,
1390+
timeout=timeout,
1391+
metadata=((_ROUTE_REPLICA_HEADER, owner),),
1392+
)
1393+
try:
1394+
first = next(events, None)
1395+
except grpc.RpcError as error:
1396+
if not _retry_unrouted(error):
1397+
raise
1398+
self._owners.forget(workspace, request.sandbox)
1399+
else:
1400+
if first is not None:
1401+
yield first
1402+
yield from events
1403+
return
1404+
yield from self._stub.ExecSandbox(request, timeout=timeout)
1405+
13111406
def exec(
13121407
self,
13131408
sandbox: str,

‎python/openshell/sandbox_test.py‎

Lines changed: 107 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,22 +3,25 @@
33

44
from __future__ import annotations
55

6+
import contextlib
67
import json
78
import os
89
import pickle
910
import threading
1011
import time
12+
from concurrent import futures
1113
from copy import deepcopy
1214
from dataclasses import asdict
1315
from pathlib import Path
1416
from types import SimpleNamespace
15-
from typing import Any, cast
17+
from typing import TYPE_CHECKING, Any, Never, cast
1618

19+
import grpc
1720
import pytest
1821
from google.protobuf.struct_pb2 import Struct
1922

2023
import openshell.sandbox as sandbox_module
21-
from openshell._proto import openshell_pb2
24+
from openshell._proto import openshell_pb2, openshell_pb2_grpc
2225
from openshell.mutations import DeletionOutcome
2326
from openshell.sandbox import (
2427
_OIDC_TOKEN_EXPIRY_GRACE_SECONDS,
@@ -43,11 +46,15 @@
4346
_make_cluster_bearer_provider,
4447
_normalize_bearer,
4548
_OidcRefresher,
49+
_OwnerCache,
4650
_read_oidc_token_bundle,
4751
_sandbox_ref,
4852
_validate_oauth_url,
4953
)
5054

55+
if TYPE_CHECKING:
56+
from collections.abc import Iterator
57+
5158

5259
def _request_workspace(request: Any) -> str | None:
5360
scope = request.workspace_scope
@@ -412,6 +419,11 @@ def handler(request: Any) -> Any:
412419

413420

414421
class _FakeStub:
422+
class GetSandbox:
423+
@staticmethod
424+
def with_call(*_args: object, **_kwargs: object) -> Never:
425+
raise grpc.RpcError()
426+
415427
def __init__(self) -> None:
416428
self.request: openshell_pb2.ExecSandboxRequest | None = None
417429

@@ -431,9 +443,102 @@ def _client_with_fake_stub(stub: object) -> SandboxClient:
431443
client = object.__new__(SandboxClient)
432444
client._timeout = 30.0
433445
client._stub = cast("Any", stub)
446+
client._owners = _OwnerCache()
434447
return client
435448

436449

450+
class _RoutingGateway(openshell_pb2_grpc.OpenShellServicer):
451+
"""Gateway that reports an owner and, behind a replica route, can answer
452+
like Envoy with a trailers-only UNAVAILABLE."""
453+
454+
def __init__(
455+
self, owner: str, *, reject_routed: str | None = None, fail: str = ""
456+
) -> None:
457+
self.owner = owner
458+
self.reject_routed = reject_routed
459+
self.fail = fail
460+
self.wanted_owner: list[str] = []
461+
self.routes: list[str] = []
462+
463+
def GetSandbox(self, request, context):
464+
_ = request
465+
headers = dict(context.invocation_metadata())
466+
self.wanted_owner.append(headers.get("x-openshell-want-owner", ""))
467+
if self.owner:
468+
context.send_initial_metadata((("x-openshell-owner", self.owner),))
469+
return openshell_pb2.SandboxResponse()
470+
471+
def ExecSandbox(self, request, context):
472+
_ = request
473+
route = dict(context.invocation_metadata()).get("x-openshell-replica", "")
474+
self.routes.append(route)
475+
if route and self.reject_routed is not None:
476+
context.abort(grpc.StatusCode.UNAVAILABLE, self.reject_routed)
477+
if self.fail:
478+
context.abort(grpc.StatusCode.UNAVAILABLE, self.fail)
479+
yield openshell_pb2.ExecSandboxEvent(
480+
stdout=openshell_pb2.ExecSandboxStdout(data=b"ok")
481+
)
482+
yield openshell_pb2.ExecSandboxEvent(
483+
exit=openshell_pb2.ExecSandboxExit(exit_code=0)
484+
)
485+
486+
487+
@contextlib.contextmanager
488+
def _routing_client(gateway: _RoutingGateway) -> Iterator[SandboxClient]:
489+
server = grpc.server(futures.ThreadPoolExecutor(max_workers=2))
490+
openshell_pb2_grpc.add_OpenShellServicer_to_server(gateway, server)
491+
port = server.add_insecure_port("127.0.0.1:0")
492+
server.start()
493+
try:
494+
with SandboxClient(f"127.0.0.1:{port}", timeout=5) as client:
495+
yield client
496+
finally:
497+
server.stop(None)
498+
499+
500+
def test_exec_routes_to_reported_owner_and_caches_it() -> None:
501+
gateway = _RoutingGateway("openshell-1")
502+
with _routing_client(gateway) as client:
503+
for _ in range(2):
504+
result = client.exec("box", ["true"], workspace="default")
505+
assert result.stdout == "ok"
506+
507+
assert gateway.wanted_owner == ["1"]
508+
assert gateway.routes == ["openshell-1", "openshell-1"]
509+
510+
511+
def test_exec_without_owner_sends_no_hint() -> None:
512+
gateway = _RoutingGateway("")
513+
with _routing_client(gateway) as client:
514+
assert client.exec("box", ["true"], workspace="default").exit_code == 0
515+
516+
assert gateway.routes == [""]
517+
518+
519+
@pytest.mark.parametrize("message", ["", "no healthy upstream"])
520+
def test_exec_retries_once_unrouted_when_owner_route_is_empty(message: str) -> None:
521+
gateway = _RoutingGateway("openshell-1", reject_routed=message)
522+
with _routing_client(gateway) as client:
523+
for _ in range(2):
524+
result = client.exec("box", ["true"], workspace="default")
525+
assert result.stdout == "ok"
526+
527+
assert gateway.routes == ["openshell-1", "", "openshell-1", ""]
528+
assert gateway.wanted_owner == ["1", "1"]
529+
530+
531+
def test_exec_does_not_retry_gateway_errors() -> None:
532+
gateway = _RoutingGateway("openshell-1", fail="supervisor not connected")
533+
with (
534+
_routing_client(gateway) as client,
535+
pytest.raises(grpc.RpcError, match="supervisor not connected"),
536+
):
537+
client.exec("box", ["true"], workspace="default")
538+
539+
assert gateway.routes == ["openshell-1"]
540+
541+
437542
def _template_client_with_fake_stub(stub: object) -> SandboxTemplateClient:
438543
client = object.__new__(SandboxTemplateClient)
439544
client._timeout = 30.0

0 commit comments

Comments
 (0)