33
44from __future__ import annotations
55
6+ import contextlib
67import json
78import os
89import pickle
910import threading
1011import time
12+ from concurrent import futures
1113from copy import deepcopy
1214from dataclasses import asdict
1315from pathlib import Path
1416from types import SimpleNamespace
15- from typing import Any , cast
17+ from typing import TYPE_CHECKING , Any , Never , cast
1618
19+ import grpc
1720import pytest
1821from google .protobuf .struct_pb2 import Struct
1922
2023import openshell .sandbox as sandbox_module
21- from openshell ._proto import openshell_pb2
24+ from openshell ._proto import openshell_pb2 , openshell_pb2_grpc
2225from openshell .mutations import DeletionOutcome
2326from openshell .sandbox import (
2427 _OIDC_TOKEN_EXPIRY_GRACE_SECONDS ,
4346 _make_cluster_bearer_provider ,
4447 _normalize_bearer ,
4548 _OidcRefresher ,
49+ _OwnerCache ,
4650 _read_oidc_token_bundle ,
4751 _sandbox_ref ,
4852 _validate_oauth_url ,
4953)
5054
55+ if TYPE_CHECKING :
56+ from collections .abc import Iterator
57+
5158
5259def _request_workspace (request : Any ) -> str | None :
5360 scope = request .workspace_scope
@@ -412,6 +419,11 @@ def handler(request: Any) -> Any:
412419
413420
414421class _FakeStub :
422+ class GetSandbox :
423+ @staticmethod
424+ def with_call (* _args : object , ** _kwargs : object ) -> Never :
425+ raise grpc .RpcError ()
426+
415427 def __init__ (self ) -> None :
416428 self .request : openshell_pb2 .ExecSandboxRequest | None = None
417429
@@ -431,9 +443,102 @@ def _client_with_fake_stub(stub: object) -> SandboxClient:
431443 client = object .__new__ (SandboxClient )
432444 client ._timeout = 30.0
433445 client ._stub = cast ("Any" , stub )
446+ client ._owners = _OwnerCache ()
434447 return client
435448
436449
450+ class _RoutingGateway (openshell_pb2_grpc .OpenShellServicer ):
451+ """Gateway that reports an owner and, behind a replica route, can answer
452+ like Envoy with a trailers-only UNAVAILABLE."""
453+
454+ def __init__ (
455+ self , owner : str , * , reject_routed : str | None = None , fail : str = ""
456+ ) -> None :
457+ self .owner = owner
458+ self .reject_routed = reject_routed
459+ self .fail = fail
460+ self .wanted_owner : list [str ] = []
461+ self .routes : list [str ] = []
462+
463+ def GetSandbox (self , request , context ):
464+ _ = request
465+ headers = dict (context .invocation_metadata ())
466+ self .wanted_owner .append (headers .get ("x-openshell-want-owner" , "" ))
467+ if self .owner :
468+ context .send_initial_metadata ((("x-openshell-owner" , self .owner ),))
469+ return openshell_pb2 .SandboxResponse ()
470+
471+ def ExecSandbox (self , request , context ):
472+ _ = request
473+ route = dict (context .invocation_metadata ()).get ("x-openshell-replica" , "" )
474+ self .routes .append (route )
475+ if route and self .reject_routed is not None :
476+ context .abort (grpc .StatusCode .UNAVAILABLE , self .reject_routed )
477+ if self .fail :
478+ context .abort (grpc .StatusCode .UNAVAILABLE , self .fail )
479+ yield openshell_pb2 .ExecSandboxEvent (
480+ stdout = openshell_pb2 .ExecSandboxStdout (data = b"ok" )
481+ )
482+ yield openshell_pb2 .ExecSandboxEvent (
483+ exit = openshell_pb2 .ExecSandboxExit (exit_code = 0 )
484+ )
485+
486+
487+ @contextlib .contextmanager
488+ def _routing_client (gateway : _RoutingGateway ) -> Iterator [SandboxClient ]:
489+ server = grpc .server (futures .ThreadPoolExecutor (max_workers = 2 ))
490+ openshell_pb2_grpc .add_OpenShellServicer_to_server (gateway , server )
491+ port = server .add_insecure_port ("127.0.0.1:0" )
492+ server .start ()
493+ try :
494+ with SandboxClient (f"127.0.0.1:{ port } " , timeout = 5 ) as client :
495+ yield client
496+ finally :
497+ server .stop (None )
498+
499+
500+ def test_exec_routes_to_reported_owner_and_caches_it () -> None :
501+ gateway = _RoutingGateway ("openshell-1" )
502+ with _routing_client (gateway ) as client :
503+ for _ in range (2 ):
504+ result = client .exec ("box" , ["true" ], workspace = "default" )
505+ assert result .stdout == "ok"
506+
507+ assert gateway .wanted_owner == ["1" ]
508+ assert gateway .routes == ["openshell-1" , "openshell-1" ]
509+
510+
511+ def test_exec_without_owner_sends_no_hint () -> None :
512+ gateway = _RoutingGateway ("" )
513+ with _routing_client (gateway ) as client :
514+ assert client .exec ("box" , ["true" ], workspace = "default" ).exit_code == 0
515+
516+ assert gateway .routes == ["" ]
517+
518+
519+ @pytest .mark .parametrize ("message" , ["" , "no healthy upstream" ])
520+ def test_exec_retries_once_unrouted_when_owner_route_is_empty (message : str ) -> None :
521+ gateway = _RoutingGateway ("openshell-1" , reject_routed = message )
522+ with _routing_client (gateway ) as client :
523+ for _ in range (2 ):
524+ result = client .exec ("box" , ["true" ], workspace = "default" )
525+ assert result .stdout == "ok"
526+
527+ assert gateway .routes == ["openshell-1" , "" , "openshell-1" , "" ]
528+ assert gateway .wanted_owner == ["1" , "1" ]
529+
530+
531+ def test_exec_does_not_retry_gateway_errors () -> None :
532+ gateway = _RoutingGateway ("openshell-1" , fail = "supervisor not connected" )
533+ with (
534+ _routing_client (gateway ) as client ,
535+ pytest .raises (grpc .RpcError , match = "supervisor not connected" ),
536+ ):
537+ client .exec ("box" , ["true" ], workspace = "default" )
538+
539+ assert gateway .routes == ["openshell-1" ]
540+
541+
437542def _template_client_with_fake_stub (stub : object ) -> SandboxTemplateClient :
438543 client = object .__new__ (SandboxTemplateClient )
439544 client ._timeout = 30.0
0 commit comments