Skip to content

Commit 33e086e

Browse files
fix(client): retain relay credentials through slow ticket retries
1 parent c8379fd commit 33e086e

3 files changed

Lines changed: 483 additions & 144 deletions

File tree

‎packages/client-runtime/src/authorization/layer.test.ts‎

Lines changed: 123 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -46,14 +46,18 @@ const BOOTSTRAP: RelayEnvironmentConnectResponse = {
4646
expiresAt: "2026-06-06T01:00:00.000Z",
4747
};
4848

49-
type RecordedResponse = Response | ((init: RequestInit) => Response);
49+
type RecordedResponse = Response | "stall" | ((init: RequestInit) => Response);
5050

51-
function recordedFetch(responses: ReadonlyArray<RecordedResponse>) {
51+
function recordedFetch(responses: ReadonlyArray<RecordedResponse>, onStall: () => void) {
5252
const calls: Array<readonly [RequestInfo | URL, RequestInit]> = [];
5353
let responseIndex = 0;
5454
const fetchFn = ((input, init) => {
5555
calls.push([input, init ?? {}]);
5656
const response = responses[responseIndex++];
57+
if (response === "stall") {
58+
onStall();
59+
return new Promise<Response>(() => {});
60+
}
5761
return response === undefined
5862
? Promise.reject(new Error(`Unexpected fetch call to ${String(input)}`))
5963
: Promise.resolve(typeof response === "function" ? response(init ?? {}) : response);
@@ -136,7 +140,8 @@ const makeHarness = Effect.fn("TestRemoteAuthorization.makeHarness")(function* (
136140
readonly accessToken?: string;
137141
}>
138142
>([]);
139-
const fetch = recordedFetch(input.responses);
143+
const stalls = yield* Queue.unbounded<void>();
144+
const fetch = recordedFetch(input.responses, () => Queue.offerUnsafe(stalls, undefined));
140145

141146
const tokenStore = TokenStore.RemoteDpopAccessTokenStore.of({
142147
get: (environmentId) =>
@@ -227,6 +232,7 @@ const makeHarness = Effect.fn("TestRemoteAuthorization.makeHarness")(function* (
227232
session,
228233
thumbprint,
229234
tokenReads,
235+
stalls,
230236
};
231237
});
232238

@@ -354,18 +360,36 @@ describe("RemoteEnvironmentAuthorization", () => {
354360
});
355361
const harness = yield* makeHarness({
356362
initialToken: cached,
357-
responses: [Response.json(DESCRIPTOR), websocketTicket("lan-ticket")],
363+
responses: [
364+
Response.json(DESCRIPTOR),
365+
websocketTicket("lan-ticket"),
366+
"stall",
367+
websocketTicket("fallback-ticket"),
368+
],
358369
});
359370

360371
const authorized = yield* Effect.gen(function* () {
361372
const remote = yield* RemoteEnvironmentAuthorization.RemoteEnvironmentAuthorization;
362-
return yield* remote.authorizeDpop({
373+
const direct = remote.authorizeDpop({
363374
expectedEnvironmentId: ENVIRONMENT_ID,
364375
directEndpoint: {
365376
httpBaseUrl: "http://192.168.1.10:3773/",
366377
wsBaseUrl: "ws://192.168.1.10:3773/",
367378
},
368379
});
380+
const authorized = yield* direct;
381+
const stalled = yield* direct.pipe(Effect.flip, Effect.forkChild);
382+
yield* Queue.take(harness.stalls);
383+
yield* TestClock.adjust("3 seconds");
384+
expect(harness.fetch.calls).toHaveLength(3);
385+
expect(yield* Fiber.join(stalled)).toMatchObject({
386+
_tag: "ConnectionTransientError",
387+
reason: "timeout",
388+
});
389+
const fallback = yield* remote.authorizeDpop({ expectedEnvironmentId: ENVIRONMENT_ID });
390+
expect(fallback.socketUrl).toContain("wsTicket=fallback-ticket");
391+
expect(fallback.httpAuthorization).toMatchObject({ accessToken: cached.accessToken });
392+
return authorized;
369393
}).pipe(Effect.provide(harness.layer));
370394

371395
expect(authorized.httpBaseUrl).toBe("http://192.168.1.10:3773/");
@@ -374,6 +398,8 @@ describe("RemoteEnvironmentAuthorization", () => {
374398
expect(harness.fetch.calls.map(([url]) => String(url))).toEqual([
375399
"http://192.168.1.10:3773/.well-known/t3/environment",
376400
"http://192.168.1.10:3773/api/auth/websocket-ticket",
401+
"http://192.168.1.10:3773/api/auth/websocket-ticket",
402+
"https://environment.example.test/api/auth/websocket-ticket",
377403
]);
378404
}),
379405
);
@@ -666,6 +692,98 @@ describe("RemoteEnvironmentAuthorization", () => {
666692
}),
667693
);
668694

695+
it.effect("resets relay ticket timeouts after recovery and after credential replacement", () =>
696+
Effect.gen(function* () {
697+
const harness = yield* makeHarness({
698+
initialToken: persistedToken(),
699+
responses: [
700+
"stall",
701+
"stall",
702+
"stall",
703+
"stall",
704+
"stall",
705+
websocketTicket("recovered-ticket"),
706+
"stall",
707+
"stall",
708+
"stall",
709+
"stall",
710+
Response.json(DESCRIPTOR),
711+
accessToken("replacement-token"),
712+
"stall",
713+
"stall",
714+
websocketTicket("replacement-ticket"),
715+
],
716+
});
717+
yield* Effect.gen(function* () {
718+
const remote = yield* RemoteEnvironmentAuthorization.RemoteEnvironmentAuthorization;
719+
const authorize = remote.authorizeDpop({ expectedEnvironmentId: ENVIRONMENT_ID });
720+
const timeOut = (expectedMints: number) =>
721+
Effect.gen(function* () {
722+
const pending = yield* authorize.pipe(Effect.flip, Effect.forkChild);
723+
yield* Queue.take(harness.stalls);
724+
yield* TestClock.adjust("3 seconds");
725+
expect(yield* Ref.get(harness.bootstrapCalls)).toBe(expectedMints);
726+
yield* Queue.take(harness.stalls);
727+
yield* TestClock.adjust("7 seconds");
728+
expect(yield* Ref.get(harness.bootstrapCalls)).toBe(expectedMints);
729+
expect(yield* Fiber.join(pending)).toMatchObject({
730+
_tag: "ConnectionTransientError",
731+
reason: "timeout",
732+
});
733+
});
734+
yield* timeOut(0);
735+
yield* timeOut(0);
736+
const recovery = yield* authorize.pipe(Effect.forkChild);
737+
yield* Queue.take(harness.stalls);
738+
yield* TestClock.adjust("3 seconds");
739+
expect((yield* Fiber.join(recovery)).socketUrl).toContain("wsTicket=recovered-ticket");
740+
yield* timeOut(0);
741+
yield* timeOut(0);
742+
743+
yield* remote.authorizeDpopHttp({
744+
expectedEnvironmentId: ENVIRONMENT_ID,
745+
rejectedAccessToken: "cached-access-token",
746+
});
747+
yield* timeOut(1);
748+
const replacement = yield* authorize;
749+
expect(replacement.socketUrl).toContain("wsTicket=replacement-ticket");
750+
expect(replacement.httpAuthorization).toMatchObject({ accessToken: "replacement-token" });
751+
expect(yield* Ref.get(harness.bootstrapCalls)).toBe(1);
752+
}).pipe(Effect.provide(harness.layer));
753+
}),
754+
);
755+
756+
it.effect("reuses a newly minted credential after its first relay ticket times out", () =>
757+
Effect.gen(function* () {
758+
const harness = yield* makeHarness({
759+
responses: [
760+
Response.json(DESCRIPTOR),
761+
accessToken("fresh-token"),
762+
"stall",
763+
websocketTicket("recovered-ticket"),
764+
],
765+
});
766+
yield* Effect.gen(function* () {
767+
const remote = yield* RemoteEnvironmentAuthorization.RemoteEnvironmentAuthorization;
768+
const authorize = remote.authorizeDpop({ expectedEnvironmentId: ENVIRONMENT_ID });
769+
const pending = yield* authorize.pipe(Effect.flip, Effect.forkChild);
770+
yield* Queue.take(harness.stalls);
771+
yield* TestClock.adjust("10 seconds");
772+
expect(yield* Fiber.join(pending)).toMatchObject({
773+
_tag: "ConnectionTransientError",
774+
reason: "timeout",
775+
});
776+
expect((yield* Ref.get(harness.tokens)).get(ENVIRONMENT_ID)?.accessToken).toBe(
777+
"fresh-token",
778+
);
779+
const recovered = yield* authorize;
780+
expect(recovered.socketUrl).toContain("wsTicket=recovered-ticket");
781+
expect(recovered.httpAuthorization).toMatchObject({ accessToken: "fresh-token" });
782+
expect(yield* Ref.get(harness.bootstrapCalls)).toBe(1);
783+
}).pipe(Effect.provide(harness.layer));
784+
}),
785+
);
786+
669787
it.effect("removes a newly refreshed token when websocket ticket authorization fails", () =>
670788
Effect.gen(function* () {
671789
const harness = yield* makeHarness({

‎packages/client-runtime/src/authorization/service.ts‎

Lines changed: 51 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,9 @@ export class RemoteEnvironmentAuthorization extends Context.Service<
8686
>()("@t3tools/client-runtime/authorization/service/RemoteEnvironmentAuthorization") {}
8787

8888
const CACHED_ENDPOINT_SOCKET_TIMEOUT_MS = 3_000;
89+
// The 3 + 7 second ticket budget leaves 5 seconds of the supervisor's setup deadline.
90+
const RELAY_SOCKET_RETRY_TIMEOUT_MS = 7_000;
91+
const RELAY_TICKET_TIMEOUT_LIMIT = 3;
8992
const BEARER_DESCRIPTOR_CACHE_TTL_MS = 10_000;
9093
const DPOP_AUTHORIZATION_TIMEOUT_MS = 30_000;
9194

@@ -118,6 +121,14 @@ export const make = Effect.gen(function* () {
118121
EnvironmentId,
119122
{ readonly accessToken: string; readonly identity: ClientCapabilities.CloudSessionIdentity }
120123
>();
124+
const relayTicketTimeouts = new Map<
125+
EnvironmentId,
126+
{
127+
readonly accessToken: string;
128+
readonly identity: ClientCapabilities.CloudSessionIdentity;
129+
readonly count: number;
130+
}
131+
>();
121132
const pendingTokens = new Map<
122133
EnvironmentId,
123134
{
@@ -510,14 +521,24 @@ export const make = Effect.gen(function* () {
510521
});
511522
let selected = yield* getDpopToken(input);
512523
if (selected.fromCache) {
524+
const cachedToken = selected.token;
513525
const cachedSocket = yield* createDpopSocketUrl(
514-
selected.token,
526+
cachedToken,
515527
CACHED_ENDPOINT_SOCKET_TIMEOUT_MS,
516528
endpoint,
517-
).pipe(Effect.result);
529+
).pipe(
530+
Effect.catchTags({
531+
RemoteEnvironmentAuthTimeoutError: (error) =>
532+
endpoint === undefined
533+
? createDpopSocketUrl(cachedToken, RELAY_SOCKET_RETRY_TIMEOUT_MS)
534+
: Effect.fail(error),
535+
}),
536+
Effect.result,
537+
);
518538
if (Result.isSuccess(cachedSocket)) {
519539
yield* assertSession(selected.identity);
520-
return authorized(selected.token, cachedSocket.success);
540+
if (endpoint === undefined) relayTicketTimeouts.delete(input.expectedEnvironmentId);
541+
return authorized(cachedToken, cachedSocket.success);
521542
}
522543
if (cachedSocket.failure._tag === "ConnectionBlockedError") {
523544
return yield* mapDpopSocketError(cachedSocket.failure);
@@ -528,23 +549,48 @@ export const make = Effect.gen(function* () {
528549
return yield* mapDpopSocketError(cachedSocket.failure);
529550
}
530551
yield* assertSession(selected.identity);
552+
if (cachedSocket.failure._tag === "RemoteEnvironmentAuthTimeoutError") {
553+
const previous = relayTicketTimeouts.get(input.expectedEnvironmentId);
554+
const count =
555+
previous?.accessToken === cachedToken.accessToken &&
556+
previous.identity === selected.identity
557+
? previous.count + 1
558+
: 1;
559+
if (count < RELAY_TICKET_TIMEOUT_LIMIT) {
560+
relayTicketTimeouts.set(input.expectedEnvironmentId, {
561+
accessToken: cachedToken.accessToken,
562+
identity: selected.identity,
563+
count,
564+
});
565+
// A slow ticket response does not establish a credential or network failure.
566+
return yield* new ConnectionTransientError({
567+
reason: "timeout",
568+
detail: `${cachedToken.label} did not respond during connection setup.`,
569+
});
570+
}
571+
}
572+
if (endpoint === undefined) relayTicketTimeouts.delete(input.expectedEnvironmentId);
531573
selected = yield* getDpopToken({
532574
...input,
533-
rejectedAccessToken: selected.token.accessToken,
575+
rejectedAccessToken: cachedToken.accessToken,
534576
});
535577
}
536578
const socket = yield* createDpopSocketUrl(selected.token, undefined, endpoint).pipe(
537579
Effect.result,
538580
);
539581
if (Result.isFailure(socket)) {
540-
if (endpoint === undefined || socket.failure._tag === "ConnectionBlockedError") {
582+
if (
583+
(endpoint === undefined && socket.failure._tag !== "RemoteEnvironmentAuthTimeoutError") ||
584+
socket.failure._tag === "ConnectionBlockedError"
585+
) {
541586
yield* tokenLock.withPermits(1)(
542587
removeRejectedToken(input.expectedEnvironmentId, selected.token.accessToken),
543588
);
544589
}
545590
return yield* mapDpopSocketError(socket.failure);
546591
}
547592
yield* assertSession(selected.identity);
593+
if (endpoint === undefined) relayTicketTimeouts.delete(input.expectedEnvironmentId);
548594
return authorized(selected.token, socket.success);
549595
});
550596

0 commit comments

Comments
 (0)