Skip to content

Commit f398fa6

Browse files
committed
fix(connect): preserve rotated credentials during shutdown
1 parent 7ac952c commit f398fa6

4 files changed

Lines changed: 189 additions & 5 deletions

File tree

‎apps/server/src/cloud/CliTokenManager.test.ts‎

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,17 @@ const tokenGranted = {
140140
}),
141141
};
142142

143+
const rotatedTokenGranted = {
144+
status: 200,
145+
body: encodeTestTokenResponse({
146+
access_token: "access-token-2",
147+
refresh_token: "refresh-token-2",
148+
id_token: idTokenWithEmail,
149+
expires_in: 3600,
150+
token_type: "bearer",
151+
}),
152+
};
153+
143154
const layerDeviceFlow = (server: DeviceFlowServer) =>
144155
Layer.succeed(
145156
HttpClient.HttpClient,
@@ -189,6 +200,140 @@ const expiredStoredToken = new TextEncoder().encode(
189200
JSON.stringify({ accessToken: "expired", refreshToken: "refresh-token-1", expiresAtEpochMs: 0 }),
190201
);
191202

203+
const layerRefreshResponse = (reply: { readonly status: number; readonly body: string }) =>
204+
Layer.succeed(
205+
HttpClient.HttpClient,
206+
HttpClient.make((request) =>
207+
Effect.succeed(
208+
HttpClientResponse.fromWeb(
209+
request,
210+
new Response(reply.body, {
211+
status: reply.status,
212+
headers: { "content-type": "application/json" },
213+
}),
214+
),
215+
),
216+
),
217+
);
218+
219+
it.effect("persists a rotated refresh token before parent interruption is delivered", () =>
220+
Effect.gen(function* () {
221+
const requestStarted = yield* Deferred.make<void>();
222+
const releaseRequest = yield* Deferred.make<void>();
223+
const persistStarted = yield* Deferred.make<Uint8Array>();
224+
const releasePersist = yield* Deferred.make<void>();
225+
let persisted: Uint8Array | undefined;
226+
227+
const httpClient = Layer.succeed(
228+
HttpClient.HttpClient,
229+
HttpClient.make((request) =>
230+
Effect.gen(function* () {
231+
yield* Deferred.succeed(requestStarted, undefined);
232+
yield* Deferred.await(releaseRequest);
233+
return HttpClientResponse.fromWeb(
234+
request,
235+
new Response(rotatedTokenGranted.body, {
236+
status: rotatedTokenGranted.status,
237+
headers: { "content-type": "application/json" },
238+
}),
239+
);
240+
}),
241+
),
242+
);
243+
const secrets = Layer.mock(ServerSecretStore.ServerSecretStore)({
244+
get: () => Effect.succeedSome(expiredStoredToken),
245+
set: (_name, value) =>
246+
Effect.gen(function* () {
247+
yield* Deferred.succeed(persistStarted, value);
248+
yield* Deferred.await(releasePersist);
249+
persisted = value;
250+
}),
251+
});
252+
yield* Effect.gen(function* () {
253+
const manager = yield* CliTokenManager.CloudCliTokenManager;
254+
const refresh = yield* manager.getExisting.pipe(Effect.forkChild);
255+
256+
yield* Deferred.await(requestStarted);
257+
const interruption = yield* Effect.forkChild(Fiber.interrupt(refresh), {
258+
startImmediately: true,
259+
});
260+
yield* Deferred.succeed(releaseRequest, undefined);
261+
const persistedBeforeExit = yield* Effect.raceFirst(
262+
Deferred.await(persistStarted).pipe(Effect.as(true)),
263+
Fiber.await(refresh).pipe(Effect.as(false)),
264+
);
265+
assert.isTrue(persistedBeforeExit, "refresh exited before persistence started");
266+
yield* Deferred.succeed(releasePersist, undefined);
267+
yield* Fiber.join(interruption);
268+
269+
// The queued interruption may be delivered immediately after the
270+
// uninterruptible refresh transaction completes. Either exit is valid;
271+
// the important guarantee is that persistence finished first.
272+
yield* Fiber.await(refresh);
273+
assert.isDefined(persisted);
274+
assert.equal(
275+
JSON.parse(new TextDecoder().decode(persisted!)).refreshToken,
276+
"refresh-token-2",
277+
);
278+
}).pipe(
279+
Effect.provide(
280+
CliTokenManager.layer.pipe(
281+
Layer.provide(
282+
Layer.mergeAll(
283+
httpClient,
284+
secrets,
285+
Layer.mock(ExternalLauncher.ExternalLauncher)({}),
286+
NodeServices.layer,
287+
),
288+
),
289+
),
290+
),
291+
provideTestEnv,
292+
);
293+
}),
294+
);
295+
296+
it.effect.each([
297+
{ status: 400, requiresLogin: true },
298+
{ status: 401, requiresLogin: true },
299+
{ status: 403, requiresLogin: true },
300+
{ status: 408, requiresLogin: false },
301+
{ status: 429, requiresLogin: false },
302+
{ status: 500, requiresLogin: false },
303+
])("marks refresh status $status as requiresLogin=$requiresLogin", ({ status, requiresLogin }) =>
304+
Effect.gen(function* () {
305+
const manager = yield* CliTokenManager.CloudCliTokenManager;
306+
const error = yield* manager.getExisting.pipe(Effect.flip);
307+
308+
assert.instanceOf(error, CliTokenManager.CloudCliCredentialRefreshError);
309+
assert.equal(error.requiresLogin, requiresLogin);
310+
assert.equal(error.message.includes("t3 connect login"), requiresLogin);
311+
assert.notInclude(error.message, "private-oauth-diagnostic");
312+
}).pipe(
313+
Effect.provide(
314+
CliTokenManager.layer.pipe(
315+
Layer.provide(
316+
Layer.mergeAll(
317+
layerRefreshResponse({
318+
status,
319+
body: JSON.stringify({
320+
error: "invalid_grant",
321+
error_description: "private-oauth-diagnostic",
322+
}),
323+
}),
324+
Layer.mock(ServerSecretStore.ServerSecretStore)({
325+
get: () => Effect.succeedSome(expiredStoredToken),
326+
}),
327+
Layer.mock(ExternalLauncher.ExternalLauncher)({}),
328+
NodeServices.layer,
329+
),
330+
),
331+
),
332+
),
333+
provideTestEnv,
334+
),
335+
);
336+
192337
it.effect("a stalled token refresh fails and releases the credential lock", () =>
193338
Effect.gen(function* () {
194339
const manager = yield* CliTokenManager.CloudCliTokenManager;

‎apps/server/src/cloud/CliTokenManager.ts‎

Lines changed: 29 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -195,13 +195,22 @@ export class CloudCliCredentialRemovalError extends Schema.TaggedError<CloudCliC
195195

196196
export class CloudCliCredentialRefreshError extends Schema.TaggedError<CloudCliCredentialRefreshError>()(
197197
"CloudCliCredentialRefreshError",
198-
{ cause: Schema.Defect() },
198+
{
199+
cause: Schema.Defect(),
200+
requiresLogin: Schema.optional(Schema.Boolean).pipe(
201+
Schema.withConstructorDefault(Effect.succeed(false)),
202+
),
203+
},
199204
) {
200205
override get message(): string {
201-
return "Could not refresh the T3 Connect CLI credential.";
206+
return this.requiresLogin
207+
? "Could not refresh the T3 Connect CLI credential. Run `t3 connect login` to authorize again."
208+
: "Could not refresh the T3 Connect CLI credential.";
202209
}
203210
}
204211

212+
export const isCloudCliCredentialRefreshError = Schema.is(CloudCliCredentialRefreshError);
213+
205214
export class CloudCliCredentialReadError extends Schema.TaggedError<CloudCliCredentialReadError>()(
206215
"CloudCliCredentialReadError",
207216
{ cause: Schema.Defect() },
@@ -538,12 +547,28 @@ export const make = Effect.gen(function* () {
538547
if (token.value.expiresAtEpochMs - CLOUD_CLI_OAUTH_REFRESH_EARLY_MS > now) {
539548
return token;
540549
}
541-
return Option.some(yield* refresh(token.value).pipe(Effect.flatMap(persist)));
550+
return Option.some(
551+
// Queue shutdown interruption until the rotated credential is persisted;
552+
// exchangeToken's inner 10-second timeout remains interruptible.
553+
yield* refresh(token.value).pipe(Effect.flatMap(persist), Effect.uninterruptible),
554+
);
542555
});
543556

544557
const getExisting = semaphore.withPermits(1)(
545558
getExistingNoLock().pipe(
546-
Effect.mapError((cause) => new CloudCliCredentialRefreshError({ cause })),
559+
Effect.mapError(
560+
(cause) =>
561+
new CloudCliCredentialRefreshError({
562+
cause,
563+
requiresLogin:
564+
HttpClientError.isHttpClientError(cause) &&
565+
cause.reason._tag === "StatusCodeError" &&
566+
cause.reason.response.status >= 400 &&
567+
cause.reason.response.status < 500 &&
568+
cause.reason.response.status !== 408 &&
569+
cause.reason.response.status !== 429,
570+
}),
571+
),
547572
Effect.provide(services),
548573
),
549574
);

‎apps/server/src/cloud/CloudLink.lifecycle.test.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1271,6 +1271,18 @@ describe("releaseManagedTunnelOnShutdown", () => {
12711271
);
12721272
});
12731273

1274+
it.each([
1275+
{ requiresLogin: true, retry: false },
1276+
{ requiresLogin: false, retry: true },
1277+
])("$requiresLogin refresh errors are retry=$retry", ({ requiresLogin, retry }) => {
1278+
const error = new CliTokenManager.CloudCliCredentialRefreshError({
1279+
cause: new Error("refresh failed"),
1280+
requiresLogin,
1281+
});
1282+
1283+
expect(CloudLink.shouldRetryCloudLink(error)).toBe(retry);
1284+
});
1285+
12741286
it.effect("keeps a tunnel configuration replaced during recovery", () => {
12751287
const { store, values } = makeMemorySecretStore([
12761288
[CLOUD_ENDPOINT_RUNTIME_CONFIG, "old-config"],

‎apps/server/src/cloud/CloudLink.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -313,7 +313,9 @@ const isPermanentLinkError = Schema.is(
313313

314314
/** Whether a failed link step may succeed on retry: not when the relay or this server refused it. */
315315
export const shouldRetryCloudLink = (error: unknown): boolean =>
316-
shouldRetryRelayRequest(error) && !isPermanentLinkError(error);
316+
shouldRetryRelayRequest(error) &&
317+
!isPermanentLinkError(error) &&
318+
!(CliTokenManager.isCloudCliCredentialRefreshError(error) && error.requiresLogin === true);
317319

318320
/** A failed rollback leaves a setting changed; it is logged, not hidden. */
319321
const rollbackFailed = (cause: unknown) =>

0 commit comments

Comments
 (0)