Skip to content

Commit 00f617d

Browse files
fix(client-runtime): recover subscriptions that stop on a healthy connection (pingdotgg#24400)
1 parent 3ac8b1f commit 00f617d

3 files changed

Lines changed: 289 additions & 9 deletions

File tree

‎packages/client-runtime/src/rpc/client.test.ts‎

Lines changed: 137 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,15 +7,17 @@ import {
77
} from "@t3tools/contracts";
88
import { describe, expect, it } from "@effect/vitest";
99
import * as Cause from "effect/Cause";
10+
import * as Clock from "effect/Clock";
1011
import * as Deferred from "effect/Deferred";
1112
import * as Effect from "effect/Effect";
1213
import * as Fiber from "effect/Fiber";
1314
import * as Option from "effect/Option";
1415
import * as Queue from "effect/Queue";
1516
import * as Ref from "effect/Ref";
17+
import * as Schema from "effect/Schema";
1618
import * as Stream from "effect/Stream";
17-
import * as TestClock from "effect/testing/TestClock";
1819
import * as SubscriptionRef from "effect/SubscriptionRef";
20+
import * as TestClock from "effect/testing/TestClock";
1921
import { RpcClientError } from "effect/rpc";
2022

2123
import {
@@ -45,6 +47,8 @@ const INSTALL_DOWNLOADING: RelayClientInstallProgressEvent = {
4547
stage: "downloading",
4648
};
4749

50+
const decodeNumber = Schema.decodeUnknownEffect(Schema.Number);
51+
4852
function session(client: WsRpcProtocolClient): RpcSession.RpcSession {
4953
return {
5054
client,
@@ -265,13 +269,145 @@ describe("environment RPC", () => {
265269
while (subscriptions.length < 2) {
266270
yield* Queue.take(subscriptionChanges);
267271
}
272+
// The pending same-session retry must not resubscribe the replaced session.
273+
yield* TestClock.adjust("1 minute");
268274
yield* Fiber.interrupt(subscriptionFiber);
269275

270276
expect(subscriptions).toEqual(["first", "second"]);
271277
expect(yield* Ref.get(retryCount)).toBe(0);
272278
}),
273279
);
274280

281+
// The session stays healthy (`closed` never resolves), as when one stream chunk fails to decode,
282+
// so the supervisor will not replace it and only resubscribing restores delivery.
283+
const resumesOnTheSameSession = Effect.fn("TestEnvironmentRpc.resumesOnTheSameSession")(
284+
function* (streamFailure: Stream.Stream<never, unknown>) {
285+
const resumedValue = { source: "resumed" } as unknown as ServerLifecycleStreamEvent;
286+
const firstSubscribed = yield* Deferred.make<void>();
287+
let subscriptions = 0;
288+
const client = {
289+
[WS_METHODS.subscribeServerLifecycle]: () => {
290+
subscriptions += 1;
291+
return subscriptions === 1
292+
? Stream.fromEffect(Deferred.succeed(firstSubscribed, undefined)).pipe(
293+
Stream.drain,
294+
Stream.concat(streamFailure),
295+
)
296+
: Stream.make(resumedValue).pipe(Stream.concat(Stream.never));
297+
},
298+
} as unknown as WsRpcProtocolClient;
299+
const { activeSession, retryCount, supervisor } = yield* makeHarness();
300+
301+
yield* SubscriptionRef.set(activeSession, Option.some(session(client)));
302+
const resultFiber = yield* subscribe(WS_METHODS.subscribeServerLifecycle, {}).pipe(
303+
Stream.runHead,
304+
Effect.timeoutOption("2 seconds"),
305+
Effect.provideService(EnvironmentSupervisor.EnvironmentSupervisor, supervisor),
306+
Effect.forkChild,
307+
);
308+
yield* Deferred.await(firstSubscribed);
309+
yield* TestClock.adjust("2 seconds");
310+
311+
expect(
312+
yield* Fiber.join(resultFiber),
313+
"durable subscription stopped delivering after its stream ended on a healthy session",
314+
).toEqual(Option.some(Option.some(resumedValue)));
315+
expect(subscriptions).toBe(2);
316+
expect(yield* Ref.get(retryCount)).toBe(0);
317+
},
318+
);
319+
320+
it.effect("resubscribes on the same session after a transport failure", () =>
321+
resumesOnTheSameSession(
322+
Stream.fail(
323+
new RpcClientError.RpcClientError({
324+
reason: new RpcClientError.RpcClientDefect({
325+
message: "stream failed",
326+
cause: new Error("stream failed"),
327+
}),
328+
}),
329+
),
330+
),
331+
);
332+
333+
it.effect("resubscribes on the same session after a stream chunk fails to decode", () =>
334+
// Mirrors RpcClient, which decodes each chunk with `Effect.orDie`.
335+
resumesOnTheSameSession(
336+
Stream.fromEffect(decodeNumber("not a number").pipe(Effect.orDie)).pipe(Stream.drain),
337+
),
338+
);
339+
340+
it.effect(
341+
"backs off a same-session stream that keeps failing to decode after its first value",
342+
() =>
343+
Effect.gen(function* () {
344+
// Like the shell stream: a valid value (metadata) arrives, then a chunk fails to decode.
345+
const value = { source: "metadata" } as unknown as ServerLifecycleStreamEvent;
346+
const decodeDefect = decodeNumber("not a number").pipe(Effect.orDie);
347+
const subscriptions = yield* Queue.unbounded<{
348+
readonly session: "first" | "second";
349+
readonly at: number;
350+
}>();
351+
let secondSubscriptions = 0;
352+
const failingClient = (name: "first" | "second") =>
353+
({
354+
[WS_METHODS.subscribeServerLifecycle]: () =>
355+
Stream.unwrap(
356+
Effect.gen(function* () {
357+
yield* Queue.offer(subscriptions, {
358+
session: name,
359+
at: yield* Clock.currentTimeMillis,
360+
});
361+
if (name === "second" && ++secondSubscriptions > 2) return Stream.never;
362+
return Stream.make(value).pipe(
363+
Stream.concat(Stream.fromEffect(decodeDefect).pipe(Stream.drain)),
364+
);
365+
}),
366+
),
367+
}) as unknown as WsRpcProtocolClient;
368+
const { activeSession, retryCount, supervisor } = yield* makeHarness();
369+
370+
yield* SubscriptionRef.set(activeSession, Option.some(session(failingClient("first"))));
371+
const subscriptionFiber = yield* subscribe(
372+
WS_METHODS.subscribeServerLifecycle,
373+
{},
374+
{
375+
onExpectedFailure: () => Effect.void,
376+
retryExpectedFailureAfter: "250 millis",
377+
},
378+
).pipe(
379+
Stream.runDrain,
380+
Effect.provideService(EnvironmentSupervisor.EnvironmentSupervisor, supervisor),
381+
Effect.forkChild,
382+
);
383+
384+
const expectedFirstSessionDelays = [250, 500, 1_000, 2_000, 4_000, 8_000, 16_000, 30_000];
385+
const firstSessionDelays: number[] = [];
386+
let previous = yield* Queue.take(subscriptions);
387+
for (const delay of expectedFirstSessionDelays) {
388+
yield* TestClock.adjust(delay);
389+
const next = yield* Queue.take(subscriptions);
390+
firstSessionDelays.push(next.at - previous.at);
391+
previous = next;
392+
}
393+
394+
// A replacement session starts a fresh backoff.
395+
yield* SubscriptionRef.set(activeSession, Option.some(session(failingClient("second"))));
396+
const replaced = yield* Queue.take(subscriptions);
397+
yield* TestClock.adjust(250);
398+
const retried = yield* Queue.take(subscriptions);
399+
yield* Fiber.interrupt(subscriptionFiber);
400+
401+
expect(
402+
firstSessionDelays,
403+
"same-session decode retries reset to the base delay after every first value",
404+
).toEqual(expectedFirstSessionDelays);
405+
expect([replaced.session, retried.session]).toEqual(["second", "second"]);
406+
expect(retried.at - replaced.at).toBe(250);
407+
expect(yield* Ref.get(retryCount)).toBe(0);
408+
}),
409+
);
410+
275411
it.effect("surfaces domain subscription failures without reconnecting", () =>
276412
Effect.gen(function* () {
277413
const domainError = new Error("terminal subscription rejected");

‎packages/client-runtime/src/rpc/client.ts‎

Lines changed: 48 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,7 @@ export const isAuthorizationFailure = (cause: Cause.Cause<unknown>) =>
114114
);
115115

116116
const MAX_EXPECTED_FAILURE_RETRY_DELAY_MS = 30_000;
117+
const STREAM_FAILURE_RETRY_DELAY = Duration.seconds(1);
117118

118119
export const subscriptionRetryDelay = (base: Duration.Input, retries: number) =>
119120
Duration.millis(
@@ -318,6 +319,8 @@ function subscribeDynamicMapped<TTag extends EnvironmentSubscriptionRpcTag, A>(
318319
EnvironmentRpcStreamFailure<TTag>
319320
>;
320321
let expectedFailureRetries = 0;
322+
// Not reset by a first value: a stream can emit before every chunk failure.
323+
let streamFailureRetries = 0;
321324
const subscribeToSession = (): Stream.Stream<A, EnvironmentRpcStreamFailure<TTag>> =>
322325
Stream.suspend(() =>
323326
Stream.unwrap(
@@ -354,6 +357,11 @@ function subscribeDynamicMapped<TTag extends EnvironmentSubscriptionRpcTag, A>(
354357
: Effect.void,
355358
),
356359
Stream.catchCause((cause) => {
360+
const retryAfter = (base: Duration.Input, retries: number) =>
361+
Stream.fromEffect(Effect.sleep(subscriptionRetryDelay(base, retries))).pipe(
362+
Stream.drain,
363+
Stream.concat(subscribeToSession()),
364+
);
357365
const hasOnlyExpectedFailures =
358366
cause.reasons.length > 0 &&
359367
cause.reasons.every((reason) => reason._tag === "Fail");
@@ -362,7 +370,22 @@ function subscribeDynamicMapped<TTag extends EnvironmentSubscriptionRpcTag, A>(
362370
cause.reasons.every(
363371
(reason) => reason._tag === "Fail" && isRpcClientError(reason.error),
364372
);
365-
if (isTransportFailure) {
373+
// The RPC client dies with a SchemaError when one stream chunk fails to
374+
// decode, ending only that request while the socket stays open. A caller
375+
// with `onDefect` owns its defect recovery.
376+
const isChunkDecodeDefect =
377+
options?.onDefect === undefined &&
378+
cause.reasons.length > 0 &&
379+
cause.reasons.every(
380+
(reason) => reason._tag === "Die" && Schema.isSchemaError(reason.defect),
381+
);
382+
// Both end one stream while the session can stay healthy, so nothing else
383+
// would replace it. A dead session is replaced by the supervisor, which
384+
// interrupts this retry through switchMap. A finite search must settle
385+
// rather than restart, so it keeps waiting for the next session.
386+
const resubscribesOnSameSession =
387+
tag !== ORCHESTRATION_V2_WS_METHODS.searchThreadStream;
388+
if (isTransportFailure && !resubscribesOnSameSession) {
366389
return Stream.fromEffect(
367390
Effect.logWarning(
368391
"Durable RPC subscription lost its transport; waiting for the next session.",
@@ -374,6 +397,29 @@ function subscribeDynamicMapped<TTag extends EnvironmentSubscriptionRpcTag, A>(
374397
),
375398
).pipe(Stream.drain);
376399
}
400+
if (
401+
resubscribesOnSameSession &&
402+
(isTransportFailure || isChunkDecodeDefect)
403+
) {
404+
return Stream.fromEffect(
405+
Effect.logWarning(
406+
"Durable RPC subscription stream ended; resubscribing.",
407+
{
408+
cause: Cause.pretty(cause),
409+
method: tag,
410+
environmentId: supervisor.target.environmentId,
411+
},
412+
),
413+
).pipe(
414+
Stream.drain,
415+
Stream.concat(
416+
retryAfter(
417+
options?.retryExpectedFailureAfter ?? STREAM_FAILURE_RETRY_DELAY,
418+
streamFailureRetries++,
419+
),
420+
),
421+
);
422+
}
377423
if (hasOnlyExpectedFailures && options?.onExpectedFailure !== undefined) {
378424
const handled = Stream.fromEffect(options.onExpectedFailure(cause)).pipe(
379425
Stream.drain,
@@ -384,16 +430,10 @@ function subscribeDynamicMapped<TTag extends EnvironmentSubscriptionRpcTag, A>(
384430
) {
385431
return handled;
386432
}
387-
const retryDelay = subscriptionRetryDelay(
388-
options.retryExpectedFailureAfter,
389-
expectedFailureRetries,
390-
);
391-
expectedFailureRetries += 1;
392433
return handled.pipe(
393434
Stream.concat(
394-
Stream.fromEffect(Effect.sleep(retryDelay)).pipe(Stream.drain),
435+
retryAfter(options.retryExpectedFailureAfter, expectedFailureRetries++),
395436
),
396-
Stream.concat(subscribeToSession()),
397437
);
398438
}
399439
// A reported defect parks until the next session or resubscribe. Failing would

0 commit comments

Comments
 (0)