Skip to content

Commit bd16b86

Browse files
fix(client-runtime): report terminated thread loads (#10216)
1 parent f12d393 commit bd16b86

4 files changed

Lines changed: 498 additions & 67 deletions

File tree

‎packages/client-runtime/src/rpc/client.test.ts‎

Lines changed: 87 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -465,36 +465,112 @@ describe("environment RPC", () => {
465465
}),
466466
);
467467

468-
it.effect("does not classify subscription defects as expected failures", () =>
468+
it.effect.each(["input", "stream"] as const)(
469+
"does not classify %s subscription defects as expected failures",
470+
(where) =>
471+
Effect.gen(function* () {
472+
const defect = new Error("subscription invariant failed");
473+
let expectedFailureCount = 0;
474+
let inputs = 0;
475+
let streams = 0;
476+
const observedDefects: unknown[] = [];
477+
const client = {
478+
[WS_METHODS.subscribeTerminalEvents]: () => {
479+
streams += 1;
480+
return where === "stream" ? Stream.die(defect) : Stream.never;
481+
},
482+
} as unknown as WsRpcProtocolClient;
483+
const { activeSession, supervisor } = yield* makeHarness();
484+
485+
yield* SubscriptionRef.set(activeSession, Option.some(session(client)));
486+
const exit = yield* subscribeDynamicWithSession(
487+
WS_METHODS.subscribeTerminalEvents,
488+
() =>
489+
Effect.sync(() => {
490+
inputs += 1;
491+
}).pipe(Effect.andThen(where === "input" ? Effect.die(defect) : Effect.succeed({}))),
492+
{
493+
onDefect: (cause) =>
494+
Effect.sync(() => {
495+
observedDefects.push(Cause.squash(cause));
496+
}),
497+
onExpectedFailure: () =>
498+
Effect.sync(() => {
499+
expectedFailureCount += 1;
500+
}),
501+
retryExpectedFailureAfter: "250 millis",
502+
},
503+
).pipe(
504+
Stream.runDrain,
505+
Effect.provideService(EnvironmentSupervisor.EnvironmentSupervisor, supervisor),
506+
Effect.exit,
507+
);
508+
509+
expect(Exit.isFailure(exit)).toBe(true);
510+
if (Exit.isFailure(exit)) {
511+
expect(Cause.hasDies(exit.cause)).toBe(true);
512+
expect(Cause.squash(exit.cause)).toBe(defect);
513+
}
514+
expect(inputs).toBe(1);
515+
expect(streams).toBe(where === "input" ? 0 : 1);
516+
expect(expectedFailureCount).toBe(0);
517+
expect(observedDefects).toEqual([defect]);
518+
}),
519+
);
520+
521+
it.effect("reports an initializer defect once after an expected failure retries", () =>
469522
Effect.gen(function* () {
470-
const defect = new Error("subscription invariant failed");
471-
let expectedFailureCount = 0;
523+
const defect = new Error("Synthetic retry initializer defect");
524+
const expectedFailure = yield* Deferred.make<void>();
525+
const observations: string[] = [];
526+
const observedDefects: unknown[] = [];
527+
let inputs = 0;
472528
const client = {
473-
[WS_METHODS.subscribeTerminalEvents]: () => Stream.die(defect),
529+
[WS_METHODS.subscribeTerminalEvents]: () => {
530+
observations.push("stream");
531+
return Stream.fail(new Error("subscription not ready"));
532+
},
474533
} as unknown as WsRpcProtocolClient;
475534
const { activeSession, supervisor } = yield* makeHarness();
476-
477535
yield* SubscriptionRef.set(activeSession, Option.some(session(client)));
478-
const exit = yield* subscribe(
536+
const fiber = yield* subscribeDynamicWithSession(
479537
WS_METHODS.subscribeTerminalEvents,
480-
{},
538+
() =>
539+
Effect.sync(() => {
540+
inputs += 1;
541+
observations.push(`input ${inputs}`);
542+
return inputs;
543+
}).pipe(
544+
Effect.flatMap((attempt) => (attempt === 1 ? Effect.succeed({}) : Effect.die(defect))),
545+
),
481546
{
482-
onExpectedFailure: () =>
547+
onDefect: (cause) =>
483548
Effect.sync(() => {
484-
expectedFailureCount += 1;
549+
observations.push("defect");
550+
observedDefects.push(Cause.squash(cause));
485551
}),
552+
onExpectedFailure: () =>
553+
Effect.sync(() => {
554+
observations.push("expected failure");
555+
}).pipe(Effect.andThen(Deferred.succeed(expectedFailure, undefined)), Effect.asVoid),
556+
retryExpectedFailureAfter: "250 millis",
486557
},
487558
).pipe(
488559
Stream.runDrain,
489560
Effect.provideService(EnvironmentSupervisor.EnvironmentSupervisor, supervisor),
490561
Effect.exit,
562+
Effect.forkChild,
491563
);
492-
564+
yield* Deferred.await(expectedFailure);
565+
yield* TestClock.adjust("250 millis");
566+
const exit = yield* Fiber.join(fiber);
493567
expect(Exit.isFailure(exit)).toBe(true);
494568
if (Exit.isFailure(exit)) {
495569
expect(Cause.hasDies(exit.cause)).toBe(true);
570+
expect(Cause.squash(exit.cause)).toBe(defect);
496571
}
497-
expect(expectedFailureCount).toBe(0);
572+
expect(observations).toEqual(["input 1", "stream", "expected failure", "input 2", "defect"]);
573+
expect(observedDefects).toEqual([defect]);
498574
}),
499575
);
500576
});

‎packages/client-runtime/src/rpc/client.ts‎

Lines changed: 56 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,10 @@ export function runStream<TTag extends EnvironmentStreamCommandRpcTag>(
171171
}
172172

173173
interface SubscriptionOptions<TTag extends EnvironmentSubscriptionRpcTag> {
174+
/** Reports protocol or programming defects without changing their recovery policy. */
175+
readonly onDefect?: (
176+
cause: Cause.Cause<EnvironmentRpcStreamFailure<TTag>>,
177+
) => Effect.Effect<void, never, never>;
174178
readonly onExpectedFailure?: (
175179
cause: Cause.Cause<EnvironmentRpcStreamFailure<TTag>>,
176180
) => Effect.Effect<void, never, never>;
@@ -228,47 +232,60 @@ function subscribeDynamicMapped<TTag extends EnvironmentSubscriptionRpcTag, A>(
228232
});
229233
return mapStream(session, method(input)).pipe(
230234
Stream.ensuring(completeObservation),
231-
Stream.catchCause((cause) => {
232-
const hasOnlyExpectedFailures =
233-
cause.reasons.length > 0 &&
234-
cause.reasons.every((reason) => reason._tag === "Fail");
235-
const isTransportFailure =
236-
hasOnlyExpectedFailures &&
237-
cause.reasons.every(
238-
(reason) => reason._tag === "Fail" && isRpcClientError(reason.error),
239-
);
240-
if (isTransportFailure) {
241-
return Stream.fromEffect(
242-
Effect.logWarning(
243-
"Durable RPC subscription lost its transport; waiting for the next session.",
244-
{
245-
cause: Cause.pretty(cause),
246-
method: tag,
247-
environmentId: supervisor.target.environmentId,
248-
},
249-
),
250-
).pipe(Stream.drain);
251-
}
252-
if (hasOnlyExpectedFailures && options?.onExpectedFailure !== undefined) {
253-
const handled = Stream.fromEffect(
254-
options.onExpectedFailure(cause),
255-
).pipe(Stream.drain);
256-
if (options.retryExpectedFailureAfter === undefined) {
257-
return handled;
258-
}
259-
return handled.pipe(
260-
Stream.concat(
261-
Stream.fromEffect(
262-
Effect.sleep(options.retryExpectedFailureAfter),
263-
).pipe(Stream.drain),
264-
),
265-
Stream.concat(subscribeToSession()),
266-
);
267-
}
268-
return Stream.failCause(cause);
269-
}),
270235
);
271236
}),
237+
).pipe(
238+
Stream.tapCause((cause) =>
239+
options?.onDefect !== undefined &&
240+
cause.reasons.some(
241+
(reason) =>
242+
reason._tag === "Die" ||
243+
(reason._tag === "Fail" &&
244+
isRpcClientError(reason.error) &&
245+
reason.error.reason._tag === "RpcClientDefect"),
246+
)
247+
? options.onDefect(cause)
248+
: Effect.void,
249+
),
250+
Stream.catchCause((cause) => {
251+
const hasOnlyExpectedFailures =
252+
cause.reasons.length > 0 &&
253+
cause.reasons.every((reason) => reason._tag === "Fail");
254+
const isTransportFailure =
255+
hasOnlyExpectedFailures &&
256+
cause.reasons.every(
257+
(reason) => reason._tag === "Fail" && isRpcClientError(reason.error),
258+
);
259+
if (isTransportFailure) {
260+
return Stream.fromEffect(
261+
Effect.logWarning(
262+
"Durable RPC subscription lost its transport; waiting for the next session.",
263+
{
264+
cause: Cause.pretty(cause),
265+
method: tag,
266+
environmentId: supervisor.target.environmentId,
267+
},
268+
),
269+
).pipe(Stream.drain);
270+
}
271+
if (hasOnlyExpectedFailures && options?.onExpectedFailure !== undefined) {
272+
const handled = Stream.fromEffect(options.onExpectedFailure(cause)).pipe(
273+
Stream.drain,
274+
);
275+
if (options.retryExpectedFailureAfter === undefined) {
276+
return handled;
277+
}
278+
return handled.pipe(
279+
Stream.concat(
280+
Stream.fromEffect(Effect.sleep(options.retryExpectedFailureAfter)).pipe(
281+
Stream.drain,
282+
),
283+
),
284+
Stream.concat(subscribeToSession()),
285+
);
286+
}
287+
return Stream.failCause(cause);
288+
}),
272289
),
273290
);
274291
return subscribeToSession();

0 commit comments

Comments
 (0)