Skip to content

Commit a7bc5e7

Browse files
committed
fix(terminal): release stalled subscribers and preserve startup errors
1 parent 0e22886 commit a7bc5e7

6 files changed

Lines changed: 176 additions & 14 deletions

File tree

‎apps/server/src/terminal/AttachStream.test.ts‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,11 @@ import {
66
type TerminalAttachStreamEvent,
77
} from "@t3tools/contracts";
88
import * as Deferred from "effect/Deferred";
9+
import * as Clock from "effect/Clock";
910
import * as Effect from "effect/Effect";
1011
import * as Fiber from "effect/Fiber";
1112
import * as Stream from "effect/Stream";
13+
import * as TestClock from "effect/testing/TestClock";
1214

1315
import * as TerminalManager from "./Manager.ts";
1416
import * as Layer from "effect/Layer";
@@ -234,3 +236,63 @@ it.effect("interrupts an attach that is still replaying when the consumer discon
234236
expect(replayInterrupted).toBe(true);
235237
}),
236238
);
239+
240+
it.effect.each(["replay", "live"] as const)(
241+
"disconnects a stalled %s consumer and releases its subscription",
242+
(phase) =>
243+
Effect.gen(function* () {
244+
const subscribed =
245+
yield* Deferred.make<
246+
Parameters<TerminalManager.TerminalManager["Service"]["attachStream"]>[1]
247+
>();
248+
const consumerStarted = yield* Deferred.make<void>();
249+
const resumeConsumer = yield* Deferred.make<void>();
250+
const detached = yield* Deferred.make<void>();
251+
const clock = yield* Clock.clockWith(Effect.succeed);
252+
const consumer = yield* terminalAttachStream(input).pipe(
253+
Stream.provideService(Clock.Clock, clock),
254+
Stream.provide(
255+
Layer.mock(TerminalManager.TerminalManager)({
256+
attachStream: (_input, listener) =>
257+
Effect.gen(function* () {
258+
yield* Deferred.succeed(subscribed, listener);
259+
if (phase === "replay") {
260+
for (let index = 0; index < 100; index += 1) {
261+
yield* listener({ type: "output", ...target, data: "history" }, "replay");
262+
}
263+
}
264+
return () => {
265+
Deferred.doneUnsafe(detached, Effect.void);
266+
};
267+
}).pipe(Effect.onInterrupt(() => Deferred.succeed(detached, undefined))),
268+
}),
269+
),
270+
Stream.runForEach(() =>
271+
Deferred.succeed(consumerStarted, undefined).pipe(
272+
Effect.andThen(Deferred.await(resumeConsumer)),
273+
),
274+
),
275+
Effect.flip,
276+
Effect.forkChild,
277+
);
278+
const publish = yield* Deferred.await(subscribed);
279+
if (phase === "live") yield* publish({ type: "replay-complete", ...target }, "replay");
280+
yield* Deferred.await(consumerStarted);
281+
const producer =
282+
phase === "live"
283+
? yield* Effect.gen(function* () {
284+
for (let index = 0; index < 33; index += 1) {
285+
yield* publish({ type: "output", ...target, data: "live" }, "live");
286+
}
287+
}).pipe(Effect.forkChild({ startImmediately: true }))
288+
: null;
289+
yield* TestClock.adjust("30 seconds");
290+
yield* Deferred.await(detached);
291+
if (producer) yield* Fiber.join(producer);
292+
yield* Deferred.succeed(resumeConsumer, undefined);
293+
expect(yield* Fiber.join(consumer)).toMatchObject({
294+
_tag: "TerminalAttachTimeoutError",
295+
...target,
296+
});
297+
}),
298+
);

‎apps/server/src/terminal/AttachStream.ts‎

Lines changed: 44 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,59 @@
1-
import type {
2-
TerminalAttachInput,
3-
TerminalAttachStreamEvent,
4-
TerminalError,
1+
import {
2+
TerminalAttachTimeoutError,
3+
type TerminalAttachInput,
4+
type TerminalAttachStreamEvent,
5+
type TerminalError,
56
} from "@t3tools/contracts";
7+
import * as Deferred from "effect/Deferred";
68
import * as Effect from "effect/Effect";
79
import * as Queue from "effect/Queue";
10+
import * as Option from "effect/Option";
811
import * as Stream from "effect/Stream";
912

1013
import * as TerminalManager from "./Manager.ts";
1114

12-
/** Backpressure preserves output and replay boundaries, including the final bytes before close. */
15+
/** Preserve ordered output under backpressure; disconnect consumers that stop draining. */
1316
export function terminalAttachStream(input: TerminalAttachInput) {
1417
return Stream.callback<TerminalAttachStreamEvent, TerminalError, TerminalManager.TerminalManager>(
1518
(queue) =>
1619
Effect.gen(function* () {
1720
const manager = yield* TerminalManager.TerminalManager;
18-
return yield* Effect.acquireRelease(
19-
manager
20-
.attachStream(input, (event) =>
21-
Queue.offer(queue, event).pipe(Effect.asVoid, Effect.ignore),
22-
)
23-
.pipe(Effect.interruptible),
24-
(unsubscribe) => Effect.sync(unsubscribe),
21+
const stalled = yield* Deferred.make<void>();
22+
const deliver = (event: TerminalAttachStreamEvent) =>
23+
Effect.suspend(() => {
24+
if (Queue.offerUnsafe(queue, event)) return Effect.void;
25+
return Queue.offer(queue, event).pipe(
26+
Effect.timeoutOption("30 seconds"),
27+
Effect.flatMap((offered) => {
28+
if (Option.isSome(offered)) return Effect.void;
29+
return Effect.gen(function* () {
30+
yield* Queue.fail(
31+
queue,
32+
new TerminalAttachTimeoutError({
33+
threadId: input.threadId,
34+
terminalId: input.terminalId,
35+
}),
36+
);
37+
// The timed-out offer is already interrupted. Closing its
38+
// queue must not race the timeout and cancel this cleanup.
39+
yield* Queue.shutdown(queue);
40+
yield* Deferred.succeed(stalled, undefined);
41+
});
42+
}),
43+
);
44+
});
45+
return yield* Effect.gen(function* () {
46+
yield* Effect.acquireRelease(
47+
manager.attachStream(input, deliver).pipe(Effect.interruptible),
48+
(unsubscribe) => Effect.sync(unsubscribe),
49+
);
50+
return yield* Effect.never;
51+
}).pipe(
52+
Effect.scoped,
53+
Effect.raceFirst(Deferred.await(stalled)),
54+
Effect.catchCause((cause) =>
55+
Queue.failCause(queue, cause).pipe(Effect.andThen(Queue.shutdown(queue))),
56+
),
2557
);
2658
}),
2759
{ bufferSize: 32, strategy: "suspend" },

‎apps/server/src/terminal/Manager.test.ts‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3147,6 +3147,38 @@ it.layer(
31473147
}),
31483148
);
31493149

3150+
it.effect("delivers a startup failure message after the attach snapshot", () =>
3151+
Effect.gen(function* () {
3152+
const { manager, ptyAdapter } = yield* createManager({
3153+
shellResolver: () => "/bin/sh",
3154+
env: {},
3155+
});
3156+
ptyAdapter.spawnFailures.push(
3157+
...Array.from({ length: 10 }, () => new Error("spawn unavailable")),
3158+
);
3159+
const received: TerminalAttachStreamEvent[] = [];
3160+
const stop = yield* manager.attachStream(
3161+
{ ...openInput(), replayBytes: DEFAULT_TERMINAL_REPLAY_BYTES },
3162+
(event) =>
3163+
Effect.sync(() => {
3164+
received.push(event);
3165+
}),
3166+
);
3167+
yield* Effect.addFinalizer(() => Effect.sync(stop));
3168+
const snapshotIndex = received.findIndex((event) => event.type === "snapshot");
3169+
const errorIndex = received.findIndex((event) => event.type === "error");
3170+
expect(received[snapshotIndex]).toMatchObject({
3171+
type: "snapshot",
3172+
snapshot: { status: "error" },
3173+
});
3174+
expect(errorIndex).toBeGreaterThan(snapshotIndex);
3175+
expect(received[errorIndex]).toMatchObject({
3176+
type: "error",
3177+
message: expect.stringContaining("Failed to spawn PTY process"),
3178+
});
3179+
}),
3180+
);
3181+
31503182
it.effect("streams extended persisted history before live terminal output", () =>
31513183
Effect.gen(function* () {
31523184
const { manager, logsDir } = yield* createManager();

‎apps/server/src/terminal/Manager.ts‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -186,7 +186,7 @@ export class TerminalManager extends Context.Service<
186186
) => Effect.Effect<void>,
187187
) => Effect.Effect<() => void, TerminalError>;
188188

189-
/** Read the current bounded snapshot for a slow-subscriber resync. */
189+
/** Read the current bounded terminal snapshot. */
190190
readonly readSnapshot: (
191191
input: TerminalClearInput,
192192
) => Effect.Effect<Option.Option<TerminalSessionSnapshot>>;
@@ -3297,6 +3297,7 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func
32973297
DEFAULT_ATTACH_BUFFERED_EVENT_LIMIT,
32983298
);
32993299
let capturedSnapshot = false;
3300+
let bootstrapError: Extract<TerminalEvent, { type: "error" }> | null = null;
33003301
let deliverLive = false;
33013302
return yield* Effect.gen(function* () {
33023303
// Old clients decode the attach stream against a union without the
@@ -3308,7 +3309,11 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func
33083309
return Effect.void;
33093310
}
33103311

3311-
if (!capturedSnapshot) return Effect.void;
3312+
if (!capturedSnapshot) {
3313+
// The snapshot carries status but no startup failure message.
3314+
if (event.type === "error") bootstrapError = event;
3315+
return Effect.void;
3316+
}
33123317
if (!deliverLive) return Queue.offer(bufferedEvents, event).pipe(Effect.asVoid);
33133318

33143319
const attachEvent = terminalEventToAttachEvent(event);
@@ -3341,6 +3346,11 @@ export const makeWithOptions = Effect.fn("TerminalManager.makeWithOptions")(func
33413346
"replay",
33423347
);
33433348

3349+
if (bootstrap.snapshot.status === "error" && bootstrapError !== null) {
3350+
yield* listener(bootstrapError, "replay");
3351+
}
3352+
bootstrapError = null;
3353+
33443354
if (bootstrap.replayHistory !== null && bootstrap.replayHistory.length > 0) {
33453355
for (const { data } of splitStringByUtf8Bytes(
33463356
bootstrap.replayHistory,

‎packages/contracts/src/terminal.test.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import {
77
EXTENDED_TERMINAL_REPLAY_BYTES,
88
MAX_TERMINAL_REPLAY_BYTES,
99
TerminalAttachInput,
10+
TerminalAttachTimeoutError,
1011
TerminalAttachStreamEvent,
1112
TerminalClearInput,
1213
TerminalCloseInput,
@@ -394,3 +395,15 @@ describe("TerminalEvent", () => {
394395
).toBe(true);
395396
});
396397
});
398+
399+
describe("TerminalAttachTimeoutError", () => {
400+
it("round-trips a stalled subscription failure", () => {
401+
const error = new TerminalAttachTimeoutError({
402+
threadId: "thread-1",
403+
terminalId: DEFAULT_TERMINAL_ID,
404+
});
405+
const decoded = decodeTerminalError(encodeTerminalError(error));
406+
expect(decoded).toEqual(error);
407+
expect(decoded.message).toBe("Terminal output subscription stalled. Reattach to resume.");
408+
});
409+
});

‎packages/contracts/src/terminal.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -394,7 +394,20 @@ export class TerminalResizeError extends Schema.TaggedError<TerminalResizeError>
394394
}
395395
}
396396

397+
export class TerminalAttachTimeoutError extends Schema.TaggedError<TerminalAttachTimeoutError>()(
398+
"TerminalAttachTimeoutError",
399+
{
400+
threadId: Schema.String,
401+
terminalId: Schema.String,
402+
},
403+
) {
404+
override get message() {
405+
return "Terminal output subscription stalled. Reattach to resume.";
406+
}
407+
}
408+
397409
export const TerminalError = Schema.Union([
410+
TerminalAttachTimeoutError,
398411
TerminalCwdError,
399412
TerminalHistoryError,
400413
TerminalSessionLookupError,

0 commit comments

Comments
 (0)