Skip to content

Commit e85539a

Browse files
tachytelicdetonationclaudejuliusmarminge
authored
fix(server): keep helper processes from outliving the backend (#12513)
Co-authored-by: Claude Fable 5.1 <[email protected]> Co-authored-by: Julius Marminge <[email protected]>
1 parent af17eb9 commit e85539a

7 files changed

Lines changed: 281 additions & 4 deletions

File tree

‎apps/server/src/processRunner.test.ts‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,24 @@
1+
import * as NodeServices from "@effect/platform-node/NodeServices";
12
import { describe, expect, it } from "@effect/vitest";
23
import * as Deferred from "effect/Deferred";
34
import * as Duration from "effect/Duration";
45
import * as Effect from "effect/Effect";
56
import * as Fiber from "effect/Fiber";
7+
import * as FileSystem from "effect/FileSystem";
68
import * as Layer from "effect/Layer";
9+
import * as Path from "effect/Path";
710
import * as PlatformError from "effect/PlatformError";
811
import * as Sink from "effect/Sink";
912
import * as Stream from "effect/Stream";
1013
import { TestClock } from "effect/testing";
1114
import { ChildProcessSpawner } from "effect/process";
15+
import * as ChildProcess from "effect/process/ChildProcess";
1216
import * as HostProcess from "@t3tools/shared/HostProcess";
1317
import { SpawnExecutableResolution } from "@t3tools/shared/shell";
1418

1519
import * as ProcessRunner from "./processRunner.ts";
20+
import { writeFakeCli } from "@t3tools/provider-testing/fakeCli";
21+
import { makeProcessProbe, processProbeSource } from "./testUtils/processProbe.ts";
1622

1723
type ChildProcessCommand = {
1824
readonly command: string;
@@ -398,6 +404,124 @@ describe("runProcess", () => {
398404
}),
399405
);
400406

407+
it.effect("kills a helper that ignores SIGTERM when its run times out", () =>
408+
Effect.gen(function* () {
409+
const fs = yield* FileSystem.FileSystem;
410+
const directory = yield* fs.makeTempDirectoryScoped({ prefix: "t3code-process-runner-" });
411+
const probe = yield* makeProcessProbe;
412+
const helper = writeFakeCli({
413+
directory,
414+
name: "stubborn",
415+
source: `process.on("SIGTERM", () => {}); ${processProbeSource(probe.port)}`,
416+
});
417+
const runner = yield* ProcessRunner.ProcessRunner;
418+
const fiber = yield* runner
419+
.run({ command: helper, args: [], timeout: "1 second" })
420+
.pipe(Effect.flip, Effect.forkScoped);
421+
yield* probe.started;
422+
423+
yield* TestClock.adjust("1 second");
424+
yield* probe.exited;
425+
expect((yield* Fiber.join(fiber))._tag).toBe("ProcessTimeoutError");
426+
}).pipe(Effect.provide(ProcessRunner.layer.pipe(Layer.provideMerge(NodeServices.layer)))),
427+
);
428+
429+
for (const wrapperExit of ["timeout", "success"] as const) {
430+
it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")(
431+
`terminates descendants after wrapper ${wrapperExit}`,
432+
() =>
433+
Effect.gen(function* () {
434+
const fs = yield* FileSystem.FileSystem;
435+
const directory = yield* fs.makeTempDirectoryScoped({ prefix: "t3code-process-tree-" });
436+
const probe = yield* makeProcessProbe;
437+
const helper = writeFakeCli({
438+
directory,
439+
name: "wrapper",
440+
source: `
441+
import { spawn } from "node:child_process";
442+
process.on("SIGTERM", () => {});
443+
const child = spawn(process.execPath, ["--input-type=module", "-e", ${JSON.stringify(`
444+
${processProbeSource(probe.port)}
445+
probe.once("connect", () => process.send("ready"));
446+
`)}], { stdio: ["ignore", "ignore", "ignore", "ipc"] });
447+
${wrapperExit === "success" ? 'child.once("message", () => process.exit(0));' : 'child.on("exit", () => process.exit(0));'}
448+
`,
449+
});
450+
const runner = yield* ProcessRunner.ProcessRunner;
451+
const fiber = yield* runner
452+
.run({ command: helper, args: [], timeout: "1 second" })
453+
.pipe(Effect.result, Effect.forkScoped);
454+
yield* probe.started;
455+
456+
if (wrapperExit === "timeout") yield* TestClock.adjust("1 second");
457+
yield* probe.exited;
458+
const result = yield* Fiber.join(fiber);
459+
if (wrapperExit === "timeout") {
460+
expect(result).toMatchObject({
461+
_tag: "Failure",
462+
failure: { _tag: "ProcessTimeoutError" },
463+
});
464+
} else {
465+
expect(result).toMatchObject({
466+
_tag: "Success",
467+
success: { code: 0, timedOut: false },
468+
});
469+
}
470+
}).pipe(Effect.provide(ProcessRunner.layer.pipe(Layer.provideMerge(NodeServices.layer)))),
471+
);
472+
}
473+
474+
it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")(
475+
"exits the backend and terminates stubborn helpers on graceful group shutdown",
476+
() =>
477+
Effect.gen(function* () {
478+
const fs = yield* FileSystem.FileSystem;
479+
const directory = yield* fs.makeTempDirectoryScoped({ prefix: "t3code-process-backend-" });
480+
const probe = yield* makeProcessProbe;
481+
const helper = writeFakeCli({
482+
directory,
483+
name: "helper",
484+
source: `process.on("SIGTERM", () => {}); ${processProbeSource(probe.port)}`,
485+
});
486+
const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
487+
const path = yield* Path.Path;
488+
const backend = yield* spawner.spawn(
489+
ChildProcess.make(
490+
process.execPath,
491+
[
492+
"--experimental-strip-types",
493+
"--input-type=module",
494+
"-e",
495+
`
496+
import * as Effect from "effect/Effect";
497+
import * as Layer from "effect/Layer";
498+
import * as NodeServices from "@effect/platform-node/NodeServices";
499+
import * as NodeRuntime from "@effect/platform-node/NodeRuntime";
500+
import * as ProcessRunner from ${JSON.stringify(new URL("./processRunner.ts", import.meta.url).href)};
501+
NodeRuntime.runMain(Effect.gen(function* () {
502+
const runner = yield* ProcessRunner.ProcessRunner;
503+
yield* runner.run({ command: ${JSON.stringify(helper)}, args: [] });
504+
}).pipe(Effect.provide(ProcessRunner.layer.pipe(Layer.provideMerge(NodeServices.layer)))));
505+
`,
506+
],
507+
{
508+
detached: true,
509+
forceKillAfter: "1 second",
510+
cwd: yield* path.fromFileUrl(new URL("..", import.meta.url)),
511+
},
512+
),
513+
);
514+
yield* probe.started;
515+
516+
// Desktop allows five seconds for NodeRuntime to finish scope finalizers.
517+
process.kill(-backend.pid, "SIGTERM");
518+
yield* Effect.all(
519+
[backend.exitCode.pipe(Effect.timeout("4 seconds"), TestClock.withLive), probe.exited],
520+
{ concurrency: "unbounded" },
521+
);
522+
}).pipe(Effect.provide(NodeServices.layer)),
523+
);
524+
401525
it.effect("returns a synthetic timed out result when timeoutBehavior is timedOutResult", () =>
402526
Effect.gen(function* () {
403527
const spawner = makeSpawner(() =>

‎apps/server/src/processRunner.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,8 @@ export class ProcessRunner extends Context.Service<
146146
>()("t3/processRunner") {}
147147

148148
const DEFAULT_TIMEOUT = "60 seconds";
149+
// Keep SIGTERM-ignoring helpers from blocking timeouts or graceful shutdown.
150+
const FORCE_KILL_AFTER = "1 second";
149151
const DEFAULT_MAX_OUTPUT_BYTES = 8 * 1024 * 1024;
150152

151153
export { isWindowsCommandNotFound } from "@t3tools/provider-core/server/snapshotProbe";
@@ -304,6 +306,7 @@ const runProcessCore = Effect.fn("processRunner.runProcessCore")(function* (
304306
}
305307
: {}),
306308
shell: spawnCommand.shell,
309+
forceKillAfter: FORCE_KILL_AFTER,
307310
}),
308311
)
309312
.pipe(
Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
// @effect-diagnostics nodeBuiltinImport:off - a socket observes fixture processes after their parent exits.
2+
import * as NodeNet from "node:net";
3+
import * as Deferred from "effect/Deferred";
4+
import * as Effect from "effect/Effect";
5+
import * as TestClock from "effect/testing/TestClock";
6+
7+
export const makeProcessProbe = Effect.gen(function* () {
8+
const started = yield* Deferred.make<number>();
9+
const exited = yield* Deferred.make<void>();
10+
const sockets = new Set<NodeNet.Socket>();
11+
const server = NodeNet.createServer((socket) => {
12+
sockets.add(socket);
13+
let output = "";
14+
socket.on("data", (chunk) => {
15+
output += chunk.toString();
16+
if (output.includes("\n")) {
17+
const pid = Number(output.trim());
18+
if (Number.isSafeInteger(pid) && pid > 0) {
19+
Deferred.doneUnsafe(started, Effect.succeed(pid));
20+
}
21+
}
22+
});
23+
socket.on("close", () => {
24+
sockets.delete(socket);
25+
Deferred.doneUnsafe(exited, Effect.void);
26+
});
27+
});
28+
yield* Effect.acquireRelease(
29+
Effect.callback<void, Error>((resume) => {
30+
server.once("error", (error) => resume(Effect.fail(error)));
31+
server.listen(0, "127.0.0.1", () => resume(Effect.void));
32+
}),
33+
() =>
34+
Effect.sync(() => {
35+
for (const socket of sockets) socket.destroy();
36+
server.close();
37+
}),
38+
);
39+
const address = server.address();
40+
if (address === null || typeof address === "string") {
41+
return yield* Effect.die("Missing process probe port");
42+
}
43+
return {
44+
port: address.port,
45+
started: Deferred.await(started).pipe(
46+
Effect.timeout("4 seconds"),
47+
TestClock.withLive,
48+
Effect.tap((pid) =>
49+
Effect.addFinalizer(() =>
50+
Effect.gen(function* () {
51+
if (yield* Deferred.isDone(exited)) return;
52+
// Only a PID reported by this fixture is eligible for failure cleanup.
53+
yield* Effect.sync(() => {
54+
try {
55+
process.kill(pid, "SIGKILL");
56+
} catch (error) {
57+
if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error;
58+
}
59+
});
60+
}),
61+
),
62+
),
63+
),
64+
exited: Deferred.await(exited).pipe(Effect.timeout("4 seconds"), TestClock.withLive),
65+
};
66+
});
67+
68+
export const processProbeSource = (port: number) => `
69+
const { createConnection } = await import("node:net");
70+
const probe = createConnection(${port}, "127.0.0.1", () => {
71+
probe.write(process.pid + "\\n");
72+
});
73+
`;

‎apps/server/src/textGeneration/ClaudeTextGeneration.test.ts‎

Lines changed: 35 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,16 @@ import { ClaudeSettings, ProviderInstanceId } from "@t3tools/contracts";
44
import * as HostProcess from "@t3tools/shared/HostProcess";
55
import { createModelSelection } from "@t3tools/shared/model";
66
import * as Effect from "effect/Effect";
7+
import * as Fiber from "effect/Fiber";
78
import * as FileSystem from "effect/FileSystem";
89
import * as Layer from "effect/Layer";
910
import * as Path from "effect/Path";
1011
import * as Schema from "effect/Schema";
12+
import * as TestClock from "effect/testing/TestClock";
1113
import { expect } from "vite-plus/test";
1214

1315
import * as ServerConfig from "../config.ts";
16+
import { makeProcessProbe, processProbeSource } from "../testUtils/processProbe.ts";
1417
import {
1518
SYNTHETIC_CLAUDE_CAPABLE_MODEL,
1619
SYNTHETIC_CLAUDE_COLLIDING_ALIAS,
@@ -31,7 +34,7 @@ const layerClaudeTextGenerationTest = ServerConfig.ServerConfig.layerTest(proces
3134
// The stub behaviour lives in Node so the same implementation runs on Windows,
3235
// where a shebang file is not executable and would fall through to the real
3336
// Claude CLI on PATH; `writeFakeCli` picks the launcher shape per host.
34-
function makeFakeClaudeBinary(dir: string) {
37+
function makeFakeClaudeBinary(dir: string, lifecyclePort?: number) {
3538
return Effect.gen(function* () {
3639
const path = yield* Path.Path;
3740
const fs = yield* FileSystem.FileSystem;
@@ -45,7 +48,10 @@ function makeFakeClaudeBinary(dir: string) {
4548
directory: binDir,
4649
name: "claude",
4750
platform,
48-
source,
51+
source:
52+
lifecyclePort === undefined
53+
? source
54+
: `process.on("SIGTERM", () => {}); ${processProbeSource(lifecyclePort)}`,
4955
});
5056
return binDir;
5157
});
@@ -54,6 +60,7 @@ function makeFakeClaudeBinary(dir: string) {
5460
function withFakeClaudeEnv<A, E, R>(
5561
input: {
5662
output: string;
63+
lifecyclePort?: number;
5764
exitCode?: number;
5865
stderr?: string;
5966
argsMustContain?: string;
@@ -68,7 +75,7 @@ function withFakeClaudeEnv<A, E, R>(
6875
return Effect.gen(function* () {
6976
const fs = yield* FileSystem.FileSystem;
7077
const tempDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3code-claude-text-" });
71-
const binDir = yield* makeFakeClaudeBinary(tempDir);
78+
const binDir = yield* makeFakeClaudeBinary(tempDir, input.lifecyclePort);
7279
const pathDelimiter = (yield* HostProcess.isWindows) ? ";" : ":";
7380
const previousPath = process.env.PATH;
7481
const previousOutput = process.env.T3_FAKE_CLAUDE_OUTPUT;
@@ -192,6 +199,31 @@ function withFakeClaudeEnv<A, E, R>(
192199
}
193200

194201
it.layer(layerClaudeTextGenerationTest)("ClaudeTextGeneration", (it) => {
202+
it.effect("kills a stalled CLI when the request times out", () =>
203+
Effect.gen(function* () {
204+
const probe = yield* makeProcessProbe;
205+
yield* withFakeClaudeEnv({ output: "", lifecyclePort: probe.port }, (textGeneration) =>
206+
Effect.gen(function* () {
207+
const fiber = yield* textGeneration
208+
.generateThreadTitle({
209+
cwd: process.cwd(),
210+
message: "Name this thread",
211+
modelSelection: {
212+
instanceId: ProviderInstanceId.make("claudeAgent"),
213+
model: SYNTHETIC_CLAUDE_STANDARD_MODEL,
214+
},
215+
})
216+
.pipe(Effect.flip, Effect.forkScoped);
217+
yield* probe.started;
218+
219+
yield* TestClock.adjust("180 seconds");
220+
yield* probe.exited;
221+
expect((yield* Fiber.join(fiber)).detail).toBe("Claude CLI request timed out.");
222+
}),
223+
);
224+
}),
225+
);
226+
195227
it.effect("forwards Claude thinking settings without passing unsupported effort", () =>
196228
withFakeClaudeEnv(
197229
{

‎apps/server/src/textGeneration/ClaudeTextGeneration.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -234,6 +234,7 @@ export const makeClaudeTextGeneration = Effect.fn("makeClaudeTextGeneration")(fu
234234
stdin: {
235235
stream: Stream.encodeText(Stream.make(prompt)),
236236
},
237+
forceKillAfter: "1 second",
237238
});
238239

239240
const child = yield* commandSpawner

0 commit comments

Comments
 (0)