Skip to content

Commit 7c5adc4

Browse files
committed
fix(server): wait for launchd shutdown safely
1 parent 11f0513 commit 7c5adc4

2 files changed

Lines changed: 372 additions & 60 deletions

File tree

‎apps/server/src/cloud/bootService.test.ts‎

Lines changed: 220 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,14 @@ import {
77
HostProcessUserId,
88
} from "@t3tools/shared/hostProcess";
99
import * as ConfigProvider from "effect/ConfigProvider";
10+
import * as Deferred from "effect/Deferred";
1011
import * as Duration from "effect/Duration";
1112
import * as Effect from "effect/Effect";
1213
import * as FileSystem from "effect/FileSystem";
14+
import * as Fiber from "effect/Fiber";
1315
import * as Layer from "effect/Layer";
1416
import * as Path from "effect/Path";
17+
import * as TestClock from "effect/testing/TestClock";
1518
import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner";
1619

1720
import * as ProcessRunner from "../processRunner.ts";
@@ -57,6 +60,56 @@ const macPlan = {
5760
unitPath: "/Users/theo/Library/LaunchAgents/com.t3tools.t3code.service.plist",
5861
};
5962

63+
const launchdServiceTarget = "gui/501/com.t3tools.t3code.service";
64+
const launchdNotLoadedMessage =
65+
'Could not find service "com.t3tools.t3code.service" in domain for user gui: 501';
66+
67+
const processResult = (input?: {
68+
readonly code?: number;
69+
readonly stdout?: string;
70+
readonly stderr?: string;
71+
}): ProcessRunner.ProcessRunOutput => ({
72+
stdout: input?.stdout ?? "",
73+
stderr: input?.stderr ?? "",
74+
code: ChildProcessSpawner.ExitCode(input?.code ?? 0),
75+
timedOut: false,
76+
stdoutTruncated: false,
77+
stderrTruncated: false,
78+
stdoutInvalidUtf8: false,
79+
stderrInvalidUtf8: false,
80+
});
81+
82+
it("recognizes only launchctl's exact service-not-loaded response", () => {
83+
expect(
84+
BootService.isConfirmedLaunchdNotLoaded(
85+
processResult({ code: 113, stderr: `Bad request.\n${launchdNotLoadedMessage}\n` }),
86+
[launchdNotLoadedMessage],
87+
),
88+
).toBe(true);
89+
expect(
90+
BootService.isConfirmedLaunchdNotLoaded(
91+
processResult({ code: 1, stderr: "Boot-out failed: 1: Operation not permitted" }),
92+
[launchdNotLoadedMessage],
93+
),
94+
).toBe(false);
95+
expect(
96+
BootService.isConfirmedLaunchdNotLoaded(
97+
processResult({ code: 125, stderr: "Could not find domain for user gui: 501" }),
98+
[launchdNotLoadedMessage],
99+
),
100+
).toBe(false);
101+
expect(
102+
BootService.isConfirmedLaunchdBootoutNotLoaded(
103+
processResult({ code: 3, stderr: "Boot-out failed: 3: No such process\n" }),
104+
),
105+
).toBe(true);
106+
expect(
107+
BootService.isConfirmedLaunchdBootoutNotLoaded(
108+
processResult({ code: 1, stderr: "Boot-out failed: 1: Operation not permitted\n" }),
109+
),
110+
).toBe(false);
111+
});
112+
60113
it("keeps launchd pinned to the stable launcher rather than a versioned server", () => {
61114
const plist = BootService.renderBootServicePlist(macPlan, { homeDir: "/Users/theo" });
62115

@@ -116,23 +169,37 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* (
116169

117170
const commands: string[] = [];
118171
const timeouts = new Map<string, unknown>();
119-
const control: { failCommand: string | undefined } = { failCommand: undefined };
172+
const control: {
173+
failCommand: string | undefined;
174+
fixtures: Map<string, (call: number) => ProcessRunner.ProcessRunOutput>;
175+
callCounts: Map<string, number>;
176+
signals: Map<string, Deferred.Deferred<void>>;
177+
} = {
178+
failCommand: undefined,
179+
fixtures: new Map(),
180+
callCounts: new Map(),
181+
signals: new Map(),
182+
};
120183
const runner = ProcessRunner.ProcessRunner.of({
121184
run: (input) =>
122-
Effect.sync(() => {
185+
Effect.gen(function* () {
123186
const command = `${input.command} ${input.args.join(" ")}`;
124187
commands.push(command);
125188
timeouts.set(command, input.timeout);
126-
return {
127-
stdout: input.args[1] === "--version" ? "t3 v1.2.3\n" : "",
128-
stderr: "",
129-
code: ChildProcessSpawner.ExitCode(command === control.failCommand ? 1 : 0),
130-
timedOut: false,
131-
stdoutTruncated: false,
132-
stderrTruncated: false,
133-
stdoutInvalidUtf8: false,
134-
stderrInvalidUtf8: false,
135-
};
189+
const call = (control.callCounts.get(command) ?? 0) + 1;
190+
control.callCounts.set(command, call);
191+
const signal = control.signals.get(command);
192+
if (signal !== undefined) yield* Deferred.succeed(signal, undefined);
193+
const fixture = control.fixtures.get(command);
194+
if (fixture) return fixture(call);
195+
if (command === control.failCommand) return processResult({ code: 1 });
196+
if (command === `launchctl print ${launchdServiceTarget}`) {
197+
return processResult({
198+
code: 113,
199+
stderr: `Bad request.\n${launchdNotLoadedMessage}\n`,
200+
});
201+
}
202+
return processResult({ stdout: input.args[1] === "--version" ? "t3 v1.2.3\n" : "" });
136203
}),
137204
});
138205
const service = yield* BootService.make({
@@ -276,9 +343,9 @@ it.layer(NodeServices.layer)("boot service install", (it) => {
276343
expect((yield* service.status).installed).toBe(false);
277344
expect(commands.some((command) => command.startsWith("npm "))).toBe(false);
278345
expect(commands.some((command) => command.startsWith("systemctl "))).toBe(false);
279-
// A bootout can block up to the plist's 90s ExitTimeOut; the runner's
280-
// 60s default would cancel it and let bootstrap race a loaded job.
281-
expect(timeouts.get("launchctl bootout --wait gui/501/com.t3tools.t3code.service")).toEqual(
346+
// The bootout command and subsequent bounded print verification both
347+
// allow launchd's 90s ExitTimeOut to elapse.
348+
expect(timeouts.get("launchctl bootout gui/501/com.t3tools.t3code.service")).toEqual(
282349
Duration.seconds(120),
283350
);
284351
}),
@@ -295,22 +362,154 @@ it.layer(NodeServices.layer)("boot service install", (it) => {
295362
const error = yield* service.install.pipe(Effect.flip);
296363
expect(error._tag).toBe("BootServiceCommandError");
297364
expect(commands.filter((command) => command.startsWith("launchctl "))).toEqual([
298-
"launchctl bootout --wait gui/501/com.t3tools.t3code.service",
365+
"launchctl bootout gui/501/com.t3tools.t3code.service",
366+
"launchctl print gui/501/com.t3tools.t3code.service",
299367
"launchctl enable gui/501/com.t3tools.t3code.service",
300368
`launchctl bootstrap gui/501 ${plistPath}`,
301369
`launchctl bootstrap gui/501 ${plistPath}`,
302370
]);
303371
}),
304372
);
305373

306-
it.effect("ignores a bootout for an agent that is not loaded", () =>
374+
it.effect("removes the launch agent when its GUI domain is absent", () =>
307375
Effect.gen(function* () {
308-
const { service, control } = yield* makeHarness("darwin");
376+
const { service, fs, control } = yield* makeHarness("darwin");
377+
const plan = yield* service.install;
378+
control.fixtures.set("launchctl bootout gui/501/com.t3tools.t3code.service", () =>
379+
processResult({ code: 125, stderr: "Could not find domain for user gui: 501" }),
380+
);
381+
control.fixtures.set(`launchctl print ${launchdServiceTarget}`, () =>
382+
processResult({ code: 125, stderr: "Could not find domain for user gui: 501" }),
383+
);
384+
385+
expect(yield* service.uninstall).toBe(true);
386+
expect(yield* fs.exists(plan.unitPath)).toBe(false);
387+
}),
388+
);
389+
390+
it.effect("accepts a bootout only when launchd confirms the agent is already absent", () =>
391+
Effect.gen(function* () {
392+
const { service, commands, control } = yield* makeHarness("darwin");
309393
yield* service.install;
310-
control.failCommand = "launchctl bootout --wait gui/501/com.t3tools.t3code.service";
394+
commands.length = 0;
395+
control.fixtures.set("launchctl bootout gui/501/com.t3tools.t3code.service", () =>
396+
processResult({ code: 3, stderr: "Boot-out failed: 3: No such process\n" }),
397+
);
311398

312399
yield* service.install;
313400
expect((yield* service.status).current).toBe(true);
401+
expect(commands.filter((command) => command.startsWith("launchctl ")).slice(0, 2)).toEqual([
402+
"launchctl bootout gui/501/com.t3tools.t3code.service",
403+
"launchctl print gui/501/com.t3tools.t3code.service",
404+
]);
405+
}),
406+
);
407+
408+
it.effect("waits for a draining launch agent before bootstrap", () =>
409+
Effect.gen(function* () {
410+
const { service, commands, control } = yield* makeHarness("darwin");
411+
yield* service.install;
412+
commands.length = 0;
413+
control.callCounts.clear();
414+
const firstPrint = yield* Deferred.make<void>();
415+
control.signals.set(`launchctl print ${launchdServiceTarget}`, firstPrint);
416+
control.fixtures.set(`launchctl print ${launchdServiceTarget}`, (call) =>
417+
call === 1
418+
? processResult()
419+
: processResult({
420+
code: 113,
421+
stderr: `Bad request.\n${launchdNotLoadedMessage}\n`,
422+
}),
423+
);
424+
425+
const installFiber = yield* service.install.pipe(Effect.forkChild);
426+
yield* Deferred.await(firstPrint);
427+
yield* Effect.yieldNow;
428+
yield* TestClock.adjust(Duration.millis(100));
429+
yield* Fiber.join(installFiber);
430+
431+
expect(commands.filter((command) => command.startsWith("launchctl ")).slice(0, 4)).toEqual([
432+
"launchctl bootout gui/501/com.t3tools.t3code.service",
433+
"launchctl print gui/501/com.t3tools.t3code.service",
434+
"launchctl print gui/501/com.t3tools.t3code.service",
435+
"launchctl enable gui/501/com.t3tools.t3code.service",
436+
]);
437+
}).pipe(Effect.provide(TestClock.layer())),
438+
);
439+
440+
it.effect("times out instead of bootstrapping while a launch agent remains loaded", () =>
441+
Effect.gen(function* () {
442+
const { service, commands, control } = yield* makeHarness("darwin");
443+
yield* service.install;
444+
commands.length = 0;
445+
control.callCounts.clear();
446+
const firstPrint = yield* Deferred.make<void>();
447+
control.signals.set(`launchctl print ${launchdServiceTarget}`, firstPrint);
448+
control.fixtures.set(`launchctl print ${launchdServiceTarget}`, () => processResult());
449+
450+
const installFiber = yield* service.install.pipe(Effect.forkChild);
451+
yield* Deferred.await(firstPrint);
452+
yield* Effect.yieldNow;
453+
yield* TestClock.adjust(Duration.seconds(120));
454+
const error = yield* Fiber.join(installFiber).pipe(Effect.flip);
455+
456+
expect(error._tag).toBe("BootServiceCommandError");
457+
expect(error.message).toContain("timed out while waiting for the launch agent to stop");
458+
expect(commands).not.toContain("launchctl enable gui/501/com.t3tools.t3code.service");
459+
}).pipe(Effect.provide(TestClock.layer())),
460+
);
461+
462+
it.effect("surfaces bootout permission failures while the launch agent remains loaded", () =>
463+
Effect.gen(function* () {
464+
const { service, commands, control } = yield* makeHarness("darwin");
465+
yield* service.install;
466+
commands.length = 0;
467+
control.fixtures.set("launchctl bootout gui/501/com.t3tools.t3code.service", () =>
468+
processResult({ code: 1, stderr: "Boot-out failed: 1: Operation not permitted\n" }),
469+
);
470+
471+
const error = yield* service.install.pipe(Effect.flip);
472+
473+
expect(error._tag).toBe("BootServiceCommandError");
474+
expect(error._tag === "BootServiceCommandError" ? error.step : undefined).toBe(
475+
"stopping the installed launch agent",
476+
);
477+
expect(commands).not.toContain("launchctl enable gui/501/com.t3tools.t3code.service");
478+
}),
479+
);
480+
481+
it.effect("surfaces launchd domain failures during stop verification", () =>
482+
Effect.gen(function* () {
483+
const { service, commands, control } = yield* makeHarness("darwin");
484+
yield* service.install;
485+
commands.length = 0;
486+
control.fixtures.set(`launchctl print ${launchdServiceTarget}`, () =>
487+
processResult({ code: 125, stderr: "Could not find domain for user gui: 501" }),
488+
);
489+
490+
const error = yield* service.install.pipe(Effect.flip);
491+
492+
expect(error._tag).toBe("BootServiceCommandError");
493+
expect(error._tag === "BootServiceCommandError" ? error.step : undefined).toBe(
494+
"checking whether the launch agent stopped",
495+
);
496+
expect(commands).not.toContain("launchctl enable gui/501/com.t3tools.t3code.service");
497+
}),
498+
);
499+
500+
it.effect("surfaces launchd enable failures", () =>
501+
Effect.gen(function* () {
502+
const { service, control } = yield* makeHarness("darwin");
503+
control.fixtures.set("launchctl enable gui/501/com.t3tools.t3code.service", () =>
504+
processResult({ code: 1, stderr: "Enable failed: 1: Operation not permitted\n" }),
505+
);
506+
507+
const error = yield* service.install.pipe(Effect.flip);
508+
509+
expect(error._tag).toBe("BootServiceCommandError");
510+
expect(error._tag === "BootServiceCommandError" ? error.step : undefined).toBe(
511+
"enabling the launch agent",
512+
);
314513
}),
315514
);
316515

@@ -336,7 +535,8 @@ it.layer(NodeServices.layer)("boot service install", (it) => {
336535
expect((yield* service.install.pipe(Effect.flip))._tag).toBe("BootServiceUpdatePendingError");
337536
expect(serviceStateHasPendingUpdate(yield* fs.readFileString(statePath))).toBe(true);
338537
expect(commands.filter((command) => command.startsWith("launchctl "))).toEqual([
339-
"launchctl bootout --wait gui/501/com.t3tools.t3code.service",
538+
"launchctl bootout gui/501/com.t3tools.t3code.service",
539+
"launchctl print gui/501/com.t3tools.t3code.service",
340540
`launchctl bootstrap gui/501 ${plistPath}`,
341541
]);
342542
}),

0 commit comments

Comments
 (0)