Skip to content

Commit 455dcf4

Browse files
fix(server): close remaining data-directory lock races
CodeRabbit on the current HEAD: record the bound port as soon as the socket exists, make the lock a dependency of persistence as well as HTTP, and serialize stale-lock takeover so two reclaimers cannot both delete and recreate the same claim. Co-authored-by: Cursor <[email protected]>
1 parent 88780e3 commit 455dcf4

3 files changed

Lines changed: 133 additions & 23 deletions

File tree

‎apps/server/src/server.ts‎

Lines changed: 20 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -225,9 +225,11 @@ const RelayClientLive = Layer.unwrap(
225225
/**
226226
* Claims the data directory before anything binds a port or opens the database.
227227
*
228-
* Provided into `HttpServerLive` rather than merged alongside it so the ordering
229-
* is structural: the lock is a dependency of the thing it protects, and a second
230-
* server cannot reach a listening socket while another holds the directory.
228+
* Provided into both `HttpServerLive` and `RuntimeDependenciesLive` (same layer
229+
* identity, so one acquire) so the lock precedes the HTTP bind *and* SQLite
230+
* open. Providing it only into the HTTP layer left persistence free to
231+
* initialize in parallel and write `state.sqlite` before a second process
232+
* refused.
231233
*/
232234
const ServerSingletonLive = Layer.effectDiscard(
233235
Effect.gen(function* () {
@@ -565,6 +567,7 @@ const RuntimeDependenciesLive = RuntimeCoreDependenciesLive.pipe(
565567
Layer.provideMerge(RemoteOpenTargets.layer),
566568
Layer.provideMerge(ServerLifecycleEvents.layer),
567569
Layer.provide(NetService.layer),
570+
Layer.provide(ServerSingletonLive),
568571
);
569572

570573
const commandReadinessLayer = HttpRouter.middleware(
@@ -620,9 +623,22 @@ const makeServerLayer = Layer.unwrap(
620623

621624
const httpListeningLayer = Layer.effectDiscard(
622625
Effect.gen(function* () {
623-
yield* HttpServer.HttpServer;
626+
const server = yield* HttpServer.HttpServer;
624627
const startup = yield* ServerRuntimeStartup.ServerRuntimeStartup;
625628
yield* startup.markHttpListening;
629+
const address = server.address;
630+
if (typeof address === "string" || !("port" in address)) {
631+
return;
632+
}
633+
// Stamp the port as soon as the socket is bound. Waiting until
634+
// runtimeStateLayer's awaitActivation left a window where the server
635+
// was listening but a second launch's refusal could not name the port.
636+
yield* ServerSingleton.serverLockPath(config.stateDir).pipe(
637+
Effect.flatMap((lockPath) =>
638+
ServerSingleton.recordServerLockPort(lockPath, address.port),
639+
),
640+
Effect.ignore,
641+
);
626642
}),
627643
);
628644
const runtimeStateLayer = Layer.effectDiscard(
@@ -638,16 +654,6 @@ const makeServerLayer = Layer.unwrap(
638654

639655
const launcher = yield* ServiceLauncherClient.ServiceLauncherClient;
640656

641-
// Stamp the port onto the lock we already hold. It is only ever read
642-
// by a *later* server's refusal message, which turns "something else
643-
// is running" into an address the user can open.
644-
yield* ServerSingleton.serverLockPath(config.stateDir).pipe(
645-
Effect.flatMap((lockPath) =>
646-
ServerSingleton.recordServerLockPort(lockPath, address.port),
647-
),
648-
Effect.ignore,
649-
);
650-
651657
const state = yield* makePersistedServerRuntimeState({
652658
config,
653659
port: address.port,

‎apps/server/src/serverSingleton.test.ts‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -314,6 +314,46 @@ layer("serverSingleton", (it) => {
314314
}),
315315
);
316316

317+
it.effect("only one of two concurrent reclaimers acquires a stale directory", () =>
318+
Effect.gen(function* () {
319+
const stateDir = yield* makeStateDir();
320+
const fs = yield* FileSystem.FileSystem;
321+
const lockPath = yield* serverLockPath(stateDir);
322+
yield* fs.writeFileString(lockPath, staleHolder(4194304));
323+
const past = DateTime.toDateUtc(
324+
DateTime.subtractDuration(yield* DateTime.now, Duration.minutes(1)),
325+
);
326+
yield* fs.utimes(lockPath, past, past);
327+
328+
// Two starters both pass the last stale read, then both used to
329+
// `fs.remove` — the second deleted the first's new claim and both
330+
// proceeded. The exclusive reclaim gate lets only one touch the path.
331+
yield* Effect.scoped(
332+
Effect.gen(function* () {
333+
const attempt = acquireServerSingleton(stateDir).pipe(
334+
Effect.match({
335+
onFailure: (error) => ({ ok: false as const, error }),
336+
onSuccess: (held) => ({ ok: true as const, held }),
337+
}),
338+
);
339+
// Shared scope keeps the winner's lock held until both fibers finish,
340+
// otherwise the winner would release before the loser observed it.
341+
const first = yield* Effect.forkChild(attempt);
342+
const second = yield* Effect.forkChild(attempt);
343+
const outcomes = [yield* Fiber.join(first), yield* Fiber.join(second)];
344+
const wins = outcomes.filter((outcome) => outcome.ok);
345+
const losses = outcomes.filter((outcome) => !outcome.ok);
346+
assert.strictEqual(wins.length, 1);
347+
assert.strictEqual(losses.length, 1);
348+
const loss = losses[0];
349+
if (loss !== undefined && !loss.ok) {
350+
assert.strictEqual(loss.error._tag, "ServerAlreadyRunningError");
351+
}
352+
}),
353+
);
354+
}),
355+
);
356+
317357
it.effect("a partial metadata update is never readable as an empty owner", () =>
318358
Effect.gen(function* () {
319359
const stateDir = yield* makeStateDir();

‎apps/server/src/serverSingleton.ts‎

Lines changed: 73 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,33 @@ const RECLAIM_OBSERVATION_DELAY_MILLIS = 200;
156156
const isAlreadyExists = (error: PlatformError.PlatformError): boolean =>
157157
error.reason._tag === "AlreadyExists";
158158

159+
const isNotFound = (error: PlatformError.PlatformError): boolean => error.reason._tag === "NotFound";
160+
161+
const reclaimGatePath = (lockPath: string): string => `${lockPath}.reclaim`;
162+
163+
/**
164+
* Serialize stale-lock takeover. Two reclaimers can both finish the last
165+
* stale read, then both `remove` — the second deletes the first's fresh
166+
* claim. An exclusive gate file makes exactly one reclaimer allowed to
167+
* touch `server.lock`; everyone else backs off and retries the create.
168+
*/
169+
const takeReclaimGate = Effect.fn("serverSingleton.takeReclaimGate")(function* (lockPath: string) {
170+
const fs = yield* FileSystem.FileSystem;
171+
const gatePath = reclaimGatePath(lockPath);
172+
const existing = yield* fs.readFileString(gatePath).pipe(
173+
Effect.catch((error) => (isNotFound(error) ? Effect.succeed(undefined) : Effect.fail(error))),
174+
);
175+
if (existing !== undefined) {
176+
const gatePid = Number.parseInt(existing.trim(), 10);
177+
if (processIsAlive(gatePid)) return false;
178+
yield* fs.remove(gatePath, { force: true }).pipe(Effect.ignore);
179+
}
180+
return yield* fs.writeFileString(gatePath, `${process.pid}\n`, { flag: "wx" }).pipe(
181+
Effect.as(true),
182+
Effect.catch((error) => (isAlreadyExists(error) ? Effect.succeed(false) : Effect.fail(error))),
183+
);
184+
});
185+
159186
/**
160187
* A pre-lock server is live against this directory. Failure-phase marker rather
161188
* than an error: `serverRuntimeState.ts` already owns the file's decode-error
@@ -250,16 +277,53 @@ const claimLock = Effect.fn("serverSingleton.claimLock")(function* (input: {
250277
// here, not on a later manual retry. Bounded, so a lock another starter
251278
// keeps recreating (or a permissions wall keeps failing to remove for)
252279
// still surfaces as itself.
253-
yield* fs.remove(input.lockPath, { force: true });
254-
reclaimRefreshes = 0;
255-
reclaimCycles += 1;
256-
if (reclaimCycles >= MAX_RECLAIM_CYCLES) {
257-
return new ServerLockUnavailableError({
258-
lockPath: input.lockPath,
259-
attempts: reclaimCycles,
260-
});
280+
const wonGate = yield* takeReclaimGate(input.lockPath);
281+
if (!wonGate) {
282+
reclaimRefreshes = 0;
283+
reclaimCycles += 1;
284+
if (reclaimCycles >= MAX_RECLAIM_CYCLES) {
285+
return new ServerLockUnavailableError({
286+
lockPath: input.lockPath,
287+
attempts: reclaimCycles,
288+
});
289+
}
290+
continue;
291+
}
292+
const gatePath = reclaimGatePath(input.lockPath);
293+
const takeover = yield* Effect.gen(function* () {
294+
const holderAfterGate = yield* readHolder(input.lockPath);
295+
if (holderAfterGate !== undefined && processIsAlive(holderAfterGate.pid)) {
296+
return new ServerAlreadyRunningError({
297+
stateDir: input.stateDir,
298+
lockPath: input.lockPath,
299+
holderPid: holderAfterGate.pid,
300+
...(holderAfterGate.port === undefined ? {} : { holderPort: holderAfterGate.port }),
301+
holderStartedAt: holderAfterGate.startedAt,
302+
});
303+
}
304+
yield* fs.remove(input.lockPath, { force: true });
305+
const created = yield* fs.writeFileString(input.lockPath, payload, { flag: "wx" }).pipe(
306+
Effect.as(true),
307+
Effect.catch((error) =>
308+
isAlreadyExists(error) ? Effect.succeed(false) : Effect.fail(error),
309+
),
310+
);
311+
return created ? undefined : "retry";
312+
}).pipe(
313+
Effect.ensuring(fs.remove(gatePath, { force: true }).pipe(Effect.ignore)),
314+
);
315+
if (takeover === "retry") {
316+
reclaimRefreshes = 0;
317+
reclaimCycles += 1;
318+
if (reclaimCycles >= MAX_RECLAIM_CYCLES) {
319+
return new ServerLockUnavailableError({
320+
lockPath: input.lockPath,
321+
attempts: reclaimCycles,
322+
});
323+
}
324+
continue;
261325
}
262-
continue;
326+
return takeover;
263327
}
264328

265329
reclaimRefreshes += 1;

0 commit comments

Comments
 (0)