Skip to content

Commit da3a9c3

Browse files
committed
fix(connect): reclaim installer locks without removing replacements
1 parent 2a576da commit da3a9c3

2 files changed

Lines changed: 148 additions & 94 deletions

File tree

‎packages/shared/src/relayClient.test.ts‎

Lines changed: 86 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { sha256 } from "@noble/hashes/sha2";
22
import * as NodeServices from "@effect/platform-node/NodeServices";
33
import { describe, expect, it } from "@effect/vitest";
4+
import * as Clock from "effect/Clock";
45
import * as ConfigProvider from "effect/ConfigProvider";
56
import * as Deferred from "effect/Deferred";
67
import * as Effect from "effect/Effect";
@@ -12,6 +13,7 @@ import * as Option from "effect/Option";
1213
import * as PlatformError from "effect/PlatformError";
1314
import * as Sink from "effect/Sink";
1415
import * as Stream from "effect/Stream";
16+
import * as TestClock from "effect/testing/TestClock";
1517
import { HttpClient, HttpClientResponse } from "effect/http";
1618
import { ChildProcess, ChildProcessSpawner } from "effect/process";
1719
import * as HostProcess from "./HostProcess.ts";
@@ -157,12 +159,12 @@ describe("RelayClient", () => {
157159
const manager = yield* RelayClient.makeCloudflaredRelayClient({ baseDir }).pipe(
158160
Effect.provideService(FileSystem.FileSystem, {
159161
...fileSystem,
160-
writeFileString: (path, contents, options) =>
162+
rename: (from, to) =>
161163
fileSystem
162-
.writeFileString(path, contents, options)
164+
.rename(from, to)
163165
.pipe(
164166
Effect.tapError(() =>
165-
path === lockPath ? Deferred.succeed(contended, undefined) : Effect.void,
167+
to === lockPath ? Deferred.succeed(contended, undefined) : Effect.void,
166168
),
167169
),
168170
}),
@@ -184,44 +186,46 @@ describe("RelayClient", () => {
184186
),
185187
);
186188

187-
it.effect("keeps an install lock whose owner is still running", () =>
189+
it.effect.each(["live owner", "recent legacy"] as const)("keeps a lock with a %s", (kind) =>
188190
Effect.gen(function* () {
189191
const fileSystem = yield* FileSystem.FileSystem;
190192
const baseDir = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-cloudflared-test-" });
191-
const directory = `${baseDir}/tools/cloudflared/${RelayClient.CLOUDFLARED_VERSION}/linux-x64`;
192-
const lockPath = `${directory}/cloudflared.lock`;
193-
const token = `${process.pid}:still-running`;
194-
yield* fileSystem.makeDirectory(directory, { recursive: true });
195-
yield* fileSystem.writeFileString(lockPath, token);
196-
const contended = yield* Deferred.make<void>();
197-
const manager = yield* RelayClient.makeCloudflaredRelayClient({ baseDir }).pipe(
198-
Effect.provideService(FileSystem.FileSystem, {
199-
...fileSystem,
200-
writeFileString: (path, contents, options) =>
201-
fileSystem
202-
.writeFileString(path, contents, options)
203-
.pipe(
204-
Effect.tapError(() =>
205-
path === lockPath ? Deferred.succeed(contended, undefined) : Effect.void,
206-
),
207-
),
193+
const lockPath = `${baseDir}/tools/cloudflared/${RelayClient.CLOUDFLARED_VERSION}/linux-x64/cloudflared.lock`;
194+
const owner = `${process.pid}-still-running`;
195+
yield* fileSystem.makeDirectory(lockPath.slice(0, lockPath.lastIndexOf("/")), {
196+
recursive: true,
197+
});
198+
if (kind === "live owner") {
199+
yield* fileSystem.makeDirectory(lockPath);
200+
yield* fileSystem.writeFileString(`${lockPath}/${owner}`, "");
201+
} else {
202+
yield* fileSystem.writeFileString(lockPath, "");
203+
}
204+
const written = yield* fileSystem.stat(lockPath);
205+
yield* TestClock.setTime(
206+
Option.getOrThrow(written.mtime).getTime() + (kind === "live owner" ? 3_600_000 : 6_000),
207+
);
208+
const waiting = yield* Deferred.make<void>();
209+
const clock = yield* Clock.Clock;
210+
const manager = yield* RelayClient.makeCloudflaredRelayClient({
211+
baseDir,
212+
releaseAsset: testReleaseAsset,
213+
});
214+
const installing = yield* manager.install.pipe(
215+
Effect.provideService(Clock.Clock, {
216+
...clock,
217+
sleep: () => Deferred.succeed(waiting, undefined).pipe(Effect.andThen(Effect.never)),
208218
}),
219+
Effect.forkChild,
209220
);
210-
const installing = yield* manager.install.pipe(Effect.forkChild);
211-
yield* Deferred.await(contended);
221+
yield* Deferred.await(waiting);
212222
yield* Fiber.interrupt(installing);
213-
expect(yield* fileSystem.readFileString(lockPath)).toBe(token);
214-
}).pipe(
215-
Effect.scoped,
216-
Effect.provide(
217-
Layer.mergeAll(
218-
NodeServices.layer,
219-
layerHttpClient(new Uint8Array()),
220-
layerSpawner([]),
221-
layerHostRuntime({ PATH: "" }),
222-
),
223-
),
224-
),
223+
if (kind === "live owner") {
224+
expect(yield* fileSystem.readDirectory(lockPath)).toEqual([owner]);
225+
} else {
226+
expect(yield* fileSystem.readFileString(lockPath)).toBe("");
227+
}
228+
}).pipe(Effect.scoped, Effect.provide(layerInstallRuntime("linux"))),
225229
);
226230

227231
it.effect.skipIf(windowsHost)("replaces an install lock left by an exited installer", () =>
@@ -234,7 +238,8 @@ describe("RelayClient", () => {
234238
yield* fileSystem.makeDirectory(lockPath.slice(0, lockPath.lastIndexOf("/")), {
235239
recursive: true,
236240
});
237-
yield* fileSystem.writeFileString(lockPath, "2147483647:exited-installer");
241+
yield* fileSystem.makeDirectory(lockPath);
242+
yield* fileSystem.writeFileString(`${lockPath}/2147483647-exited-installer`, "");
238243
const manager = yield* RelayClient.makeCloudflaredRelayClient({
239244
baseDir,
240245
releaseAsset: testReleaseAsset,
@@ -257,9 +262,8 @@ describe("RelayClient", () => {
257262
});
258263
yield* fileSystem.writeFileString(lockPath, "");
259264
const written = yield* fileSystem.stat(lockPath);
260-
const mtime = Option.getOrElse(written.mtime, () => new Date(0)).getTime();
261-
// The test clock starts at the epoch, so a lock just written looks
262-
// newer than the clock. Move past its mtime by more than the grace.
265+
const mtime = Option.getOrThrow(written.mtime).getTime();
266+
// File mtimes use wall time; advance the test clock past legacy expiry.
263267
yield* TestClock.setTime(mtime + 60 * 60 * 1_000);
264268
const manager = yield* RelayClient.makeCloudflaredRelayClient({
265269
baseDir,
@@ -271,6 +275,46 @@ describe("RelayClient", () => {
271275
}).pipe(Effect.scoped, Effect.provide(layerInstallRuntime("linux"))),
272276
);
273277

278+
it.effect("keeps a replacement lock when a stale contender resumes cleanup", () =>
279+
Effect.gen(function* () {
280+
const fileSystem = yield* FileSystem.FileSystem;
281+
const baseDir = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-cloudflared-test-" });
282+
const lockPath = `${baseDir}/tools/cloudflared/${RelayClient.CLOUDFLARED_VERSION}/linux-x64/cloudflared.lock`;
283+
const abandoned = "2147483647-exited-installer";
284+
const replacement = `${process.pid}-replacement`;
285+
yield* fileSystem.makeDirectory(lockPath, { recursive: true });
286+
yield* fileSystem.writeFileString(`${lockPath}/${abandoned}`, "");
287+
const checked = yield* Deferred.make<void>();
288+
let replaced = false;
289+
const manager = yield* RelayClient.makeCloudflaredRelayClient({ baseDir }).pipe(
290+
Effect.provideService(FileSystem.FileSystem, {
291+
...fileSystem,
292+
readDirectory: (path) =>
293+
fileSystem.readDirectory(path).pipe(
294+
Effect.tap((entries) =>
295+
Effect.gen(function* () {
296+
if (path !== lockPath) return;
297+
if (replaced) {
298+
expect(entries).toEqual([replacement]);
299+
yield* Deferred.succeed(checked, undefined);
300+
return;
301+
}
302+
replaced = true;
303+
yield* fileSystem.rename(lockPath, `${lockPath}.old`);
304+
yield* fileSystem.makeDirectory(lockPath);
305+
yield* fileSystem.writeFileString(`${lockPath}/${replacement}`, "");
306+
}),
307+
),
308+
),
309+
}),
310+
);
311+
const installing = yield* manager.install.pipe(Effect.forkChild);
312+
yield* Deferred.await(checked);
313+
yield* Fiber.interrupt(installing);
314+
expect(yield* fileSystem.readDirectory(lockPath)).toEqual([replacement]);
315+
}).pipe(Effect.scoped, Effect.provide(layerInstallRuntime("linux"))),
316+
);
317+
274318
it.effect("releases the install lock when a download is cancelled", () =>
275319
Effect.gen(function* () {
276320
const fileSystem = yield* FileSystem.FileSystem;
@@ -320,12 +364,12 @@ describe("RelayClient", () => {
320364
}).pipe(
321365
Effect.provideService(FileSystem.FileSystem, {
322366
...fileSystem,
323-
writeFileString: (path, contents, options) =>
367+
rename: (from, to) =>
324368
fileSystem
325-
.writeFileString(path, contents, options)
369+
.rename(from, to)
326370
.pipe(
327371
Effect.tap(() =>
328-
path === lockPath && pauseWrite
372+
to === lockPath && pauseWrite
329373
? Deferred.succeed(acquired, undefined).pipe(
330374
Effect.andThen(Deferred.await(completeWrite)),
331375
)

‎packages/shared/src/relayClient.ts‎

Lines changed: 62 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
// @effect-diagnostics-next-line nodeBuiltinImport:off -- FileSystem.remove cannot remove an empty directory nonrecursively.
2+
import * as NodeFSP from "node:fs/promises";
13
import * as Cache from "effect/Cache";
24
import * as Clock from "effect/Clock";
35
import type {
@@ -16,7 +18,7 @@ import * as FileSystem from "effect/FileSystem";
1618
import * as Layer from "effect/Layer";
1719
import * as Option from "effect/Option";
1820
import * as Path from "effect/Path";
19-
import * as PlatformError from "effect/PlatformError";
21+
import type * as PlatformError from "effect/PlatformError";
2022
import * as Schedule from "effect/Schedule";
2123
import * as Semaphore from "effect/Semaphore";
2224
import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http";
@@ -115,15 +117,8 @@ const CLOUDFLARED_RELEASE_ASSETS: Readonly<
115117

116118
const INSTALL_LOCK_RETRY_COUNT = 100;
117119
const INSTALL_LOCK_RETRY_DELAY = "100 millis";
118-
// Locks record `pid:token`. A dead pid is an installer that exited, including
119-
// a quit mid-download, and is removed immediately. A lock with no pid is the
120-
// previous format and names no owner, so it only blocks for a short grace.
121-
// The retry loop outlasts that grace.
122-
const INSTALL_LOCK_LEGACY_GRACE_MS = 5_000;
123-
// Longer than the download timeout, so a live installer is not stolen while a
124-
// slow transfer is still running. A pid that is still alive after this is
125-
// wedged, or the pid was reused.
126-
const INSTALL_LOCK_LIVE_STALE_MS = 12 * 60 * 1_000;
120+
// Legacy locks have no owner; allow both ten-minute download stages and the install tail.
121+
const INSTALL_LOCK_LEGACY_STALE_MS = 30 * 60 * 1_000;
127122
const VERSION_PROBE_TIMEOUT = "10 seconds";
128123

129124
const ACTIVATE_RETRY_COUNT = 40;
@@ -202,25 +197,29 @@ class VersionProbeKey extends Data.Class<{
202197
readonly mtimeMillis: number;
203198
}> {}
204199

205-
function isAlreadyExists(error: PlatformError.PlatformError): boolean {
206-
return error.reason._tag === "AlreadyExists";
200+
function isLockContention(error: PlatformError.PlatformError): boolean {
201+
const code = (error.reason.cause as NodeJS.ErrnoException | undefined)?.code;
202+
return (
203+
error.reason._tag === "AlreadyExists" ||
204+
error.reason._tag === "BadResource" ||
205+
code === "ENOTEMPTY" ||
206+
isWindowsRenameLock(error)
207+
);
207208
}
208209

209-
function parseLockOwnerPid(contents: string): number | null {
210-
const separator = contents.indexOf(":");
210+
function parseLockOwnerPid(owner: string): number | null {
211+
const separator = owner.indexOf("-");
211212
if (separator <= 0) return null;
212-
const pid = Number(contents.slice(0, separator));
213+
const pid = Number(owner.slice(0, separator));
213214
return Number.isSafeInteger(pid) && pid > 0 ? pid : null;
214215
}
215216

216-
// `EPERM` means the process exists but belongs to someone else. Any other
217-
// failure, including `ESRCH`, means the installer that wrote the lock is gone.
218217
function processIsAlive(pid: number): boolean {
219218
try {
220219
process.kill(pid, 0);
221220
return true;
222221
} catch (error) {
223-
return (error as { readonly code?: unknown }).code === "EPERM";
222+
return (error as { readonly code?: unknown }).code !== "ESRCH";
224223
}
225224
}
226225

@@ -482,54 +481,65 @@ export const makeCloudflaredRelayClient = Effect.fn("cloudflared.make")(function
482481
return bytes;
483482
});
484483

485-
const releaseOwnedLock = (lockPath: string, token: string) =>
486-
fileSystem.readFileString(lockPath).pipe(
487-
Effect.flatMap((contents) =>
488-
contents.trim() === token ? fileSystem.remove(lockPath, { force: true }) : Effect.void,
489-
),
490-
Effect.ignore,
491-
);
484+
const removeEmptyLock = (lockPath: string) =>
485+
Effect.tryPromise(() => NodeFSP.rmdir(lockPath)).pipe(Effect.ignore);
486+
487+
const releaseOwnedLock = Effect.fn("cloudflared.releaseOwnedLock")(function* (
488+
lockPath: string,
489+
owner: string,
490+
) {
491+
yield* fileSystem.remove(path.join(lockPath, owner), { force: true }).pipe(Effect.ignore);
492+
// A replacement contains a different owner entry, so rmdir cannot remove it.
493+
yield* removeEmptyLock(lockPath);
494+
});
492495

493496
const acquireInstallLock = Effect.fn("cloudflared.acquireInstallLock")(function* (
494497
lockPath: string,
495498
) {
496-
const token = `${process.pid}:${yield* crypto.randomUUIDv4()}`;
499+
const owner = `${process.pid}-${yield* crypto.randomUUIDv4()}`;
500+
const candidate = yield* Effect.acquireRelease(
501+
fileSystem.makeTempDirectory({ directory: path.dirname(lockPath), prefix: ".lock-" }),
502+
(directory) =>
503+
fileSystem.remove(directory, { recursive: true, force: true }).pipe(Effect.ignore),
504+
);
505+
yield* fileSystem.writeFileString(path.join(candidate, owner), "");
497506
for (let attempt = 0; attempt < INSTALL_LOCK_RETRY_COUNT; attempt += 1) {
498507
const acquired = yield* Effect.acquireRelease(
499-
fileSystem.writeFileString(lockPath, token, { flag: "wx" }).pipe(Effect.as(token)),
500-
(owned) => releaseOwnedLock(lockPath, owned),
508+
// Publish a populated directory atomically, with no ownerless acquisition window.
509+
fileSystem.rename(candidate, lockPath),
510+
() => releaseOwnedLock(lockPath, owner),
501511
).pipe(
502512
Effect.as(true),
503-
Effect.catchIf(isAlreadyExists, () => Effect.succeed(false)),
513+
Effect.catchIf(isLockContention, () => Effect.succeed(false)),
504514
);
505515
if (acquired) return;
506516

507-
const now = yield* Clock.currentTimeMillis;
508517
const lockInfo = yield* fileSystem.stat(lockPath).pipe(Effect.option);
509-
// The holder removed it between the exclusive create and this stat.
510518
if (Option.isNone(lockInfo)) continue;
511-
const mtime = lockInfo.value.mtime;
512-
const age = Option.isSome(mtime) ? now - mtime.value.getTime() : Number.POSITIVE_INFINITY;
513-
const contents = yield* fileSystem.readFileString(lockPath).pipe(
514-
Effect.map((value) => value.trim()),
515-
Effect.orElseSucceed(() => ""),
516-
);
517-
const ownerPid = parseLockOwnerPid(contents);
518-
const abandoned =
519-
ownerPid === null
520-
? age > INSTALL_LOCK_LEGACY_GRACE_MS
521-
: !processIsAlive(ownerPid) || age > INSTALL_LOCK_LIVE_STALE_MS;
522-
if (abandoned) {
523-
// Only unlink the contents just judged abandoned. A replacement belongs
524-
// to whoever won the race.
525-
const current = yield* fileSystem.readFileString(lockPath).pipe(
526-
Effect.map((value) => value.trim()),
527-
Effect.option,
528-
);
529-
if (Option.isSome(current) && current.value === contents) {
530-
yield* fileSystem.remove(lockPath, { force: true });
519+
if (lockInfo.value.type === "Directory") {
520+
const owners = yield* fileSystem
521+
.readDirectory(lockPath)
522+
.pipe(Effect.orElseSucceed(() => []));
523+
let recovered = owners.length === 0;
524+
for (const entry of owners) {
525+
const pid = parseLockOwnerPid(entry);
526+
if (pid !== null && !processIsAlive(pid)) {
527+
yield* releaseOwnedLock(lockPath, entry);
528+
recovered = true;
529+
}
530+
}
531+
if (recovered) {
532+
yield* removeEmptyLock(lockPath);
533+
continue;
534+
}
535+
} else {
536+
const now = yield* Clock.currentTimeMillis;
537+
const mtime = lockInfo.value.mtime;
538+
if (Option.isSome(mtime) && now - mtime.value.getTime() > INSTALL_LOCK_LEGACY_STALE_MS) {
539+
// Nonrecursive removal also refuses a populated directory that replaced this file.
540+
yield* fileSystem.remove(lockPath, { force: true }).pipe(Effect.ignore);
541+
continue;
531542
}
532-
continue;
533543
}
534544
yield* Effect.sleep(INSTALL_LOCK_RETRY_DELAY);
535545
}

0 commit comments

Comments
 (0)