Skip to content

Commit 3776dfb

Browse files
jooyyycursoragentmaria-rcks
authored
fix(connect): a quit mid-download no longer wedges the relay client install (#22747)
Co-authored-by: Cursor <[email protected]> Co-authored-by: maria-rcks <[email protected]>
1 parent e94c3a7 commit 3776dfb

2 files changed

Lines changed: 215 additions & 18 deletions

File tree

‎packages/shared/src/relayClient.test.ts‎

Lines changed: 138 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,19 @@
11
import { sha256 } from "@noble/hashes/sha2";
22
import * as NodeServices from "@effect/platform-node/NodeServices";
33
import { describe, expect, it } from "@effect/vitest";
4+
import * as Clock from "effect/Clock";
45
import * as ConfigProvider from "effect/ConfigProvider";
56
import * as Deferred from "effect/Deferred";
67
import * as Effect from "effect/Effect";
78
import * as Fiber from "effect/Fiber";
89
import * as Hex from "effect/encoding/Hex";
910
import * as FileSystem from "effect/FileSystem";
1011
import * as Layer from "effect/Layer";
12+
import * as Option from "effect/Option";
1113
import * as PlatformError from "effect/PlatformError";
1214
import * as Sink from "effect/Sink";
1315
import * as Stream from "effect/Stream";
16+
import * as TestClock from "effect/testing/TestClock";
1417
import { HttpClient, HttpClientResponse } from "effect/http";
1518
import { ChildProcess, ChildProcessSpawner } from "effect/process";
1619
import * as HostProcess from "./HostProcess.ts";
@@ -156,12 +159,12 @@ describe("RelayClient", () => {
156159
const manager = yield* RelayClient.makeCloudflaredRelayClient({ baseDir }).pipe(
157160
Effect.provideService(FileSystem.FileSystem, {
158161
...fileSystem,
159-
writeFileString: (path, contents, options) =>
162+
rename: (from, to) =>
160163
fileSystem
161-
.writeFileString(path, contents, options)
164+
.rename(from, to)
162165
.pipe(
163166
Effect.tapError(() =>
164-
path === lockPath ? Deferred.succeed(contended, undefined) : Effect.void,
167+
to === lockPath ? Deferred.succeed(contended, undefined) : Effect.void,
165168
),
166169
),
167170
}),
@@ -183,6 +186,135 @@ describe("RelayClient", () => {
183186
),
184187
);
185188

189+
it.effect.each(["live owner", "recent legacy"] as const)("keeps a lock with a %s", (kind) =>
190+
Effect.gen(function* () {
191+
const fileSystem = yield* FileSystem.FileSystem;
192+
const baseDir = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-cloudflared-test-" });
193+
const lockPath = `${baseDir}/tools/cloudflared/${RelayClient.CLOUDFLARED_VERSION}/linux-x64/cloudflared.lock`;
194+
const owner = `${process.pid}-still-running`;
195+
yield* fileSystem.makeDirectory(lockPath.slice(0, lockPath.lastIndexOf("/")), {
196+
recursive: true,
197+
});
198+
if (kind === "live owner") {
199+
yield* fileSystem.makeDirectory(lockPath);
200+
yield* fileSystem.writeFileString(`${lockPath}/${owner}`, "");
201+
} else {
202+
yield* fileSystem.writeFileString(lockPath, "");
203+
}
204+
const written = yield* fileSystem.stat(lockPath);
205+
yield* TestClock.setTime(
206+
Option.getOrThrow(written.mtime).getTime() + (kind === "live owner" ? 3_600_000 : 6_000),
207+
);
208+
const waiting = yield* Deferred.make<void>();
209+
const clock = yield* Clock.Clock;
210+
const manager = yield* RelayClient.makeCloudflaredRelayClient({
211+
baseDir,
212+
releaseAsset: testReleaseAsset,
213+
});
214+
const installing = yield* manager.install.pipe(
215+
Effect.provideService(Clock.Clock, {
216+
...clock,
217+
sleep: () => Deferred.succeed(waiting, undefined).pipe(Effect.andThen(Effect.never)),
218+
}),
219+
Effect.forkChild,
220+
);
221+
yield* Deferred.await(waiting);
222+
yield* Fiber.interrupt(installing);
223+
if (kind === "live owner") {
224+
expect(yield* fileSystem.readDirectory(lockPath)).toEqual([owner]);
225+
} else {
226+
expect(yield* fileSystem.readFileString(lockPath)).toBe("");
227+
}
228+
}).pipe(Effect.scoped, Effect.provide(layerInstallRuntime("linux"))),
229+
);
230+
231+
it.effect.skipIf(windowsHost)("replaces an install lock left by an exited installer", () =>
232+
Effect.gen(function* () {
233+
const fileSystem = yield* FileSystem.FileSystem;
234+
const baseDir = yield* fileSystem.makeTempDirectoryScoped({
235+
prefix: "t3-cloudflared-test-",
236+
});
237+
const lockPath = `${baseDir}/tools/cloudflared/${RelayClient.CLOUDFLARED_VERSION}/linux-x64/cloudflared.lock`;
238+
yield* fileSystem.makeDirectory(lockPath.slice(0, lockPath.lastIndexOf("/")), {
239+
recursive: true,
240+
});
241+
yield* fileSystem.makeDirectory(lockPath);
242+
yield* fileSystem.writeFileString(`${lockPath}/2147483647-exited-installer`, "");
243+
const manager = yield* RelayClient.makeCloudflaredRelayClient({
244+
baseDir,
245+
releaseAsset: testReleaseAsset,
246+
});
247+
248+
expect(yield* manager.install).toMatchObject({ status: "available" });
249+
expect(yield* fileSystem.exists(lockPath)).toBe(false);
250+
}).pipe(Effect.scoped, Effect.provide(layerInstallRuntime("linux"))),
251+
);
252+
253+
it.effect.skipIf(windowsHost)("replaces an empty install lock left by an exited installer", () =>
254+
Effect.gen(function* () {
255+
const fileSystem = yield* FileSystem.FileSystem;
256+
const baseDir = yield* fileSystem.makeTempDirectoryScoped({
257+
prefix: "t3-cloudflared-test-",
258+
});
259+
const lockPath = `${baseDir}/tools/cloudflared/${RelayClient.CLOUDFLARED_VERSION}/linux-x64/cloudflared.lock`;
260+
yield* fileSystem.makeDirectory(lockPath.slice(0, lockPath.lastIndexOf("/")), {
261+
recursive: true,
262+
});
263+
yield* fileSystem.writeFileString(lockPath, "");
264+
const written = yield* fileSystem.stat(lockPath);
265+
const mtime = Option.getOrThrow(written.mtime).getTime();
266+
// File mtimes use wall time; advance the test clock past legacy expiry.
267+
yield* TestClock.setTime(mtime + 60 * 60 * 1_000);
268+
const manager = yield* RelayClient.makeCloudflaredRelayClient({
269+
baseDir,
270+
releaseAsset: testReleaseAsset,
271+
});
272+
273+
expect(yield* manager.install).toMatchObject({ status: "available" });
274+
expect(yield* fileSystem.exists(lockPath)).toBe(false);
275+
}).pipe(Effect.scoped, Effect.provide(layerInstallRuntime("linux"))),
276+
);
277+
278+
it.effect("keeps a replacement lock when a stale contender resumes cleanup", () =>
279+
Effect.gen(function* () {
280+
const fileSystem = yield* FileSystem.FileSystem;
281+
const baseDir = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-cloudflared-test-" });
282+
const lockPath = `${baseDir}/tools/cloudflared/${RelayClient.CLOUDFLARED_VERSION}/linux-x64/cloudflared.lock`;
283+
const abandoned = "2147483647-exited-installer";
284+
const replacement = `${process.pid}-replacement`;
285+
yield* fileSystem.makeDirectory(lockPath, { recursive: true });
286+
yield* fileSystem.writeFileString(`${lockPath}/${abandoned}`, "");
287+
const checked = yield* Deferred.make<void>();
288+
let replaced = false;
289+
const manager = yield* RelayClient.makeCloudflaredRelayClient({ baseDir }).pipe(
290+
Effect.provideService(FileSystem.FileSystem, {
291+
...fileSystem,
292+
readDirectory: (path) =>
293+
fileSystem.readDirectory(path).pipe(
294+
Effect.tap((entries) =>
295+
Effect.gen(function* () {
296+
if (path !== lockPath) return;
297+
if (replaced) {
298+
expect(entries).toEqual([replacement]);
299+
yield* Deferred.succeed(checked, undefined);
300+
return;
301+
}
302+
replaced = true;
303+
yield* fileSystem.rename(lockPath, `${lockPath}.old`);
304+
yield* fileSystem.makeDirectory(lockPath);
305+
yield* fileSystem.writeFileString(`${lockPath}/${replacement}`, "");
306+
}),
307+
),
308+
),
309+
}),
310+
);
311+
const installing = yield* manager.install.pipe(Effect.forkChild);
312+
yield* Deferred.await(checked);
313+
yield* Fiber.interrupt(installing);
314+
expect(yield* fileSystem.readDirectory(lockPath)).toEqual([replacement]);
315+
}).pipe(Effect.scoped, Effect.provide(layerInstallRuntime("linux"))),
316+
);
317+
186318
it.effect("releases the install lock when a download is cancelled", () =>
187319
Effect.gen(function* () {
188320
const fileSystem = yield* FileSystem.FileSystem;
@@ -232,12 +364,12 @@ describe("RelayClient", () => {
232364
}).pipe(
233365
Effect.provideService(FileSystem.FileSystem, {
234366
...fileSystem,
235-
writeFileString: (path, contents, options) =>
367+
rename: (from, to) =>
236368
fileSystem
237-
.writeFileString(path, contents, options)
369+
.rename(from, to)
238370
.pipe(
239371
Effect.tap(() =>
240-
path === lockPath && pauseWrite
372+
to === lockPath && pauseWrite
241373
? Deferred.succeed(acquired, undefined).pipe(
242374
Effect.andThen(Deferred.await(completeWrite)),
243375
)

‎packages/shared/src/relayClient.ts‎

Lines changed: 77 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
// @effect-diagnostics-next-line nodeBuiltinImport:off -- FileSystem.remove cannot remove an empty directory nonrecursively.
2+
import * as NodeFSP from "node:fs/promises";
13
import * as Cache from "effect/Cache";
24
import * as Clock from "effect/Clock";
35
import type {
@@ -16,7 +18,7 @@ import * as FileSystem from "effect/FileSystem";
1618
import * as Layer from "effect/Layer";
1719
import * as Option from "effect/Option";
1820
import * as Path from "effect/Path";
19-
import * as PlatformError from "effect/PlatformError";
21+
import type * as PlatformError from "effect/PlatformError";
2022
import * as Schedule from "effect/Schedule";
2123
import * as Semaphore from "effect/Semaphore";
2224
import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http";
@@ -115,7 +117,8 @@ const CLOUDFLARED_RELEASE_ASSETS: Readonly<
115117

116118
const INSTALL_LOCK_RETRY_COUNT = 100;
117119
const INSTALL_LOCK_RETRY_DELAY = "100 millis";
118-
const INSTALL_LOCK_STALE_MS = 5 * 60 * 1_000;
120+
// Legacy locks have no owner; allow both ten-minute download stages and the install tail.
121+
const INSTALL_LOCK_LEGACY_STALE_MS = 30 * 60 * 1_000;
119122
const VERSION_PROBE_TIMEOUT = "10 seconds";
120123

121124
const ACTIVATE_RETRY_COUNT = 40;
@@ -194,8 +197,30 @@ class VersionProbeKey extends Data.Class<{
194197
readonly mtimeMillis: number;
195198
}> {}
196199

197-
function isAlreadyExists(error: PlatformError.PlatformError): boolean {
198-
return error.reason._tag === "AlreadyExists";
200+
function isLockContention(error: PlatformError.PlatformError): boolean {
201+
const code = (error.reason.cause as NodeJS.ErrnoException | undefined)?.code;
202+
return (
203+
error.reason._tag === "AlreadyExists" ||
204+
error.reason._tag === "BadResource" ||
205+
code === "ENOTEMPTY" ||
206+
isWindowsRenameLock(error)
207+
);
208+
}
209+
210+
function parseLockOwnerPid(owner: string): number | null {
211+
const separator = owner.indexOf("-");
212+
if (separator <= 0) return null;
213+
const pid = Number(owner.slice(0, separator));
214+
return Number.isSafeInteger(pid) && pid > 0 ? pid : null;
215+
}
216+
217+
function processIsAlive(pid: number): boolean {
218+
try {
219+
process.kill(pid, 0);
220+
return true;
221+
} catch (error) {
222+
return (error as { readonly code?: unknown }).code !== "ESRCH";
223+
}
199224
}
200225

201226
const wrapInstallFailure =
@@ -456,25 +481,65 @@ export const makeCloudflaredRelayClient = Effect.fn("cloudflared.make")(function
456481
return bytes;
457482
});
458483

484+
const removeEmptyLock = (lockPath: string) =>
485+
Effect.tryPromise(() => NodeFSP.rmdir(lockPath)).pipe(Effect.ignore);
486+
487+
const releaseOwnedLock = Effect.fn("cloudflared.releaseOwnedLock")(function* (
488+
lockPath: string,
489+
owner: string,
490+
) {
491+
yield* fileSystem.remove(path.join(lockPath, owner), { force: true }).pipe(Effect.ignore);
492+
// A replacement contains a different owner entry, so rmdir cannot remove it.
493+
yield* removeEmptyLock(lockPath);
494+
});
495+
459496
const acquireInstallLock = Effect.fn("cloudflared.acquireInstallLock")(function* (
460497
lockPath: string,
461498
) {
499+
const owner = `${process.pid}-${yield* crypto.randomUUIDv4()}`;
500+
const candidate = yield* Effect.acquireRelease(
501+
fileSystem.makeTempDirectory({ directory: path.dirname(lockPath), prefix: ".lock-" }),
502+
(directory) =>
503+
fileSystem.remove(directory, { recursive: true, force: true }).pipe(Effect.ignore),
504+
);
505+
yield* fileSystem.writeFileString(path.join(candidate, owner), "");
462506
for (let attempt = 0; attempt < INSTALL_LOCK_RETRY_COUNT; attempt += 1) {
463507
const acquired = yield* Effect.acquireRelease(
464-
fileSystem.writeFileString(lockPath, "", { flag: "wx" }),
465-
() => fileSystem.remove(lockPath, { force: true }).pipe(Effect.ignore),
508+
// Publish a populated directory atomically, with no ownerless acquisition window.
509+
fileSystem.rename(candidate, lockPath),
510+
() => releaseOwnedLock(lockPath, owner),
466511
).pipe(
467512
Effect.as(true),
468-
Effect.catchIf(isAlreadyExists, () => Effect.succeed(false)),
513+
Effect.catchIf(isLockContention, () => Effect.succeed(false)),
469514
);
470515
if (acquired) return;
471516

472-
const now = yield* Clock.currentTimeMillis;
473517
const lockInfo = yield* fileSystem.stat(lockPath).pipe(Effect.option);
474-
const mtime = Option.flatMap(lockInfo, (info) => info.mtime);
475-
if (Option.isSome(mtime) && now - mtime.value.getTime() > INSTALL_LOCK_STALE_MS) {
476-
yield* fileSystem.remove(lockPath, { force: true });
477-
continue;
518+
if (Option.isNone(lockInfo)) continue;
519+
if (lockInfo.value.type === "Directory") {
520+
const owners = yield* fileSystem
521+
.readDirectory(lockPath)
522+
.pipe(Effect.orElseSucceed(() => []));
523+
let recovered = owners.length === 0;
524+
for (const entry of owners) {
525+
const pid = parseLockOwnerPid(entry);
526+
if (pid !== null && !processIsAlive(pid)) {
527+
yield* releaseOwnedLock(lockPath, entry);
528+
recovered = true;
529+
}
530+
}
531+
if (recovered) {
532+
yield* removeEmptyLock(lockPath);
533+
continue;
534+
}
535+
} else {
536+
const now = yield* Clock.currentTimeMillis;
537+
const mtime = lockInfo.value.mtime;
538+
if (Option.isSome(mtime) && now - mtime.value.getTime() > INSTALL_LOCK_LEGACY_STALE_MS) {
539+
// Nonrecursive removal also refuses a populated directory that replaced this file.
540+
yield* fileSystem.remove(lockPath, { force: true }).pipe(Effect.ignore);
541+
continue;
542+
}
478543
}
479544
yield* Effect.sleep(INSTALL_LOCK_RETRY_DELAY);
480545
}

0 commit comments

Comments
 (0)