Skip to content

Commit 790be6d

Browse files
fix(devices): safely reclaim obsolete managed tool versions (pingdotgg#12819)
1 parent 61b25b8 commit 790be6d

6 files changed

Lines changed: 325 additions & 1 deletion

File tree

‎apps/server/src/device/DeviceService.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1111,7 +1111,7 @@ export const make = Effect.gen(function* () {
11111111
return {
11121112
...service,
11131113
agentCli: resolveNodeExecutable("Device automation").pipe(
1114-
Effect.flatMap(() => ensureAgentDevice(config.baseDir)),
1114+
Effect.andThen(ensureAgentDevice(config.baseDir)),
11151115
Effect.provideService(FileSystem.FileSystem, fs),
11161116
Effect.provideService(Path.Path, path),
11171117
Effect.provideService(ProcessRunner.ProcessRunner, runner),

‎apps/server/src/device/LocalDeviceHost.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import { pruneLocalDeviceTools } from "./deviceToolMaintenance.ts";
12
import { deviceToolInstallMessage } from "@t3tools/contracts";
23
/**
34
* The device host that is this machine.
@@ -593,6 +594,11 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () {
593594
);
594595
yield* onPhase("starting");
595596
const hub = yield* spawnHub(hubTool, nodePath);
597+
yield* pruneLocalDeviceTools(config.baseDir, nodePath, "hub").pipe(
598+
Effect.provideService(Path.Path, path),
599+
Effect.provideService(ProcessRunner.ProcessRunner, runner),
600+
Effect.ignore,
601+
);
596602
const candidate = helperPaths(hubTool);
597603
const [axExists, cliExists] = yield* Effect.all([
598604
fs.exists(candidate.serveSimAxSettings).pipe(Effect.orElseSucceed(() => false)),
@@ -655,6 +661,11 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () {
655661
agentToolRef = { entryPath: agentTool.entryPath, nodePath: running.hub.nodePath };
656662
yield* onPhase("starting");
657663
const agentDevice = yield* startAgentDeviceDaemon(agentTool, running.hub.nodePath);
664+
yield* pruneLocalDeviceTools(config.baseDir, running.hub.nodePath, "agent").pipe(
665+
Effect.provideService(Path.Path, path),
666+
Effect.provideService(ProcessRunner.ProcessRunner, runner),
667+
Effect.ignore,
668+
);
658669
const next = { ...running, agentDevice };
659670
yield* Ref.set(runningRef, next);
660671
return { ...toReady(next), agentDevice };
Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
import * as Effect from "effect/Effect";
2+
import * as NodePathLayer from "@effect/platform-node/NodePath";
3+
import * as ProcessRunner from "../processRunner.ts";
4+
import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner";
5+
// @effect-diagnostics nodeBuiltinImport:off - tests the same standalone script used by local and SSH hosts.
6+
import { describe, expect, it } from "@effect/vitest";
7+
import * as NodeFSP from "node:fs/promises";
8+
import * as NodeOS from "node:os";
9+
import * as NodePath from "node:path";
10+
import * as NodeChildProcess from "node:child_process";
11+
import * as NodeUtil from "node:util";
12+
import { pruneLocalDeviceTools, deviceToolMaintenanceScript } from "./deviceToolMaintenance.ts";
13+
14+
const exec = NodeUtil.promisify(NodeChildProcess.execFile);
15+
16+
describe.each([false, true])("device tool cleanup, flat=%s", (flat) => {
17+
it("keeps current, previous, active and incomplete installs, pruning unused completed versions", async () => {
18+
const root = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "t3-tool-cleanup-"));
19+
const name = "expo-device-hub";
20+
const directory = (version: string) =>
21+
flat ? NodePath.join(root, `${name}@${version}`) : NodePath.join(root, name, version);
22+
try {
23+
for (const version of ["0.1.0", "0.2.0", "0.3.0", "0.4.0", "0.5.0", "0.6.0"]) {
24+
await NodeFSP.mkdir(directory(version), { recursive: true });
25+
if (version === "0.5.0") continue;
26+
const sentinel = NodePath.join(directory(version), ".install-complete");
27+
await NodeFSP.writeFile(sentinel, version);
28+
await NodeFSP.utimes(
29+
sentinel,
30+
Number(version.split(".")[1]),
31+
Number(version.split(".")[1]),
32+
);
33+
}
34+
const script =
35+
deviceToolMaintenanceScript +
36+
`
37+
(async () => {
38+
const root = ${JSON.stringify(root)};
39+
await pruneTools(root, [['${name}', '0.6.0']], ${flat});
40+
})().catch(error => { console.error(error); process.exitCode = 1; });`;
41+
await exec(process.execPath, [
42+
"-e",
43+
script,
44+
NodePath.join(directory("0.2.0"), "active-helper.cjs"),
45+
]);
46+
await expect(NodeFSP.stat(directory("0.1.0"))).rejects.toThrow();
47+
await expect(NodeFSP.stat(directory("0.3.0"))).rejects.toThrow();
48+
for (const version of ["0.2.0", "0.4.0", "0.5.0", "0.6.0"])
49+
expect((await NodeFSP.stat(directory(version))).isDirectory()).toBe(true);
50+
} finally {
51+
await NodeFSP.rm(root, { recursive: true, force: true });
52+
}
53+
});
54+
55+
it("keeps every install when the process scan fails", async () => {
56+
const root = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "t3-tool-scan-"));
57+
try {
58+
for (const version of ["0.1.0", "0.2.0", "0.3.0"]) {
59+
const dir = flat
60+
? NodePath.join(root, `expo-device-hub@${version}`)
61+
: NodePath.join(root, "expo-device-hub", version);
62+
await NodeFSP.mkdir(dir, { recursive: true });
63+
await NodeFSP.writeFile(NodePath.join(dir, ".install-complete"), version);
64+
}
65+
await exec(process.execPath, [
66+
"-e",
67+
deviceToolMaintenanceScript +
68+
`
69+
require('node:child_process').spawnSync = () => ({ status: 1, stdout: '' });
70+
pruneTools(${JSON.stringify(root)}, [['expo-device-hub', '0.3.0']], ${flat}).catch(() => process.exitCode = 1);
71+
`,
72+
]);
73+
const parent = flat ? root : NodePath.join(root, "expo-device-hub");
74+
expect((await NodeFSP.readdir(parent)).length).toBe(3);
75+
} finally {
76+
await NodeFSP.rm(root, { recursive: true, force: true });
77+
}
78+
});
79+
80+
it("does not prune before the required version has completed installation", async () => {
81+
const root = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "t3-tool-cleanup-"));
82+
try {
83+
const dir = flat
84+
? NodePath.join(root, "[email protected]")
85+
: NodePath.join(root, "expo-device-hub/0.1.0");
86+
await NodeFSP.mkdir(dir, { recursive: true });
87+
await NodeFSP.writeFile(NodePath.join(dir, ".install-complete"), "0.1.0");
88+
await exec(process.execPath, [
89+
"-e",
90+
deviceToolMaintenanceScript +
91+
`pruneTools(${JSON.stringify(root)}, [['expo-device-hub','0.6.0']], ${flat}).catch(() => process.exitCode = 1);`,
92+
]);
93+
expect((await NodeFSP.stat(dir)).isDirectory()).toBe(true);
94+
} finally {
95+
await NodeFSP.rm(root, { recursive: true, force: true });
96+
}
97+
});
98+
});
99+
100+
it.effect("maintenance failures retain safe context and the original process result", () =>
101+
Effect.gen(function* () {
102+
const output = {
103+
code: ChildProcessSpawner.ExitCode(1),
104+
stdout: "",
105+
stderr: "private child diagnostics",
106+
timedOut: false,
107+
stdoutTruncated: false,
108+
stderrTruncated: false,
109+
stdoutInvalidUtf8: false,
110+
stderrInvalidUtf8: false,
111+
};
112+
for (const [operation, run] of [["prune", pruneLocalDeviceTools]] as const) {
113+
const error = yield* run("/tools", process.execPath, "hub").pipe(
114+
Effect.provideService(ProcessRunner.ProcessRunner, { run: () => Effect.succeed(output) }),
115+
Effect.flip,
116+
);
117+
expect(error).toMatchObject({
118+
_tag: "DeviceToolMaintenanceError",
119+
operation,
120+
tool: "hub",
121+
exitCode: 1,
122+
cause: output,
123+
});
124+
expect(error.message).toBe(`Device tool ${operation} failed for hub (exit code 1).`);
125+
expect(error.message).not.toContain(output.stderr);
126+
}
127+
}).pipe(Effect.provide(NodePathLayer.layer)),
128+
);
129+
130+
it("serializes competing maintenance processes after reclaiming a stale lock", async () => {
131+
const root = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "t3-tool-contention-"));
132+
try {
133+
const lock = NodePath.join(root, ".maintenance-lock");
134+
await NodeFSP.mkdir(lock);
135+
await NodeFSP.writeFile(
136+
NodePath.join(lock, "stale-owner.json"),
137+
JSON.stringify({ pid: 2147483647, identity: "dead" }),
138+
);
139+
const script =
140+
deviceToolMaintenanceScript +
141+
`
142+
(async () => {
143+
const root = ${JSON.stringify(root)};
144+
const marker = maintenancePath.join(root, 'critical-section');
145+
for (let attempt = 0; attempt < 8; attempt++) await withToolMaintenance(root, () => {
146+
maintenanceFs.writeFileSync(marker, String(process.pid), { flag: 'wx' });
147+
for (let check = 0; check < 100; check++) {
148+
if (maintenanceFs.readFileSync(marker, 'utf8') !== String(process.pid)) throw Error('Overlapping maintenance');
149+
}
150+
maintenanceFs.unlinkSync(marker);
151+
});
152+
})().catch(error => { console.error(error); process.exitCode = 1; });`;
153+
await Promise.all(Array.from({ length: 6 }, () => exec(process.execPath, ["-e", script])));
154+
await expect(NodeFSP.stat(lock)).rejects.toThrow();
155+
expect(await NodeFSP.readdir(root)).toEqual([]);
156+
} finally {
157+
await NodeFSP.rm(root, { recursive: true, force: true });
158+
}
159+
});
Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
// @effect-diagnostics preferSchemaOverJson:off - JSON string literals safely embed paths and arguments in generated JavaScript.
2+
import * as Schema from "effect/Schema";
3+
import * as Effect from "effect/Effect";
4+
import * as Path from "effect/Path";
5+
import * as ProcessRunner from "../processRunner.ts";
6+
import { AGENT_DEVICE_VERSION, DEVICE_HUB_VERSION } from "./DeviceToolchain.ts";
7+
8+
/** Shared with the SSH bootstrap. Cleanup runs only after successful startup. */
9+
export const deviceToolMaintenanceScript = String.raw`
10+
const maintenanceFs = require('node:fs');
11+
const maintenancePath = require('node:path');
12+
const maintenanceAlive = pid => {
13+
try { process.kill(pid, 0); return true; }
14+
catch (error) { return error.code !== 'ESRCH'; }
15+
};
16+
async function withToolMaintenance(root, operation) {
17+
maintenanceFs.mkdirSync(root, { recursive: true });
18+
const lock = maintenancePath.join(root, '.maintenance-lock');
19+
const nonce = require('node:crypto').randomUUID();
20+
const ownerFile = process.pid + '.' + nonce + '.json';
21+
const candidate = lock + '.' + nonce;
22+
const holder = { pid: process.pid };
23+
const deadline = Date.now() + 30000;
24+
const removeEmptyLock = () => {
25+
try { maintenanceFs.rmdirSync(lock); }
26+
catch (error) { if (!['ENOENT', 'ENOTEMPTY', 'EEXIST', 'EPERM'].includes(error.code)) throw error; }
27+
};
28+
maintenanceFs.mkdirSync(candidate);
29+
try {
30+
maintenanceFs.writeFileSync(maintenancePath.join(candidate, ownerFile), JSON.stringify(holder));
31+
while (true) {
32+
try {
33+
// Publish a populated directory atomically; rename cannot replace another populated lock.
34+
maintenanceFs.renameSync(candidate, lock);
35+
break;
36+
}
37+
catch (error) {
38+
if (!['EEXIST', 'ENOTEMPTY', 'EPERM', 'EACCES'].includes(error.code)) throw error;
39+
let files = [];
40+
try { files = maintenanceFs.readdirSync(lock); } catch (error) { if (error.code !== 'ENOENT') throw error; }
41+
if (files.length === 1) {
42+
const previousFile = maintenancePath.join(lock, files[0]);
43+
let previous;
44+
try { previous = JSON.parse(maintenanceFs.readFileSync(previousFile, 'utf8')); } catch {}
45+
if (Number.isSafeInteger(previous?.pid) && previous.pid > 0 && !maintenanceAlive(previous.pid)) {
46+
// The unique filename belongs only to that owner. Never unlink a replacement owner's file.
47+
try { maintenanceFs.unlinkSync(previousFile); } catch (error) { if (error.code !== 'ENOENT') throw error; }
48+
}
49+
}
50+
// A concurrent acquirer publishes its owner file with the directory, so this cannot remove it.
51+
removeEmptyLock();
52+
if (Date.now() >= deadline) throw Error('Device tool maintenance is locked. Retry when the other operation finishes.');
53+
await new Promise(resolve => setTimeout(resolve, 50));
54+
}
55+
}
56+
try { return operation(); }
57+
finally {
58+
maintenanceFs.unlinkSync(maintenancePath.join(lock, ownerFile));
59+
removeEmptyLock();
60+
}
61+
} finally {
62+
maintenanceFs.rmSync(candidate, { recursive: true, force: true });
63+
}
64+
}
65+
function pruneTools(root, specs, flat) {
66+
return withToolMaintenance(root, () => {
67+
// Keep installs used by any running helper, including older T3 releases.
68+
const scan = process.platform === 'win32'
69+
? require('node:child_process').spawnSync('powershell.exe', ['-NoProfile', '-NonInteractive', '-Command', 'Get-CimInstance Win32_Process | Select-Object -ExpandProperty CommandLine'], { encoding: 'utf8', timeout: 10000 })
70+
: require('node:child_process').spawnSync('ps', ['-ax', '-o', 'command='], { encoding: 'utf8', timeout: 10000 });
71+
if (scan.status !== 0 || !scan.stdout) return;
72+
for (const [name, required] of specs) {
73+
const parent = flat ? root : maintenancePath.join(root, name);
74+
let names;
75+
try { names = maintenanceFs.readdirSync(parent); } catch { continue; }
76+
const completed = [];
77+
for (const item of names) {
78+
const version = flat ? (item.startsWith(name + '@') ? item.slice(name.length + 1) : '') : item;
79+
if (!/^[0-9]+\.[0-9]+\.[0-9]+(?:-[a-zA-Z0-9.-]+)?$/.test(version)) continue;
80+
const directory = maintenancePath.join(parent, item);
81+
try {
82+
if (!maintenanceFs.lstatSync(directory).isDirectory()) continue;
83+
if (maintenanceFs.readFileSync(maintenancePath.join(directory, '.install-complete'), 'utf8').trim() !== version) continue;
84+
completed.push({ version, directory, modified: maintenanceFs.statSync(maintenancePath.join(directory, '.install-complete')).mtimeMs });
85+
} catch {}
86+
}
87+
// Never prune until the required install has completed. Retain the last other successful install.
88+
if (!completed.some(value => value.version === required)) continue;
89+
const previous = completed.filter(value => value.version !== required).sort((a, b) => b.modified - a.modified || b.version.localeCompare(a.version, 'en', { numeric: true }))[0]?.version;
90+
for (const { version, directory } of completed) {
91+
if (version === required || version === previous || scan.stdout.includes(directory + maintenancePath.sep)) continue;
92+
maintenanceFs.rmSync(directory, { recursive: true, force: true });
93+
}
94+
}
95+
});
96+
}
97+
`;
98+
99+
class DeviceToolMaintenanceError extends Schema.TaggedError<DeviceToolMaintenanceError>()(
100+
"DeviceToolMaintenanceError",
101+
{
102+
operation: Schema.Literal("prune"),
103+
tool: Schema.Literals(["hub", "agent"]),
104+
exitCode: Schema.NullOr(Schema.Int),
105+
cause: Schema.Defect(),
106+
},
107+
) {
108+
override get message() {
109+
return `Device tool ${this.operation} failed for ${this.tool} (exit code ${this.exitCode ?? "unknown"}).`;
110+
}
111+
}
112+
113+
const runMaintenance = Effect.fn("DeviceToolchain.maintenance")(function* (
114+
nodePath: string,
115+
script: string,
116+
operation: "prune",
117+
tool: "hub" | "agent",
118+
) {
119+
const runner = yield* ProcessRunner.ProcessRunner;
120+
const result = yield* runner.run({
121+
command: nodePath,
122+
args: [
123+
"-e",
124+
deviceToolMaintenanceScript +
125+
"\n" +
126+
script +
127+
".catch(error => { console.error(error.message); process.exitCode = 1; });",
128+
],
129+
});
130+
if (result.code !== 0)
131+
return yield* Effect.fail(
132+
new DeviceToolMaintenanceError({ operation, tool, exitCode: result.code, cause: result }),
133+
);
134+
});
135+
136+
export const pruneLocalDeviceTools = Effect.fn("DeviceToolchain.prune")(function* (
137+
baseDir: string,
138+
nodePath: string,
139+
tool: "hub" | "agent",
140+
) {
141+
const path = yield* Path.Path;
142+
yield* runMaintenance(
143+
nodePath,
144+
`pruneTools(${JSON.stringify(path.join(baseDir, "tools"))}, ${JSON.stringify(tool === "hub" ? [["expo-device-hub", DEVICE_HUB_VERSION]] : [["agent-device", AGENT_DEVICE_VERSION]])}, false)`,
145+
"prune",
146+
tool,
147+
);
148+
});

‎apps/server/src/device/sshDeviceScript.test.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,9 @@ else { const child=spawn(process.execPath,[path.join(path.dirname(process.argv[1
125125
);
126126
await NodeFSP.mkdir(NodePath.join(root, "hosts/one"), { recursive: true });
127127
await NodeFSP.writeFile(NodePath.join(root, "hosts/one/fail-start-once"), "");
128+
// Unavailable advisory bookkeeping must not prevent either helper from starting.
129+
await NodeFSP.writeFile(NodePath.join(root, "tools/.maintenance-lock"), "blocked");
130+
await NodeFSP.writeFile(NodePath.join(root, "tools/.users"), "unwritable lease directory");
128131
try {
129132
const [manual, concurrent] = await Promise.all([
130133
invoke("one", "start"),

‎apps/server/src/device/sshDeviceScript.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import { deviceToolMaintenanceScript } from "./deviceToolMaintenance.ts";
12
import { AGENT_DEVICE_VERSION, DEVICE_HUB_VERSION } from "./DeviceToolchain.ts";
23

34
export const quoteRemoteArg = (value: string) => `'${value.replaceAll("'", "'\"'\"'")}'`;
@@ -28,6 +29,7 @@ const mode = ${JSON.stringify(mode)};
2829
const hubVersion = ${JSON.stringify(DEVICE_HUB_VERSION)};
2930
const agentVersion = ${JSON.stringify(AGENT_DEVICE_VERSION)};
3031
` +
32+
deviceToolMaintenanceScript +
3133
String.raw`
3234
const fs = require('node:fs');
3335
const path = require('node:path');
@@ -214,6 +216,7 @@ async function install(name, version, entry) {
214216
}
215217
const vendor = path.resolve(path.dirname(hubEntry), '../../vendor/serve-sim/dist');
216218
const optional = file => fs.existsSync(file) ? file : null;
219+
await pruneTools(path.join(root, 'tools'), [['expo-device-hub', hubVersion], ...(mode === 'agent-start' ? [['agent-device', agentVersion]] : [])], true).catch(() => {});
217220
console.log(JSON.stringify({ nodePath: process.execPath, platforms, tools: versions(), hubPort: hub.port, ...agentResult,
218221
helpers: { serveSimAxSettings: optional(path.join(vendor, 'simax/serve-sim-ax-settings')), serveSimCli: optional(path.join(vendor, 'serve-sim.js')) } }));
219222
} finally { releaseHost(); }

0 commit comments

Comments
 (0)