|
| 1 | +import fs from "node:fs"; |
| 2 | +import path from "node:path"; |
| 3 | +import { getProcessStartTime } from "../shared/pid-alive.js"; |
| 4 | + |
| 5 | +export const BUNDLED_RUNTIME_DEPS_LOCK_DIR = ".openclaw-runtime-deps.lock"; |
| 6 | + |
| 7 | +const BUNDLED_RUNTIME_DEPS_LOCK_OWNER_FILE = "owner.json"; |
| 8 | +const BUNDLED_RUNTIME_DEPS_LOCK_WAIT_MS = 100; |
| 9 | +const BUNDLED_RUNTIME_DEPS_LOCK_TIMEOUT_MS = 5 * 60_000; |
| 10 | +const BUNDLED_RUNTIME_DEPS_LOCK_STALE_MS = 10 * 60_000; |
| 11 | +const BUNDLED_RUNTIME_DEPS_OWNERLESS_LOCK_STALE_MS = 30_000; |
| 12 | + |
| 13 | +type RuntimeDepsLockOwner = { |
| 14 | + pid?: number; |
| 15 | + starttime?: number; |
| 16 | + createdAtMs?: number; |
| 17 | + ownerFileState: "ok" | "missing" | "invalid"; |
| 18 | + ownerFilePath: string; |
| 19 | + ownerFileMtimeMs?: number; |
| 20 | + ownerFileIsSymlink?: boolean; |
| 21 | + lockDirMtimeMs?: number; |
| 22 | +}; |
| 23 | + |
| 24 | +const CURRENT_PROCESS_STARTTIME = getProcessStartTime(process.pid); |
| 25 | + |
| 26 | +function sleepSync(ms: number): void { |
| 27 | + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); |
| 28 | +} |
| 29 | + |
| 30 | +async function sleep(ms: number): Promise<void> { |
| 31 | + await new Promise((resolve) => setTimeout(resolve, ms)); |
| 32 | +} |
| 33 | + |
| 34 | +function isProcessAlive(pid: number): boolean { |
| 35 | + if (!Number.isInteger(pid) || pid <= 0) { |
| 36 | + return false; |
| 37 | + } |
| 38 | + try { |
| 39 | + process.kill(pid, 0); |
| 40 | + return true; |
| 41 | + } catch (error) { |
| 42 | + return (error as NodeJS.ErrnoException).code === "EPERM"; |
| 43 | + } |
| 44 | +} |
| 45 | + |
| 46 | +function readRuntimeDepsLockOwner(lockDir: string): RuntimeDepsLockOwner { |
| 47 | + const ownerFilePath = path.join(lockDir, BUNDLED_RUNTIME_DEPS_LOCK_OWNER_FILE); |
| 48 | + let owner: Record<string, unknown> | null = null; |
| 49 | + let ownerFileState: RuntimeDepsLockOwner["ownerFileState"] = "missing"; |
| 50 | + let ownerFileMtimeMs: number | undefined; |
| 51 | + let ownerFileIsSymlink: boolean | undefined; |
| 52 | + try { |
| 53 | + const ownerFileStat = fs.lstatSync(ownerFilePath); |
| 54 | + ownerFileMtimeMs = ownerFileStat.mtimeMs; |
| 55 | + ownerFileIsSymlink = ownerFileStat.isSymbolicLink(); |
| 56 | + } catch { |
| 57 | + // The owner file may not exist yet, or may have been removed by the lock owner. |
| 58 | + } |
| 59 | + try { |
| 60 | + const parsed = JSON.parse(fs.readFileSync(ownerFilePath, "utf8")) as unknown; |
| 61 | + if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { |
| 62 | + owner = parsed as Record<string, unknown>; |
| 63 | + ownerFileState = "ok"; |
| 64 | + } else { |
| 65 | + ownerFileState = "invalid"; |
| 66 | + } |
| 67 | + } catch (error) { |
| 68 | + ownerFileState = |
| 69 | + (error as NodeJS.ErrnoException).code === "ENOENT" && ownerFileMtimeMs === undefined |
| 70 | + ? "missing" |
| 71 | + : "invalid"; |
| 72 | + } |
| 73 | + let lockDirMtimeMs: number | undefined; |
| 74 | + try { |
| 75 | + lockDirMtimeMs = fs.statSync(lockDir).mtimeMs; |
| 76 | + } catch { |
| 77 | + // The lock may have disappeared between the mkdir failure and diagnostics. |
| 78 | + } |
| 79 | + return { |
| 80 | + pid: typeof owner?.pid === "number" ? owner.pid : undefined, |
| 81 | + starttime: typeof owner?.starttime === "number" ? owner.starttime : undefined, |
| 82 | + createdAtMs: typeof owner?.createdAtMs === "number" ? owner.createdAtMs : undefined, |
| 83 | + ownerFileState, |
| 84 | + ownerFilePath, |
| 85 | + ownerFileMtimeMs, |
| 86 | + ownerFileIsSymlink, |
| 87 | + lockDirMtimeMs, |
| 88 | + }; |
| 89 | +} |
| 90 | + |
| 91 | +function latestFiniteMs(values: readonly (number | undefined)[]): number | undefined { |
| 92 | + let latest: number | undefined; |
| 93 | + for (const value of values) { |
| 94 | + if (typeof value !== "number" || !Number.isFinite(value)) { |
| 95 | + continue; |
| 96 | + } |
| 97 | + if (latest === undefined || value > latest) { |
| 98 | + latest = value; |
| 99 | + } |
| 100 | + } |
| 101 | + return latest; |
| 102 | +} |
| 103 | + |
| 104 | +export function shouldRemoveRuntimeDepsLock( |
| 105 | + owner: Pick< |
| 106 | + RuntimeDepsLockOwner, |
| 107 | + "pid" | "starttime" | "createdAtMs" | "lockDirMtimeMs" | "ownerFileMtimeMs" |
| 108 | + >, |
| 109 | + nowMs: number, |
| 110 | + isAlive: (pid: number) => boolean = isProcessAlive, |
| 111 | + readStarttime: (pid: number) => number | null = getProcessStartTime, |
| 112 | +): boolean { |
| 113 | + if (typeof owner.pid === "number") { |
| 114 | + if (!isAlive(owner.pid)) { |
| 115 | + return true; |
| 116 | + } |
| 117 | + if (typeof owner.starttime === "number") { |
| 118 | + const liveStarttime = readStarttime(owner.pid); |
| 119 | + if (liveStarttime !== null && liveStarttime !== owner.starttime) { |
| 120 | + return true; |
| 121 | + } |
| 122 | + } |
| 123 | + return false; |
| 124 | + } |
| 125 | + |
| 126 | + if (typeof owner.createdAtMs === "number") { |
| 127 | + return nowMs - owner.createdAtMs > BUNDLED_RUNTIME_DEPS_LOCK_STALE_MS; |
| 128 | + } |
| 129 | + |
| 130 | + const ownerlessObservedAtMs = latestFiniteMs([owner.lockDirMtimeMs, owner.ownerFileMtimeMs]); |
| 131 | + return ( |
| 132 | + typeof ownerlessObservedAtMs === "number" && |
| 133 | + nowMs - ownerlessObservedAtMs > BUNDLED_RUNTIME_DEPS_OWNERLESS_LOCK_STALE_MS |
| 134 | + ); |
| 135 | +} |
| 136 | + |
| 137 | +function formatDurationMs(ms: number | undefined): string { |
| 138 | + return typeof ms === "number" && Number.isFinite(ms) ? `${Math.max(0, Math.round(ms))}ms` : "n/a"; |
| 139 | +} |
| 140 | + |
| 141 | +export function formatRuntimeDepsLockTimeoutMessage(params: { |
| 142 | + lockDir: string; |
| 143 | + owner: RuntimeDepsLockOwner; |
| 144 | + waitedMs: number; |
| 145 | + nowMs: number; |
| 146 | +}): string { |
| 147 | + const ownerAgeMs = |
| 148 | + typeof params.owner.createdAtMs === "number" |
| 149 | + ? params.nowMs - params.owner.createdAtMs |
| 150 | + : undefined; |
| 151 | + const lockAgeMs = |
| 152 | + typeof params.owner.lockDirMtimeMs === "number" |
| 153 | + ? params.nowMs - params.owner.lockDirMtimeMs |
| 154 | + : undefined; |
| 155 | + const ownerFileAgeMs = |
| 156 | + typeof params.owner.ownerFileMtimeMs === "number" |
| 157 | + ? params.nowMs - params.owner.ownerFileMtimeMs |
| 158 | + : undefined; |
| 159 | + const pidDetail = |
| 160 | + typeof params.owner.pid === "number" |
| 161 | + ? `pid=${params.owner.pid} alive=${isProcessAlive(params.owner.pid)}` |
| 162 | + : "pid=missing"; |
| 163 | + const ownerFileSymlink = |
| 164 | + typeof params.owner.ownerFileIsSymlink === "boolean" ? params.owner.ownerFileIsSymlink : "n/a"; |
| 165 | + return ( |
| 166 | + `Timed out waiting for bundled runtime deps lock at ${params.lockDir} ` + |
| 167 | + `(waited=${formatDurationMs(params.waitedMs)}, ownerFile=${params.owner.ownerFileState}, ownerFileSymlink=${ownerFileSymlink}, ` + |
| 168 | + `${pidDetail}, ownerAge=${formatDurationMs(ownerAgeMs)}, ownerFileAge=${formatDurationMs(ownerFileAgeMs)}, lockAge=${formatDurationMs(lockAgeMs)}, ` + |
| 169 | + `ownerFilePath=${params.owner.ownerFilePath}). If no OpenClaw/npm install is running, remove the lock directory and retry.` |
| 170 | + ); |
| 171 | +} |
| 172 | + |
| 173 | +export function removeRuntimeDepsLockIfStale(lockDir: string, nowMs: number): boolean { |
| 174 | + const owner = readRuntimeDepsLockOwner(lockDir); |
| 175 | + if (!shouldRemoveRuntimeDepsLock(owner, nowMs)) { |
| 176 | + return false; |
| 177 | + } |
| 178 | + |
| 179 | + try { |
| 180 | + fs.rmSync(lockDir, { recursive: true, force: true }); |
| 181 | + return true; |
| 182 | + } catch { |
| 183 | + return false; |
| 184 | + } |
| 185 | +} |
| 186 | + |
| 187 | +function writeRuntimeDepsLockOwner(lockDir: string): void { |
| 188 | + try { |
| 189 | + fs.writeFileSync( |
| 190 | + path.join(lockDir, BUNDLED_RUNTIME_DEPS_LOCK_OWNER_FILE), |
| 191 | + `${JSON.stringify( |
| 192 | + { |
| 193 | + pid: process.pid, |
| 194 | + ...(typeof CURRENT_PROCESS_STARTTIME === "number" |
| 195 | + ? { starttime: CURRENT_PROCESS_STARTTIME } |
| 196 | + : {}), |
| 197 | + createdAtMs: Date.now(), |
| 198 | + }, |
| 199 | + null, |
| 200 | + 2, |
| 201 | + )}\n`, |
| 202 | + "utf8", |
| 203 | + ); |
| 204 | + } catch (ownerWriteError) { |
| 205 | + fs.rmSync(lockDir, { recursive: true, force: true }); |
| 206 | + throw ownerWriteError; |
| 207 | + } |
| 208 | +} |
| 209 | + |
| 210 | +function tryAcquireRuntimeDepsLock(lockDir: string): boolean { |
| 211 | + try { |
| 212 | + fs.mkdirSync(lockDir); |
| 213 | + writeRuntimeDepsLockOwner(lockDir); |
| 214 | + return true; |
| 215 | + } catch (error) { |
| 216 | + const code = (error as NodeJS.ErrnoException).code; |
| 217 | + if (code !== "EEXIST") { |
| 218 | + throw error; |
| 219 | + } |
| 220 | + return false; |
| 221 | + } |
| 222 | +} |
| 223 | + |
| 224 | +function createRuntimeDepsLockTimeoutError(params: { |
| 225 | + lockDir: string; |
| 226 | + startedAt: number; |
| 227 | + nowMs: number; |
| 228 | + cause: unknown; |
| 229 | +}): Error { |
| 230 | + return new Error( |
| 231 | + formatRuntimeDepsLockTimeoutMessage({ |
| 232 | + lockDir: params.lockDir, |
| 233 | + owner: readRuntimeDepsLockOwner(params.lockDir), |
| 234 | + waitedMs: params.nowMs - params.startedAt, |
| 235 | + nowMs: params.nowMs, |
| 236 | + }), |
| 237 | + { cause: params.cause }, |
| 238 | + ); |
| 239 | +} |
| 240 | + |
| 241 | +export function withBundledRuntimeDepsFilesystemLock<T>( |
| 242 | + installRoot: string, |
| 243 | + lockName: string, |
| 244 | + run: () => T, |
| 245 | +): T { |
| 246 | + fs.mkdirSync(installRoot, { recursive: true }); |
| 247 | + const lockDir = path.join(installRoot, lockName); |
| 248 | + const startedAt = Date.now(); |
| 249 | + let locked = false; |
| 250 | + while (!locked) { |
| 251 | + locked = tryAcquireRuntimeDepsLock(lockDir); |
| 252 | + if (!locked) { |
| 253 | + removeRuntimeDepsLockIfStale(lockDir, Date.now()); |
| 254 | + const nowMs = Date.now(); |
| 255 | + if (nowMs - startedAt > BUNDLED_RUNTIME_DEPS_LOCK_TIMEOUT_MS) { |
| 256 | + throw createRuntimeDepsLockTimeoutError({ |
| 257 | + lockDir, |
| 258 | + startedAt, |
| 259 | + nowMs, |
| 260 | + cause: new Error("runtime deps lock already exists"), |
| 261 | + }); |
| 262 | + } |
| 263 | + sleepSync(BUNDLED_RUNTIME_DEPS_LOCK_WAIT_MS); |
| 264 | + } |
| 265 | + } |
| 266 | + try { |
| 267 | + return run(); |
| 268 | + } finally { |
| 269 | + fs.rmSync(lockDir, { recursive: true, force: true }); |
| 270 | + } |
| 271 | +} |
| 272 | + |
| 273 | +export async function withBundledRuntimeDepsFilesystemLockAsync<T>( |
| 274 | + installRoot: string, |
| 275 | + lockName: string, |
| 276 | + run: () => Promise<T>, |
| 277 | +): Promise<T> { |
| 278 | + fs.mkdirSync(installRoot, { recursive: true }); |
| 279 | + const lockDir = path.join(installRoot, lockName); |
| 280 | + const startedAt = Date.now(); |
| 281 | + let locked = false; |
| 282 | + while (!locked) { |
| 283 | + locked = tryAcquireRuntimeDepsLock(lockDir); |
| 284 | + if (!locked) { |
| 285 | + removeRuntimeDepsLockIfStale(lockDir, Date.now()); |
| 286 | + const nowMs = Date.now(); |
| 287 | + if (nowMs - startedAt > BUNDLED_RUNTIME_DEPS_LOCK_TIMEOUT_MS) { |
| 288 | + throw createRuntimeDepsLockTimeoutError({ |
| 289 | + lockDir, |
| 290 | + startedAt, |
| 291 | + nowMs, |
| 292 | + cause: new Error("runtime deps lock already exists"), |
| 293 | + }); |
| 294 | + } |
| 295 | + await sleep(BUNDLED_RUNTIME_DEPS_LOCK_WAIT_MS); |
| 296 | + } |
| 297 | + } |
| 298 | + try { |
| 299 | + return await run(); |
| 300 | + } finally { |
| 301 | + fs.rmSync(lockDir, { recursive: true, force: true }); |
| 302 | + } |
| 303 | +} |
0 commit comments