|
| 1 | +import type { DaemonRequest } from '../daemon/daemon-request.ts'; |
| 2 | +import { AppError } from '@agent-device/kernel/errors'; |
| 3 | +import { createRequestId, emitDiagnostic } from '@agent-device/host-kit/diagnostics'; |
| 4 | +import { INTERNAL_COMMANDS } from '@agent-device/command-registry/catalog'; |
| 5 | +import { resolveCommandTimeoutPolicy } from '@agent-device/command-registry/registry'; |
| 6 | +import { resolveCommandRequestTimeoutMs } from '@agent-device/command-registry/timeout-policy'; |
| 7 | +import { |
| 8 | + isInactiveLeaseError, |
| 9 | + leaseScopeFromRequest, |
| 10 | + leaseScopeToRequestMeta, |
| 11 | + type LeaseScope, |
| 12 | +} from '@agent-device/contracts/lease-scope'; |
| 13 | +import type { DaemonClientSettings } from './daemon-client-lifecycle.ts'; |
| 14 | +import { isRemoteDaemon, type DaemonInfo } from './daemon-client-metadata.ts'; |
| 15 | +import { sendRequest } from './daemon-client-transport.ts'; |
| 16 | + |
| 17 | +// The lease beat: what keeps a remote lease alive across a client-side phase the daemon never sees. |
| 18 | +// |
| 19 | +// A lease renews when a request is admitted, and the daemon protects a lease while admitted work runs |
| 20 | +// on it (#2509, ADR 0007). An artifact upload is the mirror image: it runs on the caller's side, |
| 21 | +// before the install that consumes it is admitted, so nothing renews the lease while it runs and a |
| 22 | +// large enough artifact expired the lease that was paying for the device it was being uploaded to |
| 23 | +// (#2946). This module owns the answer — a beat with a budget that ends before the window does — and |
| 24 | +// knows nothing about transports beyond the `send` it is handed. |
| 25 | + |
| 26 | +/** |
| 27 | + * The fastest cadence a phase beats at, and the budget each beat gets until the window is known. |
| 28 | + * |
| 29 | + * Before the first beat answers, the window is unknown and the worst legal case is the registry's |
| 30 | + * five-second minimum: a beat that stalls must be abandoned and retried inside that window, or the |
| 31 | + * thing the beat exists to prevent happens while it waits. One second is a fifth of that minimum, |
| 32 | + * and no window-derived cadence is ever allowed below it, so it is also the loop's floor: a |
| 33 | + * misreported or pathologically short window cannot turn the beat into a request loop faster than |
| 34 | + * this. |
| 35 | + */ |
| 36 | +const MIN_LEASE_BEAT_INTERVAL_MS = 1_000; |
| 37 | + |
| 38 | +/** |
| 39 | + * Why a beat stopped protecting the lease even though the lease may live on: this client's request |
| 40 | + * will never be the one that renews it. |
| 41 | + * |
| 42 | + * A beat refused for a missing or mismatched owner scope is a fact about the request, not the |
| 43 | + * lease, so every successor is refused identically. Surviving it would only spend the upload against |
| 44 | + * a lease that stops renewing — the #2946 failure with extra steps. |
| 45 | + */ |
| 46 | +const UNRENEWABLE_LEASE_BEAT_REASONS: ReadonlySet<unknown> = new Set([ |
| 47 | + 'LEASE_SCOPE_REQUIRED', |
| 48 | + 'LEASE_SCOPE_MISMATCH', |
| 49 | +]); |
| 50 | + |
| 51 | +/** |
| 52 | + * Whether a beat failed for a reason every successor will repeat: the lease is gone (the shared |
| 53 | + * taxonomy), the daemon refused a fact baked into the beat itself, or this request can never renew |
| 54 | + * it. A beat's scope and ttl never change across the phase, so an `INVALID_ARGS` refusal — an |
| 55 | + * out-of-range ttl, an unusable lease id — is terminal without waiting out a window it can no |
| 56 | + * longer renew. |
| 57 | + * |
| 58 | + * `LEASE_SESSION_MISMATCH` is deliberately absent: only request admission raises it, and |
| 59 | + * `lease_heartbeat` is admission-exempt, so a beat can never receive it. |
| 60 | + */ |
| 61 | +function isTerminalLeaseBeatError(error: unknown): boolean { |
| 62 | + return ( |
| 63 | + isInactiveLeaseError(error) || |
| 64 | + (error instanceof AppError && |
| 65 | + (UNRENEWABLE_LEASE_BEAT_REASONS.has(error.details?.reason) || error.code === 'INVALID_ARGS')) |
| 66 | + ); |
| 67 | +} |
| 68 | + |
| 69 | +/** |
| 70 | + * Runs one client-side phase under a lease it does not own the clock of. |
| 71 | + * |
| 72 | + * A lease renews when a request is admitted, and the daemon protects a lease while ADMITTED work |
| 73 | + * runs on it (#2509, ADR 0007). An artifact upload is the mirror image: it happens on the caller's |
| 74 | + * side, before the install request that will consume it is admitted, so nothing renews the lease |
| 75 | + * while it runs and a large enough artifact expired the lease that was paying for the device it was |
| 76 | + * being uploaded to (#2946). |
| 77 | + * |
| 78 | + * `heartbeat` is the caller's transport decision; `undefined` means there is no lease to protect and |
| 79 | + * the phase runs untouched. The first beat is fired immediately rather than one interval in, so a |
| 80 | + * lease shorter than that interval is renewed before it can lapse — and a lease already gone is |
| 81 | + * found while the upload is still hashing. Each beat answers with the window it just renewed, and |
| 82 | + * the cadence becomes a third of that window. |
| 83 | + * |
| 84 | + * A beat's budget is the cadence it started on, and its successor is armed when the beat starts |
| 85 | + * rather than when it settles. A beat that never returns — a half-open connection, a daemon wedged |
| 86 | + * before it admits anything — is therefore abandoned on schedule instead of holding the schedule: |
| 87 | + * the lease is still beaten at window/3, and the abandoned round trip is cut off by its own budget |
| 88 | + * in the transport rather than by the command's 90-second heartbeat policy. An abandoned beat is |
| 89 | + * still listened to, because the answer it eventually gives can be a lost lease. |
| 90 | + * |
| 91 | + * A beat that fails for a reason that says nothing about this lease is reported and survived — one |
| 92 | + * lost request must not fail an upload that a later beat will cover. A beat that finds the lease |
| 93 | + * gone, or finds this client can never renew it, ends the phase with that error and aborts the |
| 94 | + * signal the phase runs under: the device is no longer ours (or was never reachable through this |
| 95 | + * request), and the only honest outcome is to say so before the bytes finish. |
| 96 | + */ |
| 97 | +export async function runProtectedLeaseWork<T>( |
| 98 | + options: Readonly<{ |
| 99 | + /** |
| 100 | + * One renewal. `budgetMs` is how long this beat may take before the loop abandons it. Absent |
| 101 | + * when the request names no lease to renew, which is the ordinary unleased install. |
| 102 | + */ |
| 103 | + heartbeat?: ((budgetMs: number) => Promise<unknown>) | undefined; |
| 104 | + task: (signal: AbortSignal) => Promise<T>; |
| 105 | + }>, |
| 106 | +): Promise<T> { |
| 107 | + const { heartbeat } = options; |
| 108 | + if (!heartbeat) return await options.task(new AbortController().signal); |
| 109 | + |
| 110 | + const control = new AbortController(); |
| 111 | + // Until a beat names the window, the loop assumes the shortest window the daemon will accept: the |
| 112 | + // budget of the beat that has to prove a short lease is alive cannot itself be longer than it. |
| 113 | + let intervalMs = MIN_LEASE_BEAT_INTERVAL_MS; |
| 114 | + let timer: ReturnType<typeof setTimeout> | undefined; |
| 115 | + let stopped = false; |
| 116 | + let terminalError: unknown; |
| 117 | + let reportTerminal: ((error: unknown) => void) | undefined; |
| 118 | + const terminal = new Promise<never>((_, reject) => { |
| 119 | + reportTerminal = reject; |
| 120 | + }); |
| 121 | + |
| 122 | + const runBeat = (): void => { |
| 123 | + // Armed while this beat is still outstanding: a beat that never settles is abandoned on |
| 124 | + // schedule rather than taking the schedule with it. The beat may re-arm it on the way out. |
| 125 | + const arm = (delayMs: number): void => { |
| 126 | + if (stopped) return; |
| 127 | + if (timer) clearTimeout(timer); |
| 128 | + timer = setTimeout(runBeat, delayMs); |
| 129 | + }; |
| 130 | + arm(intervalMs); |
| 131 | + const settle = (async () => { |
| 132 | + const budgetMs = intervalMs; |
| 133 | + try { |
| 134 | + const renewed = leaseWindowFromHeartbeatResponse(await heartbeat(budgetMs)); |
| 135 | + // An answer that names no window keeps the cadence it was asked at: the loop only ever |
| 136 | + // slows down on evidence of how long the lease is good for, and never on the absence of it. |
| 137 | + if (renewed === undefined) return; |
| 138 | + const cadence = Math.max(MIN_LEASE_BEAT_INTERVAL_MS, Math.floor(renewed / 3)); |
| 139 | + if (cadence === intervalMs) return; |
| 140 | + intervalMs = cadence; |
| 141 | + // The window just moved, so the next beat is due one cadence from this answer. |
| 142 | + arm(cadence); |
| 143 | + } catch (error) { |
| 144 | + if (isTerminalLeaseBeatError(error)) { |
| 145 | + terminalError = error; |
| 146 | + if (stopped) { |
| 147 | + // The phase settled first; the outcome it returned already stands, but a lease this |
| 148 | + // client just learned is gone is worth one diagnostic on the way out. |
| 149 | + emitDiagnostic({ |
| 150 | + level: 'warn', |
| 151 | + phase: 'lease_lost_after_phase', |
| 152 | + data: { message: error instanceof Error ? error.message : String(error) }, |
| 153 | + }); |
| 154 | + return; |
| 155 | + } |
| 156 | + // The upload is the only thing still consuming this phase's time, and it is pointed at a |
| 157 | + // device this client can no longer renew. Stop it rather than finish bytes nobody owns. |
| 158 | + control.abort(); |
| 159 | + reportTerminal?.(error); |
| 160 | + return; |
| 161 | + } |
| 162 | + emitDiagnostic({ |
| 163 | + level: 'warn', |
| 164 | + phase: 'lease_heartbeat_failed', |
| 165 | + data: { message: error instanceof Error ? error.message : String(error) }, |
| 166 | + }); |
| 167 | + } |
| 168 | + })(); |
| 169 | + // A beat the loop has moved on from is still listened to, and nothing awaits it: its outcome is |
| 170 | + // swallowed here so a beat nobody is waiting on cannot surface as an unhandled rejection. |
| 171 | + void settle.catch(() => undefined); |
| 172 | + }; |
| 173 | + |
| 174 | + // Armed before the phase starts, not one interval in: a beat is what proves the lease the upload |
| 175 | + // is spending its time on is still alive. |
| 176 | + timer = setTimeout(runBeat, 0); |
| 177 | + // A beat already in flight when the phase settles can still report a lost lease; the caller reads |
| 178 | + // it from `terminalError`, so nothing may be left racing on this rejection by then. |
| 179 | + void terminal.catch(() => undefined); |
| 180 | + const phase = await captureOutcome( |
| 181 | + // The async boundary also turns a synchronous throw from the phase into a rejection, so the |
| 182 | + // timer below is always cleared. |
| 183 | + (async () => await Promise.race([options.task(control.signal), terminal]))(), |
| 184 | + ); |
| 185 | + stopped = true; |
| 186 | + if (timer) clearTimeout(timer); |
| 187 | + // No outstanding beat is awaited here: a beat on a half-open connection would hold a finished |
| 188 | + // upload behind its own budget for no decision the phase still has to make. |
| 189 | + // A beat that ended the protection outranks a phase that settled meanwhile, from either side: the |
| 190 | + // device is no longer ours, and the lease error is the reason the phase was not worth finishing. |
| 191 | + if (terminalError !== undefined) throw terminalError; |
| 192 | + if (!phase.ok) throw phase.error; |
| 193 | + return phase.value; |
| 194 | +} |
| 195 | + |
| 196 | +/** |
| 197 | + * The inactivity window a beat just renewed, read from the lease its response carries. |
| 198 | + * |
| 199 | + * `heartbeatLease` answers with the lease, whose `expiresAt - heartbeatAt` is exactly the window it |
| 200 | + * extended — the same pair `leaseOwnTtlMs` renews on. Anything unrecognizable leaves the caller on |
| 201 | + * the fallback cadence rather than guessing one. |
| 202 | + */ |
| 203 | +function leaseWindowFromHeartbeatResponse(response: unknown): number | undefined { |
| 204 | + const lease = ( |
| 205 | + response as Readonly<{ data?: Readonly<{ lease?: Readonly<Record<string, unknown>> }> }> |
| 206 | + )?.data?.lease; |
| 207 | + const expiresAt = lease?.expiresAt; |
| 208 | + const heartbeatAt = lease?.heartbeatAt; |
| 209 | + if (typeof expiresAt !== 'number' || typeof heartbeatAt !== 'number') return undefined; |
| 210 | + return expiresAt > heartbeatAt ? expiresAt - heartbeatAt : undefined; |
| 211 | +} |
| 212 | + |
| 213 | +type Outcome<T> = { ok: true; value: T } | { ok: false; error: unknown }; |
| 214 | + |
| 215 | +async function captureOutcome<T>(promise: Promise<T>): Promise<Outcome<T>> { |
| 216 | + try { |
| 217 | + return { ok: true, value: await promise }; |
| 218 | + } catch (error) { |
| 219 | + return { ok: false, error }; |
| 220 | + } |
| 221 | +} |
| 222 | + |
| 223 | +/** |
| 224 | + * The beat that keeps a remote lease alive across a long client-side phase. |
| 225 | + * |
| 226 | + * `send` is the caller's transport and `budgetMs` is how long this beat may take: the beat's answer |
| 227 | + * only matters before the next one is due, so a stalled round trip is cut off at the cadence rather |
| 228 | + * than at the command's own 90-second heartbeat policy. A beat is only ever sent to a remote daemon, |
| 229 | + * where a transport timeout performs no local cleanup. |
| 230 | + */ |
| 231 | +export function createLeaseRenewalBeat( |
| 232 | + leaseScope: LeaseScope, |
| 233 | + context: Readonly<{ |
| 234 | + session: string; |
| 235 | + sessionIsolation?: NonNullable<DaemonRequest['meta']>['sessionIsolation']; |
| 236 | + token: string; |
| 237 | + send: (request: DaemonRequest, budgetMs: number) => Promise<unknown>; |
| 238 | + }>, |
| 239 | +): (budgetMs: number) => Promise<unknown> { |
| 240 | + return async (budgetMs) => |
| 241 | + await context.send( |
| 242 | + buildLeaseHeartbeatRequest(leaseScope, { |
| 243 | + session: context.session, |
| 244 | + sessionIsolation: context.sessionIsolation, |
| 245 | + requestId: createRequestId(), |
| 246 | + token: context.token, |
| 247 | + }), |
| 248 | + budgetMs, |
| 249 | + ); |
| 250 | +} |
| 251 | + |
| 252 | +/** |
| 253 | + * The request one beat sends: the command's lease scope, its own id, and nothing else. |
| 254 | + * |
| 255 | + * It is a fresh request rather than the install rewritten, so nothing about the upload — its source, |
| 256 | + * its positional, its own request id — can be mistaken for the lease's state. The scope rides along |
| 257 | + * as the command named it, so a beat renews for whatever window that command asked for, and for the |
| 258 | + * lease's own window when it named none, which is the ordinary case for an install. Each beat gets a |
| 259 | + * fresh id because a beat that times out is canceled under its own, and a shared id would let a later |
| 260 | + * beat inherit an earlier cancellation. |
| 261 | + */ |
| 262 | +export function buildLeaseHeartbeatRequest( |
| 263 | + leaseScope: LeaseScope, |
| 264 | + context: Readonly<{ |
| 265 | + session: string; |
| 266 | + sessionIsolation?: NonNullable<DaemonRequest['meta']>['sessionIsolation']; |
| 267 | + requestId: string; |
| 268 | + token: string; |
| 269 | + }>, |
| 270 | +): DaemonRequest { |
| 271 | + return { |
| 272 | + command: INTERNAL_COMMANDS.leaseHeartbeat, |
| 273 | + positionals: [], |
| 274 | + session: context.session, |
| 275 | + token: context.token, |
| 276 | + meta: { |
| 277 | + ...leaseScopeToRequestMeta(leaseScope), |
| 278 | + sessionIsolation: context.sessionIsolation, |
| 279 | + requestId: context.requestId, |
| 280 | + }, |
| 281 | + }; |
| 282 | +} |
| 283 | + |
| 284 | +/** The lease a request is running under, when it names one. */ |
| 285 | +export function leaseScopeForHeartbeat( |
| 286 | + request: Pick<DaemonRequest, 'flags' | 'meta'>, |
| 287 | +): LeaseScope | undefined { |
| 288 | + const scope = leaseScopeFromRequest(request); |
| 289 | + return scope.leaseId ? scope : undefined; |
| 290 | +} |
| 291 | + |
| 292 | +/** |
| 293 | + * The beat that renews a remote lease across an artifact upload, or `undefined` when there is no |
| 294 | + * lease to protect: only a remote daemon uploads, so only one can be waiting on a billed device, and |
| 295 | + * a command that names no lease has nothing to renew. |
| 296 | + * |
| 297 | + * Each beat's transport timeout is its budget, capped by the command's heartbeat policy: the answer |
| 298 | + * is worthless once the next beat is due, so a stalled round trip is cut off and destroyed at the |
| 299 | + * cadence instead of holding a socket for 90 seconds. `sendToDaemon` wires this once per upload. |
| 300 | + */ |
| 301 | +export function buildUploadLeaseHeartbeat( |
| 302 | + info: DaemonInfo, |
| 303 | + settings: DaemonClientSettings, |
| 304 | + request: Omit<DaemonRequest, 'token'>, |
| 305 | +): ((budgetMs: number) => Promise<unknown>) | undefined { |
| 306 | + if (!isRemoteDaemon(info)) return undefined; |
| 307 | + const leaseScope = leaseScopeForHeartbeat(request); |
| 308 | + if (!leaseScope) return undefined; |
| 309 | + const policyTimeoutMs = resolveCommandRequestTimeoutMs( |
| 310 | + resolveCommandTimeoutPolicy(INTERNAL_COMMANDS.leaseHeartbeat), |
| 311 | + { positionals: [] }, |
| 312 | + ); |
| 313 | + return createLeaseRenewalBeat(leaseScope, { |
| 314 | + session: request.session, |
| 315 | + sessionIsolation: request.meta?.sessionIsolation, |
| 316 | + token: info.token, |
| 317 | + send: async (beat, budgetMs) => |
| 318 | + await sendRequest( |
| 319 | + info, |
| 320 | + beat, |
| 321 | + settings.transportPreference, |
| 322 | + settings.paths, |
| 323 | + // The beat's own budget governs; the command's heartbeat policy only ever caps it, and an |
| 324 | + // unbounded policy leaves the budget standing on its own. |
| 325 | + policyTimeoutMs === undefined ? budgetMs : Math.min(policyTimeoutMs, budgetMs), |
| 326 | + ), |
| 327 | + }); |
| 328 | +} |
0 commit comments