Skip to content

Commit 9850f4c

Browse files
thymikeeclaude
andcommitted
fix(daemon): recover provider leases a killed daemon left rented
The daemon kept live provider leases only in memory and persisted a lease only once it had expired and its release failed. A daemon killed with `kill -9` while it held a lease forgot the lease, so a rented cloud device stayed billed until the provider's own idle timeout. The lease registry now reports its provider leases whenever one is issued or leaves, and the daemon journals those whose provider can recover an expired lease in `live-provider-leases.json`. A daemon that starts and finds the journal moves its leases into the expired-lease recovery, which runs at startup and ends each rented device. The journal is dropped only after the leases are in the expired record, so a crash between the two keeps them in one file or the other. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 65ec68b commit 9850f4c

6 files changed

Lines changed: 150 additions & 21 deletions

File tree

‎docs/adr/0007-remote-device-leases.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,12 @@ In-flight work and human-control holds still keep a lease from expiring.
8585
After daemon shutdown begins, no new lease-end teardown starts.
8686
Shutdown waits for any lease-end teardown that already started before it tears sessions down itself.
8787

88+
A provider lease that ends with its daemon still holds a rented device on the provider side, and that device stays billed.
89+
Thus, the daemon journals the live leases of each provider that can recover an expired lease, in `live-provider-leases.json` in its state directory.
90+
A daemon that starts and finds this journal hands each lease in it to the expired-lease recovery of its provider, which ends the rented device.
91+
The lease itself is not restored, and its controller connects again.
92+
Restoring the live lease was rejected: the sessions and device claims of the dead daemon are gone, so a restored lease would name a device that no session can reach.
93+
8894
When a request closes a session, the daemon ends the lease of that session, unless the lease was allocated with `retainOnClose`.
8995
The reply to that request reports each lease that it ended once, at response level, as `endedLeases: [{ leaseId, outcome: 'released' }]`.
9096
On success, the report is in `data`.

‎packages/daemon/src/resources/lease-registry-scope.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,12 @@ export type LeaseRegistryOptions = {
1616
providerSessionRetentionMs?: number;
1717
now?: () => number;
1818
onLeaseExpired?: (lease: DeviceLease) => void;
19+
/**
20+
* Called with every live provider lease after a provider lease is issued or leaves the registry, so
21+
* the daemon can journal what a crash would otherwise forget. An expired lease leaves only after
22+
* `onLeaseExpired` has taken it over.
23+
*/
24+
onProviderLeasesChanged?: (leases: readonly DeviceLease[]) => void;
1925
};
2026

2127
export type AllocateLeaseRequest = {

‎packages/daemon/src/resources/lease-registry.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@ export class LeaseRegistry {
6363
private readonly resolveLeaseTtlMs: ReturnType<typeof createLeaseTtlResolver>;
6464
private readonly now: () => number;
6565
private readonly onLeaseExpired?: (lease: DeviceLease) => void;
66+
private readonly onProviderLeasesChanged?: (leases: readonly DeviceLease[]) => void;
6667
private readonly providerSessionOwnership: ProviderSessionOwnershipRegistry;
6768

6869
constructor(options: LeaseRegistryOptions = {}) {
@@ -72,6 +73,7 @@ export class LeaseRegistry {
7273
this.resolveLeaseTtlMs = createLeaseTtlResolver(options);
7374
this.now = options.now ?? (() => Date.now());
7475
this.onLeaseExpired = options.onLeaseExpired;
76+
this.onProviderLeasesChanged = options.onProviderLeasesChanged;
7577
this.providerSessionOwnership = new ProviderSessionOwnershipRegistry({
7678
now: this.now,
7779
retentionMs: options.providerSessionRetentionMs,
@@ -175,6 +177,16 @@ export class LeaseRegistry {
175177
this.issuedLeaseIds.add(lease.leaseId);
176178
this.leases.set(lease.leaseId, lease);
177179
this.bindLease(lease);
180+
this.reportProviderLeases(lease);
181+
}
182+
183+
private reportProviderLeases(changed: DeviceLease): void {
184+
if (!changed.leaseProvider || !this.onProviderLeasesChanged) return;
185+
this.onProviderLeasesChanged(
186+
[...this.leases.values()]
187+
.filter((lease) => lease.leaseProvider)
188+
.map((lease) => ({ ...lease })),
189+
);
178190
}
179191

180192
private refreshExistingRunBinding(
@@ -249,6 +261,7 @@ export class LeaseRegistry {
249261
this.leases.delete(lease.leaseId);
250262
this.unbindLease(lease);
251263
this.inFlightWork.forget(lease.leaseId);
264+
this.reportProviderLeases(lease);
252265
return { released: true, lease };
253266
}
254267

@@ -585,6 +598,7 @@ export class LeaseRegistry {
585598
const expiredLease = { ...lease };
586599
expired.push(expiredLease);
587600
this.onLeaseExpired?.(expiredLease);
601+
this.reportProviderLeases(expiredLease);
588602
}
589603
return expired;
590604
}
@@ -604,6 +618,7 @@ export class LeaseRegistry {
604618
this.unbindLease(currentLease, currentLease.expiresAt);
605619
const expiredLease = { ...currentLease };
606620
this.onLeaseExpired?.(expiredLease);
621+
this.reportProviderLeases(expiredLease);
607622
return expiredLease;
608623
}
609624

‎packages/daemon/src/server/__tests__/provider-lease-expiry.test.ts‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import fs from 'node:fs';
22
import { expect, test, vi } from 'vitest';
33
import { createExpiredProviderLeaseReleaser } from '../provider-lease-expiry.ts';
44
import { LeaseRegistry } from '../../resources/lease-registry.ts';
5+
import { leaseReleaseRequestFor } from '../../resources/lease-registry-scope.ts';
56
import type { DeviceLease } from '@agent-device/contracts/device';
67
import { mkdtempForTestSync } from '@agent-device/test-kit';
78

@@ -164,3 +165,56 @@ test('shutdown drain reports releases completed before and during its final retr
164165
releaser.shutdown();
165166
}
166167
});
168+
169+
test('a daemon killed while holding a provider lease recovers it on restart', async () => {
170+
const stateDir = mkdtempForTestSync('agent-device-live-provider-lease-');
171+
const recoverExpiredLease = vi.fn(async (_lease: DeviceLease) => undefined);
172+
const options = { recoverExpiredLease, stateDir, recoverableProviderIds: ['revyl'] };
173+
const killedDaemon = createExpiredProviderLeaseReleaser(options);
174+
const registry = new LeaseRegistry({
175+
onProviderLeasesChanged: (leases) => killedDaemon.recordLiveLeases(leases),
176+
});
177+
try {
178+
const held = registry.allocateLease({ tenantId: 't', runId: 'run-1', leaseProvider: 'revyl' });
179+
// A provider with no recovery has nothing a restart could do, so it is not journaled.
180+
registry.allocateLease({ tenantId: 't', runId: 'run-2', leaseProvider: 'browserstack' });
181+
// The process dies here: no release, no expiry, nothing drained.
182+
killedDaemon.shutdown();
183+
184+
const restartedDaemon = createExpiredProviderLeaseReleaser(options);
185+
await restartedDaemon.retryPending();
186+
expect(recoverExpiredLease.mock.calls.map(([lease]) => lease.leaseId)).toEqual([held.leaseId]);
187+
expect(fs.existsSync(`${stateDir}/live-provider-leases.json`)).toBe(false);
188+
restartedDaemon.shutdown();
189+
190+
// Recovered once means recovered: the next daemon inherits nothing.
191+
const thirdDaemon = createExpiredProviderLeaseReleaser(options);
192+
await thirdDaemon.retryPending();
193+
expect(recoverExpiredLease).toHaveBeenCalledTimes(1);
194+
thirdDaemon.shutdown();
195+
} finally {
196+
fs.rmSync(stateDir, { recursive: true, force: true });
197+
}
198+
});
199+
200+
test('a provider lease released before the daemon stops is not recovered again', async () => {
201+
const stateDir = mkdtempForTestSync('agent-device-live-provider-lease-');
202+
const recoverExpiredLease = vi.fn(async (_lease: DeviceLease) => undefined);
203+
const options = { recoverExpiredLease, stateDir, recoverableProviderIds: ['revyl'] };
204+
const daemon = createExpiredProviderLeaseReleaser(options);
205+
const registry = new LeaseRegistry({
206+
onProviderLeasesChanged: (leases) => daemon.recordLiveLeases(leases),
207+
});
208+
try {
209+
const lease = registry.allocateLease({ tenantId: 't', runId: 'run-1', leaseProvider: 'revyl' });
210+
registry.releaseLease(leaseReleaseRequestFor(lease));
211+
daemon.shutdown();
212+
213+
const restartedDaemon = createExpiredProviderLeaseReleaser(options);
214+
await restartedDaemon.retryPending();
215+
expect(recoverExpiredLease).not.toHaveBeenCalled();
216+
restartedDaemon.shutdown();
217+
} finally {
218+
fs.rmSync(stateDir, { recursive: true, force: true });
219+
}
220+
});

‎packages/daemon/src/server/http/daemon-runtime.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -383,6 +383,7 @@ async function startDaemonRuntimeOnHost(
383383
void expiredProviderLeaseReleaser.release(lease);
384384
void endSessionsForLease(lease.leaseId, 'LEASE_EXPIRED');
385385
},
386+
onProviderLeasesChanged: (leases) => expiredProviderLeaseReleaser.recordLiveLeases(leases),
386387
});
387388
// The registry only notices an expired lease when something calls it; this sweep ends an
388389
// abandoned lease (and, through `onLeaseExpired`, its session and claim) without a later request.

‎packages/daemon/src/server/provider-lease-expiry.ts‎

Lines changed: 68 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,11 @@ import { sleep } from '@agent-device/host-kit/retry';
1212

1313
const DEFAULT_RETRY_DELAY_MS = 1_000;
1414
const PENDING_RELEASES_FILE = 'expired-provider-leases.json';
15+
/**
16+
* The recoverable provider leases this daemon holds right now. A daemon that starts and finds this
17+
* file inherited it from one that died without releasing them, so every lease in it is recovered.
18+
*/
19+
const LIVE_LEASES_FILE = 'live-provider-leases.json';
1520
const PENDING_RELEASES_VERSION = 1;
1621
const PERSISTED_LEASE_STRING_FIELDS = [
1722
'leaseId',
@@ -32,6 +37,8 @@ type PersistedExpiredProviderLeases = {
3237
export type ExpiredProviderLeaseReleaser = {
3338
beginShutdown: () => void;
3439
release: (lease: DeviceLease) => Promise<void>;
40+
/** Journals the registry's live provider leases, so a daemon killed while holding them recovers them. */
41+
recordLiveLeases: (leases: readonly DeviceLease[]) => void;
3542
retryPending: () => Promise<void>;
3643
/**
3744
* `pending` holds every lease still unreleased; `unknown` is the part of it whose release was
@@ -54,7 +61,7 @@ export function createExpiredProviderLeaseReleaser(options: {
5461
retryDelayMs?: number;
5562
}): ExpiredProviderLeaseReleaser {
5663
const pendingLiveLeases = new Map<string, DeviceLease>();
57-
const pendingRecoveryLeases = loadPendingLeases(options.stateDir);
64+
const pendingRecoveryLeases = loadLeaseRecord(options.stateDir, PENDING_RELEASES_FILE);
5865
const persistedRecoveryLeaseIds = new Set(pendingRecoveryLeases.keys());
5966
const retryDelayMs = options.retryDelayMs ?? DEFAULT_RETRY_DELAY_MS;
6067
let retryTimer: Timer | undefined;
@@ -74,23 +81,10 @@ export function createExpiredProviderLeaseReleaser(options: {
7481
);
7582
return false;
7683
}
77-
const filePath = pendingReleasesPath(options.stateDir);
7884
try {
79-
if (pendingRecoveryLeases.size === 0) {
80-
fs.rmSync(filePath, { force: true });
81-
} else {
82-
fs.mkdirSync(options.stateDir, { recursive: true, mode: 0o700 });
83-
const record: PersistedExpiredProviderLeases = {
84-
version: PENDING_RELEASES_VERSION,
85-
leases: [...pendingRecoveryLeases.values()],
86-
};
87-
publishFileSync({
88-
destination: filePath,
89-
contents: `${JSON.stringify(record)}\n`,
90-
mode: 0o600,
91-
});
92-
fs.chmodSync(filePath, 0o600);
93-
}
85+
writeLeaseRecord(options.stateDir, PENDING_RELEASES_FILE, [
86+
...pendingRecoveryLeases.values(),
87+
]);
9488
persistedRecoveryLeaseIds.clear();
9589
for (const leaseId of pendingRecoveryLeases.keys()) {
9690
persistedRecoveryLeaseIds.add(leaseId);
@@ -102,6 +96,43 @@ export function createExpiredProviderLeaseReleaser(options: {
10296
}
10397
};
10498

99+
const adoptOrphanedLeases = (): void => {
100+
const orphaned = loadLeaseRecord(options.stateDir, LIVE_LEASES_FILE);
101+
if (orphaned.size === 0 || !options.stateDir) return;
102+
for (const [leaseId, lease] of orphaned) {
103+
if (!pendingRecoveryLeases.has(leaseId)) pendingRecoveryLeases.set(leaseId, lease);
104+
}
105+
// The journal is dropped only once its leases are in the pending record, so a crash in between
106+
// leaves them in one file or the other.
107+
if (!persistPendingLeases()) return;
108+
fs.rmSync(leaseRecordPath(options.stateDir, LIVE_LEASES_FILE), { force: true });
109+
emitDiagnostic({
110+
level: 'warn',
111+
phase: 'provider_lease_orphans_adopted',
112+
data: { leaseIds: [...orphaned.keys()] },
113+
});
114+
};
115+
adoptOrphanedLeases();
116+
117+
const recordLiveLeases = (leases: readonly DeviceLease[]): void => {
118+
if (!options.stateDir) return;
119+
const recoverable = leases.filter((lease) =>
120+
isRecoverableProviderAvailable(lease, options.recoverableProviderIds),
121+
);
122+
try {
123+
writeLeaseRecord(options.stateDir, LIVE_LEASES_FILE, recoverable);
124+
} catch (error) {
125+
emitDiagnostic({
126+
level: 'error',
127+
phase: 'provider_lease_journal_write_failed',
128+
data: {
129+
leaseCount: recoverable.length,
130+
error: error instanceof Error ? error.message : String(error),
131+
},
132+
});
133+
}
134+
};
135+
105136
const scheduleRetry = (): void => {
106137
if (
107138
retryTimer ||
@@ -207,6 +238,7 @@ export function createExpiredProviderLeaseReleaser(options: {
207238
scheduleRetry();
208239
}
209240
},
241+
recordLiveLeases,
210242
retryPending,
211243
drain: async (timeoutMs) => {
212244
// Shutdown gets one final, bounded attempt at every pending release. A
@@ -297,9 +329,24 @@ function emitPersistenceFailure(pendingLeaseCount: number, error: unknown): void
297329
});
298330
}
299331

300-
function loadPendingLeases(stateDir: string | undefined): Map<string, DeviceLease> {
332+
function writeLeaseRecord(stateDir: string, file: string, leases: readonly DeviceLease[]): void {
333+
const filePath = leaseRecordPath(stateDir, file);
334+
if (leases.length === 0) {
335+
fs.rmSync(filePath, { force: true });
336+
return;
337+
}
338+
fs.mkdirSync(stateDir, { recursive: true, mode: 0o700 });
339+
const record: PersistedExpiredProviderLeases = {
340+
version: PENDING_RELEASES_VERSION,
341+
leases: [...leases],
342+
};
343+
publishFileSync({ destination: filePath, contents: `${JSON.stringify(record)}\n`, mode: 0o600 });
344+
fs.chmodSync(filePath, 0o600);
345+
}
346+
347+
function loadLeaseRecord(stateDir: string | undefined, file: string): Map<string, DeviceLease> {
301348
if (!stateDir) return new Map();
302-
const filePath = pendingReleasesPath(stateDir);
349+
const filePath = leaseRecordPath(stateDir, file);
303350
try {
304351
const parsed = JSON.parse(fs.readFileSync(filePath, 'utf8')) as PersistedExpiredProviderLeases;
305352
if (parsed.version !== PENDING_RELEASES_VERSION || !Array.isArray(parsed.leases)) {
@@ -328,6 +375,6 @@ function isPersistedDeviceLease(value: unknown): value is DeviceLease {
328375
);
329376
}
330377

331-
function pendingReleasesPath(stateDir: string): string {
332-
return path.join(stateDir, PENDING_RELEASES_FILE);
378+
function leaseRecordPath(stateDir: string, file: string): string {
379+
return path.join(stateDir, file);
333380
}

0 commit comments

Comments
 (0)