Skip to content

Commit 8fd6e2d

Browse files
thymikeeclaude
andcommitted
fix(daemon): keep an expiring lease journaled until its record is durable
The journal now also holds every lease that left the registry for recovery but whose expired record has not been written, retries a failed write with the latest leases, and survives a failed removal at startup. ADR 0007 names the trigger as a daemon that dies and records the canceled-allocation gap. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 9850f4c commit 8fd6e2d

3 files changed

Lines changed: 109 additions & 14 deletions

File tree

‎docs/adr/0007-remote-device-leases.md‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,11 +85,13 @@ In-flight work and human-control holds still keep a lease from expiring.
8585
After daemon shutdown begins, no new lease-end teardown starts.
8686
Shutdown waits for any lease-end teardown that already started before it tears sessions down itself.
8787

88-
A provider lease that ends with its daemon still holds a rented device on the provider side, and that device stays billed.
88+
A daemon that dies while it holds a provider lease leaves the rented device running on the provider side, and that device stays billed.
8989
Thus, the daemon journals the live leases of each provider that can recover an expired lease, in `live-provider-leases.json` in its state directory.
9090
A daemon that starts and finds this journal hands each lease in it to the expired-lease recovery of its provider, which ends the rented device.
9191
The lease itself is not restored, and its controller connects again.
9292
Restoring the live lease was rejected: the sessions and device claims of the dead daemon are gone, so a restored lease would name a device that no session can reach.
93+
A lease that leaves the registry for expiry stays in the journal until its expired-lease record is durable, and a journal write that fails is retried.
94+
One gap remains: the registry releases a canceled first allocation without a provider release, so that lease leaves the journal while its provider start may still run, and a crash in that interval loses it.
9395

9496
When a request closes a session, the daemon ends the lease of that session, unless the lease was allocated with `retainOnClose`.
9597
The reply to that request reports each lease that it ended once, at response level, as `endedLeases: [{ leaseId, outcome: 'released' }]`.

‎packages/daemon/src/server/__tests__/provider-lease-expiry.test.ts‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -218,3 +218,64 @@ test('a provider lease released before the daemon stops is not recovered again',
218218
fs.rmSync(stateDir, { recursive: true, force: true });
219219
}
220220
});
221+
222+
test('an expired lease stays journaled until its expired record is durable', async () => {
223+
const stateDir = mkdtempForTestSync('agent-device-live-provider-lease-');
224+
// A directory where the expired record goes makes every write of it fail.
225+
fs.mkdirSync(`${stateDir}/expired-provider-leases.json`);
226+
let now = 1_000;
227+
const recoverExpiredLease = vi.fn(async (_lease: DeviceLease) => undefined);
228+
const daemon = createExpiredProviderLeaseReleaser({
229+
recoverExpiredLease,
230+
stateDir,
231+
recoverableProviderIds: ['revyl'],
232+
});
233+
const registry = new LeaseRegistry({
234+
defaultLeaseTtlMs: 10,
235+
minLeaseTtlMs: 1,
236+
now: () => now,
237+
onLeaseExpired: (lease) => void daemon.release(lease),
238+
onProviderLeasesChanged: (leases) => daemon.recordLiveLeases(leases),
239+
});
240+
try {
241+
const lease = registry.allocateLease({ tenantId: 't', runId: 'run-1', leaseProvider: 'revyl' });
242+
now = 1_011;
243+
registry.consumeExpiredLease(lease.leaseId);
244+
245+
const journal = JSON.parse(fs.readFileSync(`${stateDir}/live-provider-leases.json`, 'utf8'));
246+
expect(journal.leases.map((entry: DeviceLease) => entry.leaseId)).toEqual([lease.leaseId]);
247+
expect(recoverExpiredLease).not.toHaveBeenCalled();
248+
} finally {
249+
daemon.shutdown();
250+
fs.rmSync(stateDir, { recursive: true, force: true });
251+
}
252+
});
253+
254+
test('a journal write that fails is retried with the latest leases', async () => {
255+
vi.useFakeTimers();
256+
const stateDir = mkdtempForTestSync('agent-device-live-provider-lease-');
257+
const journalPath = `${stateDir}/live-provider-leases.json`;
258+
fs.mkdirSync(journalPath);
259+
const daemon = createExpiredProviderLeaseReleaser({
260+
recoverExpiredLease: async () => undefined,
261+
stateDir,
262+
recoverableProviderIds: ['revyl'],
263+
retryDelayMs: 10,
264+
});
265+
const registry = new LeaseRegistry({
266+
onProviderLeasesChanged: (leases) => daemon.recordLiveLeases(leases),
267+
});
268+
try {
269+
const lease = registry.allocateLease({ tenantId: 't', runId: 'run-1', leaseProvider: 'revyl' });
270+
expect(fs.statSync(journalPath).isDirectory()).toBe(true);
271+
272+
fs.rmdirSync(journalPath);
273+
await vi.advanceTimersByTimeAsync(10);
274+
const journal = JSON.parse(fs.readFileSync(journalPath, 'utf8'));
275+
expect(journal.leases.map((entry: DeviceLease) => entry.leaseId)).toEqual([lease.leaseId]);
276+
} finally {
277+
daemon.shutdown();
278+
vi.useRealTimers();
279+
fs.rmSync(stateDir, { recursive: true, force: true });
280+
}
281+
});

‎packages/daemon/src/server/provider-lease-expiry.ts‎

Lines changed: 45 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,13 @@ export function createExpiredProviderLeaseReleaser(options: {
105105
// The journal is dropped only once its leases are in the pending record, so a crash in between
106106
// leaves them in one file or the other.
107107
if (!persistPendingLeases()) return;
108-
fs.rmSync(leaseRecordPath(options.stateDir, LIVE_LEASES_FILE), { force: true });
108+
try {
109+
fs.rmSync(leaseRecordPath(options.stateDir, LIVE_LEASES_FILE), { force: true });
110+
} catch (error) {
111+
// The leases are durable in the pending record; a journal left behind is adopted again, and the
112+
// pending record already holds each of its leases once.
113+
emitJournalFailure('provider_lease_journal_remove_failed', orphaned.size, error);
114+
}
109115
emitDiagnostic({
110116
level: 'warn',
111117
phase: 'provider_lease_orphans_adopted',
@@ -114,24 +120,40 @@ export function createExpiredProviderLeaseReleaser(options: {
114120
};
115121
adoptOrphanedLeases();
116122

117-
const recordLiveLeases = (leases: readonly DeviceLease[]): void => {
123+
let liveLeases: readonly DeviceLease[] = [];
124+
let journalRetryTimer: Timer | undefined;
125+
/**
126+
* Writes the journal: the registry's live leases, plus every lease that left the registry for recovery
127+
* but whose expired record is not durable yet, so a crash never finds a lease in neither file. A write
128+
* that fails is retried with the latest leases until one lands.
129+
*/
130+
const writeJournal = (): void => {
118131
if (!options.stateDir) return;
119-
const recoverable = leases.filter((lease) =>
120-
isRecoverableProviderAvailable(lease, options.recoverableProviderIds),
132+
const unpersisted = [...pendingRecoveryLeases.values()].filter(
133+
(lease) => !persistedRecoveryLeaseIds.has(lease.leaseId),
134+
);
135+
const journal = new Map(
136+
[...liveLeases, ...unpersisted]
137+
.filter((lease) => isRecoverableProviderAvailable(lease, options.recoverableProviderIds))
138+
.map((lease) => [lease.leaseId, lease] as const),
121139
);
122140
try {
123-
writeLeaseRecord(options.stateDir, LIVE_LEASES_FILE, recoverable);
141+
writeLeaseRecord(options.stateDir, LIVE_LEASES_FILE, [...journal.values()]);
142+
if (journalRetryTimer) clearTimeout(journalRetryTimer);
143+
journalRetryTimer = undefined;
124144
} catch (error) {
125-
emitDiagnostic({
126-
level: 'error',
127-
phase: 'provider_lease_journal_write_failed',
128-
data: {
129-
leaseCount: recoverable.length,
130-
error: error instanceof Error ? error.message : String(error),
131-
},
132-
});
145+
emitJournalFailure('provider_lease_journal_write_failed', journal.size, error);
146+
journalRetryTimer ??= setTimeout(() => {
147+
journalRetryTimer = undefined;
148+
writeJournal();
149+
}, retryDelayMs);
150+
journalRetryTimer.unref?.();
133151
}
134152
};
153+
const recordLiveLeases = (leases: readonly DeviceLease[]): void => {
154+
liveLeases = leases;
155+
writeJournal();
156+
};
135157

136158
const scheduleRetry = (): void => {
137159
if (
@@ -263,6 +285,8 @@ export function createExpiredProviderLeaseReleaser(options: {
263285
shutdown: () => {
264286
if (retryTimer) clearTimeout(retryTimer);
265287
retryTimer = undefined;
288+
if (journalRetryTimer) clearTimeout(journalRetryTimer);
289+
journalRetryTimer = undefined;
266290
},
267291
};
268292
}
@@ -318,6 +342,14 @@ function isRecoverableProviderAvailable(
318342
);
319343
}
320344

345+
function emitJournalFailure(phase: string, leaseCount: number, error: unknown): void {
346+
emitDiagnostic({
347+
level: 'error',
348+
phase,
349+
data: { leaseCount, error: error instanceof Error ? error.message : String(error) },
350+
});
351+
}
352+
321353
function emitPersistenceFailure(pendingLeaseCount: number, error: unknown): void {
322354
emitDiagnostic({
323355
level: 'error',

0 commit comments

Comments
 (0)