Skip to content

Commit 495303c

Browse files
fix(relay): count the deletion budget from the start of the sweep
The 90-second budget started after listing, so a slow listing could push the sweep past the cron's two-minute timeout, which drops its counters. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 4329456 commit 495303c

2 files changed

Lines changed: 44 additions & 5 deletions

File tree

‎infra/relay/src/environments/ManagedEndpointReaper.test.ts‎

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,8 @@ function harness(input?: {
7777
readonly legacyTunnelGraceMinutes?: number;
7878
/** Simulated time each release takes, advanced on the test clock. */
7979
readonly releaseDelayMs?: number;
80+
/** Simulated time each Cloudflare list request takes. */
81+
readonly listDelayMs?: number;
8082
}) {
8183
const listRequests: ManagedEndpointProvider.ManagedEndpointTunnelListRequest[] = [];
8284
const deleted: string[] = [];
@@ -121,7 +123,10 @@ function harness(input?: {
121123
return Effect.succeed(found);
122124
}),
123125
list: (request) =>
124-
Effect.sync(() => {
126+
Effect.gen(function* () {
127+
if (input?.listDelayMs !== undefined) {
128+
yield* TestClock.adjust(input.listDelayMs);
129+
}
125130
listRequests.push(request);
126131
const matching = remaining.filter((entry) => entry.status === request.status);
127132
const start = ((request.page ?? 1) - 1) * (request.perPage ?? 100);
@@ -750,6 +755,39 @@ describe("ManagedEndpointReaper", () => {
750755
}).pipe(Effect.provide(state.layer));
751756
});
752757

758+
it.effect("counts listing time against the deletion budget", () => {
759+
const entries = Array.from({ length: 40 }, (_, index) =>
760+
tunnel({
761+
id: `slow-${index}`,
762+
suffix: index.toString(16).padStart(16, "0"),
763+
status: "down",
764+
timestamp: "2026-08-25T11:00:00.000Z",
765+
}),
766+
);
767+
const sweepWith = (listDelayMs: number) =>
768+
Effect.gen(function* () {
769+
yield* TestClock.setTime(NOW_MILLIS);
770+
const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
771+
return (yield* reaper.sweep).attempted;
772+
}).pipe(
773+
Effect.provide(
774+
harness({
775+
tunnels: entries,
776+
allocations: recoverableOwners(entries),
777+
releaseDelayMs: 10_000,
778+
listDelayMs,
779+
}).layer,
780+
),
781+
);
782+
783+
return Effect.gen(function* () {
784+
const fastListing = yield* sweepWith(0);
785+
// Two list requests of 20 seconds each use 40 of the 90-second budget.
786+
const slowListing = yield* sweepWith(20_000);
787+
expect(slowListing).toBeLessThan(fastListing);
788+
});
789+
});
790+
753791
it.effect("continues past a page of older hosts to find recoverable tunnels", () => {
754792
const entries = Array.from({ length: 101 }, (_, index) =>
755793
tunnel({

‎infra/relay/src/environments/ManagedEndpointReaper.ts‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,9 @@ export const MANAGED_ENDPOINT_LEGACY_GRACE_PERIOD_DAYS = 30;
3535
// connection while it runs; keep this well under the 20-connection origin
3636
// limit that request handlers share.
3737
export const MANAGED_ENDPOINT_SWEEP_DELETE_CONCURRENCY = 4;
38-
// Stop starting deletions after this long, leaving room under the cron's
39-
// two-minute timeout to finish in-flight ones and record the counters.
38+
// Stop starting deletions this long after the sweep starts, leaving room under
39+
// the cron's two-minute timeout to finish in-flight ones and record the
40+
// counters. Counted from sweep start so slow listing eats into it.
4041
export const MANAGED_ENDPOINT_SWEEP_DELETE_BUDGET_MS = 90_000;
4142

4243
export interface ManagedEndpointSweepResult {
@@ -191,6 +192,7 @@ export const make = Effect.gen(function* () {
191192
if ((mode === "off" && legacyMode === "off") || !namespace) {
192193
return emptyResult(mode, legacyMode);
193194
}
195+
const sweepStartedAtMillis = yield* Clock.currentTimeMillis;
194196
// The override exists for the disposable canary stage; prod always
195197
// waits the full grace period.
196198
const legacyGraceMinutes =
@@ -366,8 +368,7 @@ export const make = Effect.gen(function* () {
366368
});
367369
}
368370

369-
const deleteDeadline =
370-
(yield* Clock.currentTimeMillis) + MANAGED_ENDPOINT_SWEEP_DELETE_BUDGET_MS;
371+
const deleteDeadline = sweepStartedAtMillis + MANAGED_ENDPOINT_SWEEP_DELETE_BUDGET_MS;
371372
let stopDeleting = false;
372373
yield* Effect.forEach(
373374
candidates,

0 commit comments

Comments
 (0)