Skip to content

Commit 9f4458d

Browse files
fix(relay): tunnel limit counts abandoned reprovisions and frees failed revokes
The live-tunnel branch of the quota count ignored any withdrawn link, so a revoked environment that was reprovisioned but never had its link rewritten held a Cloudflare tunnel while counting as zero. Bound it the same way as the pending branch: a link withdrawn before the allocation's last provisioning write no longer hides it. Teardown claims (claimDeprovision, unmarked claimRelease) no longer move updatedAt, so a revoke whose Cloudflare cleanup fails stays excluded instead of being counted again on every retry. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 593fc2a commit 9f4458d

6 files changed

Lines changed: 225 additions & 78 deletions

File tree

‎infra/relay/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
"@cloudflare/workers-types": "^4.20260601.1",
3434
"@effect/platform-node": "catalog:",
3535
"@effect/vitest": "catalog:",
36+
"@electric-sql/pglite": "0.3.15",
3637
"@types/node": "catalog:",
3738
"drizzle-kit": "1.0.0-rc.5-ab785fc",
3839
"typescript": "catalog:",

‎infra/relay/src/environments/ManagedEndpointAllocations.test.ts‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,4 +74,31 @@ describe("ManagedEndpointAllocations", () => {
7474
]);
7575
}).pipe(Effect.provide(layerWithDb(fakeDb)));
7676
});
77+
78+
it.effect("teardown claims leave updatedAt to provisioning writes", () => {
79+
// The tunnel limit compares link and allocation updatedAt; a claim that bumps it makes a
80+
// failed revoke cleanup look like a reprovision and keeps holding the user's slot.
81+
const writes: Array<Record<string, unknown>> = [];
82+
const fakeDb = {
83+
update: () => ({
84+
set: (values: Record<string, unknown>) => {
85+
writes.push(values);
86+
return {
87+
where: () => ({ returning: () => Effect.succeed([{ generation: 2 }]) }),
88+
};
89+
},
90+
}),
91+
} as unknown as RelayDb.RelayDb["Service"];
92+
93+
return Effect.gen(function* () {
94+
const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations;
95+
const key = { userId: "user-1", environmentId: "environment-1", generation: 1 };
96+
yield* allocations.claimDeprovision(key);
97+
yield* allocations.claimRelease({ ...key, tunnelId: "tunnel-1" });
98+
yield* allocations.claimRelease({ ...key, tunnelId: "tunnel-1", markReleased: true });
99+
100+
expect(writes.map((values) => "updatedAt" in values)).toEqual([false, false, true]);
101+
expect(writes[2]?.["updatedAt"]).toBe(writes[2]?.["tunnelReleasedAt"]);
102+
}).pipe(Effect.provide(layerWithDb(fakeDb)));
103+
});
77104
});

‎infra/relay/src/environments/ManagedEndpointAllocations.ts‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -512,9 +512,8 @@ const make = Effect.gen(function* () {
512512
const claimed = yield* db
513513
.update(relayManagedEndpointAllocations)
514514
.set({
515-
updatedAt: now,
516515
generation: sql`${relayManagedEndpointAllocations.generation} + 1`,
517-
...(input.markReleased === true ? { tunnelReleasedAt: now } : {}),
516+
...(input.markReleased === true ? { tunnelReleasedAt: now, updatedAt: now } : {}),
518517
})
519518
.where(
520519
and(
@@ -585,10 +584,8 @@ const make = Effect.gen(function* () {
585584
) {
586585
const claimed = yield* db
587586
.update(relayManagedEndpointAllocations)
588-
.set({
589-
updatedAt: DateTime.formatIso(yield* DateTime.now),
590-
generation: sql`${relayManagedEndpointAllocations.generation} + 1`,
591-
})
587+
// Claims move only the generation: the tunnel limit reads updatedAt as the last provisioning write.
588+
.set({ generation: sql`${relayManagedEndpointAllocations.generation} + 1` })
592589
.where(
593590
and(
594591
whereAllocation(input),

‎infra/relay/src/environments/ManagedTunnelLimits.test.ts‎

Lines changed: 137 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,8 @@ import * as Context from "effect/Context";
33
import * as Effect from "effect/Effect";
44
import * as Layer from "effect/Layer";
55
import * as Semaphore from "effect/Semaphore";
6-
import { PgDialect } from "drizzle-orm/pg-core";
6+
import { PGlite } from "@electric-sql/pglite";
7+
import { drizzle } from "drizzle-orm/pglite";
78

89
import * as RelayDb from "../db.ts";
910
import { relayManagedEndpointAllocations } from "../persistence/schema.ts";
@@ -14,7 +15,7 @@ class FakeTransaction extends Context.Service<FakeTransaction, { holdsLock: bool
1415
) {}
1516

1617
// Models Postgres just enough: the advisory lock only works inside a transaction and is held until it ends.
17-
function makeFakeDb(reserved: ReadonlyArray<unknown>, countConditions: Array<unknown> = []) {
18+
function makeFakeDb(reserved: ReadonlyArray<unknown>) {
1819
const userLock = Semaphore.makeUnsafe(1);
1920
return {
2021
$client: {
@@ -40,11 +41,10 @@ function makeFakeDb(reserved: ReadonlyArray<unknown>, countConditions: Array<unk
4041
}),
4142
select: () => ({
4243
from: (table: unknown) => ({
43-
where: (condition: unknown) => {
44+
where: () => {
4445
if (table !== relayManagedEndpointAllocations) {
4546
return { limit: () => Effect.succeed([]) };
4647
}
47-
countConditions.push(condition);
4848
return Effect.yieldNow.pipe(Effect.as([{ activeTunnels: reserved.length }]));
4949
},
5050
}),
@@ -87,29 +87,140 @@ describe("ManagedTunnelLimits", () => {
8787
);
8888
});
8989

90-
it.effect("pending reservations stop counting once released or withdrawn", () => {
91-
const countConditions: Array<unknown> = [];
92-
93-
return Effect.gen(function* () {
94-
const limits = yield* ManagedTunnelLimits.ManagedTunnelLimits;
95-
yield* limits.withCapacity({ userId: "user-1", environmentId: "environment-1" }, Effect.void);
96-
97-
expect(countConditions).toHaveLength(1);
98-
const query = new PgDialect().sqlToQuery(countConditions[0] as never);
99-
// claimRelease writes updatedAt = tunnelReleasedAt; a later reserve moves updatedAt past it.
100-
expect(query.sql).toContain(
101-
'"relay_managed_endpoint_allocations"."updated_at" > "relay_managed_endpoint_allocations"."tunnel_released_at"',
90+
it.effect("counts each allocation by its tunnel and link state", () =>
91+
Effect.gen(function* () {
92+
const pg = yield* Effect.acquireRelease(
93+
Effect.promise(() => PGlite.create()),
94+
(client) => Effect.promise(() => client.close()),
10295
);
103-
// A link revoked or made publish-only after the last reservation no longer holds a slot.
104-
expect(query.sql).toContain(
105-
'"relay_environment_links"."updated_at" >= "relay_managed_endpoint_allocations"."updated_at"',
96+
yield* Effect.promise(() =>
97+
pg.exec(`
98+
create table relay_managed_endpoint_allocations (
99+
user_id text not null, environment_id text not null, tunnel_id text,
100+
tunnel_released_at text, updated_at text not null
101+
);
102+
create table relay_environment_links (
103+
user_id text not null, environment_id text not null, revoked_at text,
104+
managed_tunnels_enabled boolean not null, updated_at text not null
105+
);
106+
`),
106107
);
107-
}).pipe(
108-
Effect.provide(
109-
ManagedTunnelLimits.layer.pipe(
110-
Layer.provide(Layer.succeed(RelayDb.RelayDb, makeFakeDb([], countConditions))),
108+
const at = (minutes: number) => `2026-10-27T12:${String(minutes).padStart(2, "0")}:00.000Z`;
109+
const cases: ReadonlyArray<{
110+
readonly environmentId: string;
111+
readonly counted: boolean;
112+
readonly allocation: {
113+
readonly tunnelId?: string;
114+
readonly releasedAt?: number;
115+
readonly updatedAt: number;
116+
};
117+
readonly link?: {
118+
readonly revoked?: boolean;
119+
readonly publishOnly?: boolean;
120+
readonly updatedAt: number;
121+
};
122+
}> = [
123+
{ environmentId: "pending", counted: true, allocation: { updatedAt: 50 } },
124+
{ environmentId: "abandoned-reservation", counted: false, allocation: { updatedAt: 10 } },
125+
{
126+
environmentId: "live",
127+
counted: true,
128+
allocation: { tunnelId: "t", updatedAt: 10 },
129+
link: { updatedAt: 12 },
130+
},
131+
{
132+
environmentId: "released",
133+
counted: false,
134+
allocation: { tunnelId: "t", releasedAt: 20, updatedAt: 20 },
135+
},
136+
{
137+
environmentId: "re-reserved-after-release",
138+
counted: true,
139+
allocation: { tunnelId: "t", releasedAt: 20, updatedAt: 50 },
140+
},
141+
{
142+
environmentId: "revoked",
143+
counted: false,
144+
allocation: { tunnelId: "t", updatedAt: 10 },
145+
link: { revoked: true, updatedAt: 20 },
146+
},
147+
{
148+
environmentId: "revoked-pending",
149+
counted: false,
150+
allocation: { updatedAt: 50 },
151+
link: { revoked: true, updatedAt: 55 },
152+
},
153+
{
154+
environmentId: "publish-only",
155+
counted: false,
156+
allocation: { tunnelId: "t", updatedAt: 10 },
157+
link: { publishOnly: true, updatedAt: 20 },
158+
},
159+
// Reprovisioned after a revoke, but the Worker died before rewriting the link.
160+
{
161+
environmentId: "reprovision-before-link",
162+
counted: true,
163+
allocation: { tunnelId: "t", updatedAt: 20 },
164+
link: { revoked: true, updatedAt: 10 },
165+
},
166+
{
167+
environmentId: "reserved-before-link-rewrite",
168+
counted: true,
169+
allocation: { updatedAt: 50 },
170+
link: { revoked: true, updatedAt: 48 },
171+
},
172+
{ environmentId: "self", counted: false, allocation: { tunnelId: "t", updatedAt: 10 } },
173+
];
174+
for (const entry of cases) {
175+
yield* Effect.promise(() =>
176+
pg.query(
177+
"insert into relay_managed_endpoint_allocations values ('user-1', $1, $2, $3, $4)",
178+
[
179+
entry.environmentId,
180+
entry.allocation.tunnelId ?? null,
181+
entry.allocation.releasedAt === undefined ? null : at(entry.allocation.releasedAt),
182+
at(entry.allocation.updatedAt),
183+
],
184+
),
185+
);
186+
if (entry.link !== undefined) {
187+
const link = entry.link;
188+
yield* Effect.promise(() =>
189+
pg.query("insert into relay_environment_links values ('user-1', $1, $2, $3, $4)", [
190+
entry.environmentId,
191+
link.revoked === true ? at(link.updatedAt) : null,
192+
link.publishOnly !== true,
193+
at(link.updatedAt),
194+
]),
195+
);
196+
}
197+
}
198+
yield* Effect.promise(() =>
199+
pg.query(
200+
"insert into relay_managed_endpoint_allocations values ('user-2', 'other-user', 't', null, $1)",
201+
[at(10)],
111202
),
112-
),
113-
);
114-
});
203+
);
204+
205+
const counted = yield* Effect.promise(() =>
206+
drizzle({ client: pg })
207+
.select({ environmentId: relayManagedEndpointAllocations.environmentId })
208+
.from(relayManagedEndpointAllocations)
209+
.where(
210+
ManagedTunnelLimits.countedAllocations({
211+
userId: "user-1",
212+
environmentId: "self",
213+
pendingCutoff: at(45),
214+
}),
215+
),
216+
);
217+
218+
expect(counted.map((row) => row.environmentId).toSorted()).toEqual(
219+
cases
220+
.filter((entry) => entry.counted)
221+
.map((entry) => entry.environmentId)
222+
.toSorted(),
223+
);
224+
}),
225+
);
115226
});

‎infra/relay/src/environments/ManagedTunnelLimits.ts‎

Lines changed: 53 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,57 @@ export class ManagedTunnelLimitExceeded extends Schema.TaggedError<ManagedTunnel
4545
}
4646
}
4747

48+
const allocation = relayManagedEndpointAllocations;
49+
50+
const withdrawnLink = (since: typeof allocation.updatedAt) =>
51+
exists(
52+
new QueryBuilder()
53+
.select({ userId: relayEnvironmentLinks.userId })
54+
.from(relayEnvironmentLinks)
55+
.where(
56+
and(
57+
eq(relayEnvironmentLinks.userId, allocation.userId),
58+
eq(relayEnvironmentLinks.environmentId, allocation.environmentId),
59+
or(
60+
isNotNull(relayEnvironmentLinks.revokedAt),
61+
eq(relayEnvironmentLinks.managedTunnelsEnabled, false),
62+
),
63+
gte(relayEnvironmentLinks.updatedAt, since),
64+
),
65+
),
66+
);
67+
68+
// Counts allocations, not links: a link is written only after its tunnel exists.
69+
// Excludes this environment so relinking at the limit stays idempotent.
70+
// updatedAt moves only on provisioning writes and release, never on teardown claims, so a link
71+
// withdrawn at or after it means cleanup is owed; an older withdrawal means a relink that never finished.
72+
export const countedAllocations = (input: {
73+
readonly userId: string;
74+
readonly environmentId: string;
75+
readonly pendingCutoff: string;
76+
}) =>
77+
and(
78+
eq(allocation.userId, input.userId),
79+
ne(allocation.environmentId, input.environmentId),
80+
or(
81+
// Reserved but not yet holding a tunnel. Release writes updatedAt = tunnelReleasedAt and
82+
// a relink reserves before rewriting its link, so only a newer reservation counts here.
83+
and(
84+
gt(allocation.updatedAt, input.pendingCutoff),
85+
or(
86+
isNull(allocation.tunnelReleasedAt),
87+
gt(allocation.updatedAt, allocation.tunnelReleasedAt),
88+
),
89+
not(withdrawnLink(allocation.updatedAt)),
90+
),
91+
and(
92+
isNotNull(allocation.tunnelId),
93+
isNull(allocation.tunnelReleasedAt),
94+
not(withdrawnLink(allocation.updatedAt)),
95+
),
96+
),
97+
);
98+
4899
export class ManagedTunnelLimits extends Context.Service<
49100
ManagedTunnelLimits,
50101
{
@@ -103,52 +154,10 @@ const make = Effect.gen(function* () {
103154
minutes: MANAGED_TUNNEL_PENDING_WINDOW_MINUTES,
104155
}),
105156
);
106-
// Counts allocations, not links: a link is written only after its tunnel exists.
107-
// Excludes this environment so relinking at the limit stays idempotent.
108-
const allocation = relayManagedEndpointAllocations;
109-
const withdrawnLink = (since?: typeof allocation.updatedAt) =>
110-
exists(
111-
new QueryBuilder()
112-
.select({ userId: relayEnvironmentLinks.userId })
113-
.from(relayEnvironmentLinks)
114-
.where(
115-
and(
116-
eq(relayEnvironmentLinks.userId, allocation.userId),
117-
eq(relayEnvironmentLinks.environmentId, allocation.environmentId),
118-
or(
119-
isNotNull(relayEnvironmentLinks.revokedAt),
120-
eq(relayEnvironmentLinks.managedTunnelsEnabled, false),
121-
),
122-
since === undefined ? undefined : gte(relayEnvironmentLinks.updatedAt, since),
123-
),
124-
),
125-
);
126157
const counted = yield* db
127158
.select({ activeTunnels: count() })
128-
.from(allocation)
129-
.where(
130-
and(
131-
eq(allocation.userId, input.userId),
132-
ne(allocation.environmentId, input.environmentId),
133-
or(
134-
// Reserved but not yet holding a tunnel. Release writes updatedAt = tunnelReleasedAt and
135-
// a relink reserves before rewriting its link, so only a newer reservation counts here.
136-
and(
137-
gt(allocation.updatedAt, pendingCutoff),
138-
or(
139-
isNull(allocation.tunnelReleasedAt),
140-
gt(allocation.updatedAt, allocation.tunnelReleasedAt),
141-
),
142-
not(withdrawnLink(allocation.updatedAt)),
143-
),
144-
and(
145-
isNotNull(allocation.tunnelId),
146-
isNull(allocation.tunnelReleasedAt),
147-
not(withdrawnLink()),
148-
),
149-
),
150-
),
151-
)
159+
.from(relayManagedEndpointAllocations)
160+
.where(countedAllocations({ ...input, pendingCutoff }))
152161
.pipe(
153162
Effect.mapError(
154163
(cause) =>

‎pnpm-lock.yaml‎

Lines changed: 4 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)