Skip to content

Commit 86c6b7e

Browse files
fix(relay,server): held webhooks keep their receive time and one cap per hook
- The relay sends `x-t3-relay-received-at` with the delivery id, and the environment logs that time and checks the per-task max age against it. The relay strips any copy a sender supplied. - The per-hook inbox cap is keyed by the decoded hook id, so `hook` and `%68ook` share one cap. - Upstream responses are read with a 64 KiB cap instead of buffered whole. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 3e2c2de commit 86c6b7e

7 files changed

Lines changed: 94 additions & 13 deletions

File tree

‎apps/server/src/scheduledTasks/webhookRoute.test.ts‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,32 @@ describe("webhook route", () => {
7777
}
7878
});
7979

80+
it("uses the relay's receive time only alongside its delivery id", async () => {
81+
const received: Array<WebhookTriggerRequest> = [];
82+
const { handler, dispose } = handlerFor((request) => {
83+
received.push(request);
84+
return Effect.succeed({
85+
_tag: "accepted",
86+
deliveryId: ScheduledTaskWebhookDeliveryId.make("delivery:1"),
87+
});
88+
});
89+
try {
90+
const receivedAt = "2026-10-04T10:00:00.000Z";
91+
await handler(
92+
post("/api/hooks/id/tok", "{}", {
93+
"x-t3-relay-delivery-id": "relay-1",
94+
"x-t3-relay-received-at": receivedAt,
95+
}),
96+
);
97+
await handler(post("/api/hooks/id/tok", "{}", { "x-t3-relay-received-at": receivedAt }));
98+
expect(received[0]?.relayDeliveryId).toBe("relay-1");
99+
expect(received[0]?.receivedAt).toBe(receivedAt);
100+
expect(received[1]?.receivedAt).toBeUndefined();
101+
} finally {
102+
await dispose();
103+
}
104+
});
105+
80106
it("maps service outcomes to status codes", async () => {
81107
const cases: ReadonlyArray<[WebhookTriggerResult["_tag"], number]> = [
82108
["not_found", 404],

‎apps/server/src/scheduledTasks/webhookRoute.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,8 +54,10 @@ const handleWebhook =
5454
if (typeof value === "string") headers[name.toLowerCase()] = value;
5555
}
5656
const queryIndex = request.url.indexOf("?");
57-
// Only the relay sets this; it strips any copy a sender supplied.
57+
// Only the relay sets these; it strips any copy a sender supplied. The
58+
// receive time matters for requests the relay held while we were offline.
5859
const relayDeliveryId = headers["x-t3-relay-delivery-id"];
60+
const relayReceivedAt = relayDeliveryId ? headers["x-t3-relay-received-at"] : undefined;
5961

6062
const result = yield* scheduledTasks
6163
.triggerWebhook({
@@ -68,6 +70,7 @@ const handleWebhook =
6870
body: body.value,
6971
bodyText: new TextDecoder().decode(body.value),
7072
...(relayDeliveryId ? { relayDeliveryId } : {}),
73+
...(relayReceivedAt ? { receivedAt: relayReceivedAt } : {}),
7174
})
7275
.pipe(
7376
Effect.catch((cause) =>

‎infra/relay/src/hooks/HookForwarder.test.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -507,7 +507,7 @@ describe("HookForwarder", () => {
507507
});
508508
const body = new Uint8Array([0, 255, 10]);
509509
const response = yield* harness.send(
510-
new Request(hookUrl("hook-1/tok%2Fen", "?a=1"), {
510+
new Request(hookUrl("%68ook-1/tok%2Fen", "?a=1"), {
511511
method: "POST",
512512
body,
513513
headers: { "x-t3-relay-delivery-id": "forged", "x-sig": "s" },
@@ -520,6 +520,8 @@ describe("HookForwarder", () => {
520520
expect(hook?.query).toBe("a=1");
521521
expect([...(hook?.body ?? [])]).toEqual([0, 255, 10]);
522522
expect(hook?.headers["x-sig"]).toBe("s");
523+
// Every spelling of the hook id shares one per-hook cap.
524+
expect(hook?.hookKey).toBe("hook-1");
523525
// The sender cannot choose the delivery id.
524526
expect(hook?.headers["x-t3-relay-delivery-id"]).toBeUndefined();
525527
expect(hook?.id).not.toBe("forged");

‎infra/relay/src/hooks/HookForwarder.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ const DROPPED_REQUEST_HEADERS = new Set([
3636
"x-real-ip",
3737
// Only the relay may set this; a sender could otherwise collide delivery ids.
3838
"x-t3-relay-delivery-id",
39+
"x-t3-relay-received-at",
3940
]);
4041
const DROPPED_REQUEST_HEADER_PREFIXES = ["proxy-", "cf-", "x-forwarded-"];
4142

@@ -275,9 +276,11 @@ const make = Effect.gen(function* () {
275276
// timeout and is later delivered from the inbox runs only once.
276277
const hook = {
277278
id: yield* crypto.randomUUIDv4.pipe(Effect.orDie),
279+
receivedAt,
278280
method: request.method,
279281
rawHookId: parsed.rawHookId,
280282
rawToken: parsed.rawToken,
283+
hookKey: parsed.hookId,
281284
query: parsed.search.replace(/^\?/, ""),
282285
headers: forwardedHeaders(request.headers),
283286
body: body.success,
@@ -293,7 +296,7 @@ const make = Effect.gen(function* () {
293296
.hold({
294297
environmentId: parsed.environmentId,
295298
baseUrl: endpoint.httpBaseUrl,
296-
hook: { ...hook, receivedAt },
299+
hook,
297300
})
298301
.pipe(
299302
Effect.catch((cause) =>

‎infra/relay/src/hooks/HookInboxStore.test.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ const hook = (id: string, overrides: Partial<HookInboxStore.HeldHook> = {}) =>
1616
receivedAt: DateTime.formatIso(DateTime.makeUnsafe(now)),
1717
method: "POST",
1818
rawHookId: "hook-1",
19+
hookKey: "hook-1",
1920
rawToken: "tok%2Fen",
2021
query: "a=1",
2122
headers: { "content-type": "application/json", "x-sig": "s" },
@@ -123,7 +124,7 @@ describe("HookInboxStore", () => {
123124
}
124125
expect(yield* HookInboxStore.hold(yield* hook("one-too-many"), BASE_URL)).toBeNull();
125126
// Another hook still has room.
126-
const other = yield* hook("other", { rawHookId: "hook-2" });
127+
const other = yield* hook("other", { rawHookId: "hook-2", hookKey: "hook-2" });
127128
expect(yield* HookInboxStore.hold(other, BASE_URL)).not.toBeNull();
128129
}),
129130
),
@@ -139,7 +140,7 @@ describe("HookInboxStore", () => {
139140
const small = new Uint8Array(11);
140141
expect(
141142
yield* HookInboxStore.hold(
142-
yield* hook("small", { rawHookId: "x", body: small }),
143+
yield* hook("small", { rawHookId: "x", hookKey: "x", body: small }),
143144
BASE_URL,
144145
),
145146
).toBeNull();

‎infra/relay/src/hooks/HookInboxStore.ts‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ export interface HeldHook {
3333
/** Path segments exactly as the sender sent them; the environment decodes them. */
3434
readonly rawHookId: string;
3535
readonly rawToken: string;
36+
/** The decoded hook id, so every spelling of one hook shares its cap. */
37+
readonly hookKey: string;
3638
/** Without the leading `?`. */
3739
readonly query: string;
3840
readonly headers: Readonly<Record<string, string>>;
@@ -56,6 +58,7 @@ interface HeldHookRow {
5658
readonly method: string;
5759
readonly raw_hook_id: string;
5860
readonly raw_token: string;
61+
readonly hook_key: string;
5962
readonly query: string;
6063
readonly headers: string;
6164
readonly body: Uint8Array;
@@ -78,12 +81,13 @@ export const migrate = Effect.gen(function* () {
7881
method TEXT NOT NULL,
7982
raw_hook_id TEXT NOT NULL,
8083
raw_token TEXT NOT NULL,
84+
hook_key TEXT NOT NULL,
8185
query TEXT NOT NULL,
8286
headers TEXT NOT NULL,
8387
body BLOB NOT NULL
8488
)
8589
`;
86-
yield* sql`CREATE INDEX IF NOT EXISTS held_hooks_hook ON held_hooks (raw_hook_id)`;
90+
yield* sql`CREATE INDEX IF NOT EXISTS held_hooks_hook ON held_hooks (hook_key)`;
8791
// Where to push, and how many attempts in a row have failed. One row.
8892
yield* sql`
8993
CREATE TABLE IF NOT EXISTS held_hooks_target (
@@ -134,11 +138,12 @@ export const hold = Effect.fn("HookInboxStore.hold")(function* (hook: HeldHook,
134138
const sql = yield* SqlClient.SqlClient;
135139
// One statement, so the caps hold however many requests arrive at once.
136140
const inserted = yield* sql<{ readonly id: string }>`
137-
INSERT INTO held_hooks (id, received_at, method, raw_hook_id, raw_token, query, headers, body)
141+
INSERT INTO held_hooks
142+
(id, received_at, method, raw_hook_id, raw_token, hook_key, query, headers, body)
138143
SELECT ${hook.id}, ${hook.receivedAt}, ${hook.method}, ${hook.rawHookId}, ${hook.rawToken},
139-
${hook.query}, ${encodeHeaders(hook.headers)}, ${hook.body}
144+
${hook.hookKey}, ${hook.query}, ${encodeHeaders(hook.headers)}, ${hook.body}
140145
WHERE (SELECT count(*) FROM held_hooks) < ${HOOK_INBOX_MAX_REQUESTS}
141-
AND (SELECT count(*) FROM held_hooks WHERE raw_hook_id = ${hook.rawHookId})
146+
AND (SELECT count(*) FROM held_hooks WHERE hook_key = ${hook.hookKey})
142147
< ${HOOK_INBOX_MAX_PER_HOOK}
143148
AND (SELECT coalesce(sum(length(body)), 0) FROM held_hooks) + ${hook.body.byteLength}
144149
<= ${HOOK_INBOX_MAX_BYTES}
@@ -180,7 +185,7 @@ export const deliverDue = Effect.fn("HookInboxStore.deliverDue")(function* <R>(
180185
yield* sql`DELETE FROM held_hooks WHERE received_at < ${iso(startedAt - HOOK_INBOX_TTL_MS)}`;
181186
const target = yield* readTarget;
182187
const batch = yield* sql<HeldHookRow>`
183-
SELECT id, received_at, method, raw_hook_id, raw_token, query, headers, body
188+
SELECT id, received_at, method, raw_hook_id, raw_token, hook_key, query, headers, body
184189
FROM held_hooks ORDER BY seq LIMIT ${DELIVERIES_PER_RUN}
185190
`;
186191
if (batch.length === 0 || target === null) {
@@ -196,6 +201,7 @@ export const deliverDue = Effect.fn("HookInboxStore.deliverDue")(function* <R>(
196201
method: row.method,
197202
rawHookId: row.raw_hook_id,
198203
rawToken: row.raw_token,
204+
hookKey: row.hook_key,
199205
query: row.query,
200206
headers: decodeHeaders(row.headers),
201207
body: row.body,

‎infra/relay/src/hooks/upstream.ts‎

Lines changed: 43 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,27 @@
11
import * as Duration from "effect/Duration";
22
import * as Effect from "effect/Effect";
3+
import * as Schema from "effect/Schema";
4+
import * as Stream from "effect/Stream";
35
import * as HttpClient from "effect/unstable/http/HttpClient";
46
import * as HttpClientRequest from "effect/unstable/http/HttpClientRequest";
7+
import type * as HttpClientResponse from "effect/unstable/http/HttpClientResponse";
58

69
import { withoutRedirects } from "../environments/EnvironmentConnector.ts";
710

811
/** Set by the relay on every forward; the environment uses it as the delivery id. */
912
const RELAY_DELIVERY_ID_HEADER = "x-t3-relay-delivery-id";
13+
/** When the relay received the request; the environment trusts it only with the delivery id. */
14+
const RELAY_RECEIVED_AT_HEADER = "x-t3-relay-received-at";
15+
/** The environment answers with a small JSON status; anything past this is cut off. */
16+
const MAX_RESPONSE_BYTES = 64 * 1024;
1017
export const RELAY_HOOK_UPSTREAM_TIMEOUT_MS = 8_000;
1118
// Cloudflare answers 530 when the tunnel for a hostname has no connected origin.
1219
export const TUNNEL_OFFLINE_STATUS = 530;
1320

1421
/** A webhook request as the relay sends it on to the environment. */
1522
export interface UpstreamHook {
1623
readonly id: string;
24+
readonly receivedAt: string;
1725
readonly method: string;
1826
/** Path segments exactly as the sender sent them; the environment decodes them. */
1927
readonly rawHookId: string;
@@ -30,6 +38,37 @@ export interface UpstreamResponse {
3038
readonly body: Uint8Array;
3139
}
3240

41+
class ResponseTooLarge extends Schema.TaggedError<ResponseTooLarge>()("ResponseTooLarge", {}) {}
42+
43+
/** Reads a response body, failing once it passes the cap rather than buffering it all. */
44+
const readCapped = (response: HttpClientResponse.HttpClientResponse) =>
45+
Effect.suspend(() => {
46+
const chunks: Array<Uint8Array> = [];
47+
let total = 0;
48+
return response.stream.pipe(
49+
Stream.runForEach((chunk) => {
50+
total += chunk.length;
51+
if (total > MAX_RESPONSE_BYTES) return Effect.fail(new ResponseTooLarge());
52+
chunks.push(chunk);
53+
return Effect.void;
54+
}),
55+
Effect.map(() => {
56+
const body = new Uint8Array(total);
57+
let offset = 0;
58+
for (const chunk of chunks) {
59+
body.set(chunk, offset);
60+
offset += chunk.length;
61+
}
62+
return body;
63+
}),
64+
// A response without a body, such as a redirect, has no stream at all.
65+
Effect.catchIf(
66+
(error) => error._tag === "HttpClientError" && error.reason._tag === "EmptyBodyError",
67+
() => Effect.succeed(new Uint8Array(0)),
68+
),
69+
);
70+
});
71+
3372
/**
3473
* Sends a webhook request through the environment's tunnel. Fails when the
3574
* environment cannot be reached, and succeeds with None on timeout.
@@ -41,6 +80,7 @@ export const sendUpstream = (baseUrl: string, hook: UpstreamHook) =>
4180
const headers: Record<string, string> = {
4281
...hook.headers,
4382
[RELAY_DELIVERY_ID_HEADER]: hook.id,
83+
[RELAY_RECEIVED_AT_HEADER]: hook.receivedAt,
4484
};
4585
let request = HttpClientRequest.make(hook.method as "GET" | "POST" | "PUT" | "PATCH")(
4686
`${base}api/hooks/${hook.rawHookId}/${hook.rawToken}${hook.query ? `?${hook.query}` : ""}`,
@@ -51,11 +91,11 @@ export const sendUpstream = (baseUrl: string, hook: UpstreamHook) =>
5191
}
5292
return yield* httpClient.execute(request).pipe(
5393
Effect.flatMap((response) =>
54-
response.arrayBuffer.pipe(
55-
Effect.map((bytes): UpstreamResponse => ({
94+
readCapped(response).pipe(
95+
Effect.map((body): UpstreamResponse => ({
5696
status: response.status,
5797
contentType: response.headers["content-type"],
58-
body: new Uint8Array(bytes),
98+
body,
5999
})),
60100
),
61101
),

0 commit comments

Comments
 (0)