Skip to content

Commit c883b9f

Browse files
feat(relay,server,web,mobile): opt-in to hold webhooks while offline
With the new Connect preference on, the relay stores webhook requests for an offline environment for up to 24 hours instead of answering 503; the environment pulls them with its credential when it is back and runs each through the usual token and signature checks. Off by default, so the relay stays a plain proxy unless an environment opts in. Every forward carries x-t3-relay-delivery-id, so a request that reached the environment before a timeout and is later replayed runs once. Webhook tasks gain an optional max age to skip requests that waited too long. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 31f46b3 commit c883b9f

22 files changed

Lines changed: 726 additions & 45 deletions

‎apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx‎

Lines changed: 20 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -893,13 +893,26 @@ function TaskForm({
893893
/>
894894
</>
895895
) : draft.schedule.mode === "webhook" ? (
896-
<WebhookScheduleDetails
897-
environmentId={environmentId}
898-
task={
899-
tasks.data?.tasks.find((task) => task.id === draft.task?.id) ?? draft.task ?? null
900-
}
901-
signatureConfigured={draft.schedule.signature !== null}
902-
/>
896+
<>
897+
<WebhookScheduleDetails
898+
environmentId={environmentId}
899+
task={
900+
tasks.data?.tasks.find((task) => task.id === draft.task?.id) ?? draft.task ?? null
901+
}
902+
signatureConfigured={draft.schedule.signature !== null}
903+
/>
904+
<FormField
905+
label="Skip requests older than (minutes)"
906+
value={draft.schedule.maxDeliveryAgeMinutes}
907+
placeholder="Run every request"
908+
keyboardType="decimal-pad"
909+
disabled={saving}
910+
borderTop
911+
onChange={(maxDeliveryAgeMinutes) =>
912+
setDraft({ ...draft, schedule: { ...draft.schedule, maxDeliveryAgeMinutes } })
913+
}
914+
/>
915+
</>
903916
) : (
904917
<>
905918
<FormField

‎apps/mobile/src/features/settings/scheduledTaskDraft.test.ts‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,22 @@ describe("scheduleDraftForTask", () => {
3232
it("round-trips a webhook schedule without a signature", () => {
3333
const draft = scheduleDraftForTask({ schedule: { type: "webhook", signature: null } });
3434
expect(draft.mode).toBe("webhook");
35-
expect(scheduleFromDraft(draft)).toEqual({ type: "webhook", signature: null });
35+
expect(scheduleFromDraft(draft)).toEqual({
36+
type: "webhook",
37+
signature: null,
38+
maxDeliveryAgeMinutes: null,
39+
});
40+
});
41+
42+
it("round-trips a webhook max age and treats blank input as no limit", () => {
43+
const draft = scheduleDraftForTask({
44+
schedule: { type: "webhook", signature: null, maxDeliveryAgeMinutes: 45 },
45+
});
46+
expect(draft.maxDeliveryAgeMinutes).toBe("45");
47+
expect(scheduleFromDraft(draft)).toMatchObject({ maxDeliveryAgeMinutes: 45 });
48+
expect(scheduleFromDraft({ ...draft, maxDeliveryAgeMinutes: "" })).toMatchObject({
49+
maxDeliveryAgeMinutes: null,
50+
});
3651
});
3752

3853
it("keeps a webhook signature on save without sending a secret", () => {
@@ -44,7 +59,7 @@ describe("scheduleDraftForTask", () => {
4459
const saved = scheduleFromDraft(
4560
scheduleDraftForTask({ schedule: { type: "webhook", signature } }),
4661
);
47-
expect(saved).toEqual({ type: "webhook", signature });
62+
expect(saved).toEqual({ type: "webhook", signature, maxDeliveryAgeMinutes: null });
4863
expect(saved?.type === "webhook" && saved.signature && "secret" in saved.signature).toBe(false);
4964
});
5065
});

‎apps/mobile/src/features/settings/scheduledTaskDraft.ts‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@ export type ScheduleDraft = {
4444
readonly intervalMinutes: string;
4545
/** A webhook signature check configured elsewhere; mobile keeps it but does not edit it. */
4646
readonly signature: ScheduledTaskWebhookSignature | null;
47+
/** Minutes as typed; empty runs every held request regardless of age. */
48+
readonly maxDeliveryAgeMinutes: string;
4749
};
4850

4951
export const DEFAULT_SCHEDULE: ScheduleDraft = {
@@ -52,6 +54,7 @@ export const DEFAULT_SCHEDULE: ScheduleDraft = {
5254
weekdays: [1, 2, 3, 4, 5],
5355
intervalMinutes: "15",
5456
signature: null,
57+
maxDeliveryAgeMinutes: "",
5558
};
5659

5760
/** Prompt a new webhook task starts with: the whole request, which the user can narrow down. */
@@ -74,10 +77,24 @@ export function scheduleDraftForTask(task: Pick<ScheduledTask, "schedule">): Sch
7477
intervalMinutes: String(Math.max(1, task.schedule.everyMs / 60_000)),
7578
};
7679
case "webhook":
77-
return { ...DEFAULT_SCHEDULE, mode: "webhook", signature: task.schedule.signature };
80+
return {
81+
...DEFAULT_SCHEDULE,
82+
mode: "webhook",
83+
signature: task.schedule.signature,
84+
maxDeliveryAgeMinutes:
85+
task.schedule.maxDeliveryAgeMinutes == null
86+
? ""
87+
: String(task.schedule.maxDeliveryAgeMinutes),
88+
};
7889
}
7990
}
8091

92+
/** Blank or invalid input means "no limit"; the server rejects values past the relay's TTL. */
93+
function parseMaxDeliveryAge(value: string): number | null {
94+
const minutes = Number(value.trim());
95+
return value.trim() !== "" && Number.isInteger(minutes) && minutes > 0 ? minutes : null;
96+
}
97+
8198
export function scheduleFromDraft(draft: ScheduleDraft): ScheduledTaskUpsertSchedule | null {
8299
if (draft.mode === "webhook") {
83100
// No secret is sent, so the server keeps the stored one.
@@ -91,6 +108,7 @@ export function scheduleFromDraft(draft: ScheduleDraft): ScheduledTaskUpsertSche
91108
encoding: draft.signature.encoding,
92109
prefix: draft.signature.prefix,
93110
},
111+
maxDeliveryAgeMinutes: parseMaxDeliveryAge(draft.maxDeliveryAgeMinutes),
94112
};
95113
}
96114
if (draft.mode === "interval") {
@@ -145,6 +163,7 @@ function draftSignature(draft: ScheduledTaskDraft): string {
145163
draft.schedule.timeOfDay,
146164
[...draft.schedule.weekdays].sort((a, b) => a - b),
147165
draft.schedule.intervalMinutes,
166+
draft.schedule.maxDeliveryAgeMinutes,
148167
draft.workspace,
149168
draft.baseRef,
150169
draft.checkoutPath,

‎apps/server/src/cloud/config.ts‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ export const RELAY_URL_SECRET = "cloud-relay-url";
1616
export const RELAY_ISSUER_SECRET = "cloud-relay-issuer";
1717
export const RELAY_ENVIRONMENT_CREDENTIAL_SECRET = "cloud-relay-environment-credential";
1818
export const PUBLISH_AGENT_ACTIVITY_SECRET = "cloud-publish-agent-activity";
19+
export const HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET = "cloud-hold-webhooks-while-offline";
1920

2021
export const encodeEndpointRuntimeConfigJson = Schema.encodeEffect(
2122
Schema.fromJsonString(RelayManagedEndpointRuntimeConfig),
@@ -73,3 +74,33 @@ export const readAgentActivityPublishingActive = (
7374
environmentCredential !== ""
7475
);
7576
}).pipe(Effect.orElseSucceed(() => false));
77+
78+
const readSecretString = (
79+
secrets: ServerSecretStore.ServerSecretStore["Service"],
80+
name: string,
81+
): Effect.Effect<string | null> =>
82+
secrets.get(name).pipe(
83+
Effect.map((bytes) =>
84+
Option.isSome(bytes) && bytes.value.length > 0 ? new TextDecoder().decode(bytes.value) : null,
85+
),
86+
Effect.orElseSucceed(() => null),
87+
);
88+
89+
/** The relay URL and environment credential, or null when not linked to T3 Connect. */
90+
export const readRelayConnection = (secrets: ServerSecretStore.ServerSecretStore["Service"]) =>
91+
Effect.all([
92+
readSecretString(secrets, RELAY_URL_SECRET),
93+
readSecretString(secrets, RELAY_ENVIRONMENT_CREDENTIAL_SECRET),
94+
]).pipe(
95+
Effect.map(([url, environmentCredential]) =>
96+
url && environmentCredential ? { url, environmentCredential } : null,
97+
),
98+
);
99+
100+
/** Whether this environment opted in to T3 Connect holding webhooks while it is offline. */
101+
export const readHoldWebhooksWhileOffline = (
102+
secrets: ServerSecretStore.ServerSecretStore["Service"],
103+
) =>
104+
readSecretString(secrets, HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET).pipe(
105+
Effect.map((value) => value === "true"),
106+
);

‎apps/server/src/cloud/http.ts‎

Lines changed: 60 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import {
88
EnvironmentCloudRelayConfigResult,
99
EnvironmentHttpApi,
1010
EnvironmentHttpBadRequestError,
11+
type EnvironmentCloudPreferencesRequest,
1112
EnvironmentHttpConflictError,
1213
EnvironmentHttpInternalServerError,
1314
EnvironmentHttpUnauthorizedError,
@@ -69,6 +70,7 @@ import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
6970
import { requireEnvironmentScope } from "../auth/http.ts";
7071
import * as ServerConfig from "../config.ts";
7172
import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
73+
import { makeRelayEnvironmentClient } from "../relay/relayEnvironmentClient.ts";
7274
import * as AgentAwarenessRelay from "../relay/AgentAwarenessRelay.ts";
7375
import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts";
7476
import {
@@ -86,6 +88,8 @@ import {
8688
encodeEndpointRuntimeConfigJson,
8789
encodeConfirmedOriginJson,
8890
PUBLISH_AGENT_ACTIVITY_SECRET,
91+
HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET,
92+
readRelayConnection,
8993
RELAY_ENVIRONMENT_CREDENTIAL_SECRET,
9094
RELAY_ISSUER_SECRET,
9195
RELAY_URL_SECRET,
@@ -1277,17 +1281,24 @@ export const releaseManagedTunnelOnShutdown = Effect.fn(
12771281
const readCloudLinkState = Effect.fn("environment.cloud.readLinkState")(function* (
12781282
dependencies: CloudHttpDependencies,
12791283
) {
1280-
const [cloudUserId, relayUrl, relayIssuer, endpointRuntimeConfig, publishAgentActivity] =
1281-
yield* Effect.all(
1282-
[
1283-
dependencies.secrets.get(CLOUD_LINKED_USER_ID),
1284-
dependencies.secrets.get(RELAY_URL_SECRET),
1285-
dependencies.secrets.get(RELAY_ISSUER_SECRET),
1286-
dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG),
1287-
dependencies.secrets.get(PUBLISH_AGENT_ACTIVITY_SECRET),
1288-
],
1289-
{ concurrency: 5 },
1290-
);
1284+
const [
1285+
cloudUserId,
1286+
relayUrl,
1287+
relayIssuer,
1288+
endpointRuntimeConfig,
1289+
publishAgentActivity,
1290+
holdWebhooks,
1291+
] = yield* Effect.all(
1292+
[
1293+
dependencies.secrets.get(CLOUD_LINKED_USER_ID),
1294+
dependencies.secrets.get(RELAY_URL_SECRET),
1295+
dependencies.secrets.get(RELAY_ISSUER_SECRET),
1296+
dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG),
1297+
dependencies.secrets.get(PUBLISH_AGENT_ACTIVITY_SECRET),
1298+
dependencies.secrets.get(HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET),
1299+
],
1300+
{ concurrency: 6 },
1301+
);
12911302
return {
12921303
linked: Option.isSome(cloudUserId),
12931304
cloudUserId: Option.isSome(cloudUserId) ? bytesToString(cloudUserId.value) : null,
@@ -1299,6 +1310,8 @@ const readCloudLinkState = Effect.fn("environment.cloud.readLinkState")(function
12991310
publishAgentActivity: Option.isSome(publishAgentActivity)
13001311
? bytesToString(publishAgentActivity.value) === "true"
13011312
: false,
1313+
holdWebhooksWhileOffline:
1314+
Option.isSome(holdWebhooks) && bytesToString(holdWebhooks.value) === "true",
13021315
} satisfies EnvironmentCloudLinkStateResult;
13031316
});
13041317

@@ -1329,8 +1342,9 @@ const cloudUnlinkHandler = Effect.fn("environment.cloud.unlink")(
13291342
dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG),
13301343
dependencies.secrets.remove(CLOUD_ENDPOINT_CONFIRMED_ORIGIN),
13311344
dependencies.secrets.remove(PUBLISH_AGENT_ACTIVITY_SECRET),
1345+
dependencies.secrets.remove(HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET),
13321346
],
1333-
{ concurrency: 8 },
1347+
{ concurrency: 9 },
13341348
);
13351349
yield* setCliDesiredCloudLink(false);
13361350
return { ok: true, endpointRuntimeStatus } satisfies EnvironmentCloudRelayConfigResult;
@@ -1343,12 +1357,42 @@ const cloudUnlinkHandler = Effect.fn("environment.cloud.unlink")(
13431357
),
13441358
);
13451359

1360+
const pushHoldWebhooksWhileOffline = Effect.fn("environment.cloud.pushHoldWebhooksWhileOffline")(
1361+
function* (dependencies: CloudHttpDependencies, holdWebhooksWhileOffline: boolean) {
1362+
const connection = yield* readRelayConnection(dependencies.secrets);
1363+
if (connection === null) {
1364+
return yield* new EnvironmentHttpBadRequestError({
1365+
message: "Link this environment to T3 Connect first.",
1366+
});
1367+
}
1368+
const environmentId = yield* dependencies.environment.getEnvironmentId;
1369+
const client = yield* makeRelayEnvironmentClient(connection);
1370+
yield* client.server
1371+
.updateLinkPreferences({
1372+
params: { environmentId },
1373+
payload: { holdWebhooksWhileOffline },
1374+
})
1375+
.pipe(
1376+
Effect.timeout("10 seconds"),
1377+
Effect.catch(
1378+
failEnvironmentCloudInternalError("Could not update T3 Connect webhook settings."),
1379+
),
1380+
);
1381+
},
1382+
);
1383+
13461384
const cloudPreferencesHandler = Effect.fn("environment.cloud.preferences")(
1347-
function* (
1348-
dependencies: CloudHttpDependencies,
1349-
payload: { readonly publishAgentActivity: boolean },
1350-
) {
1385+
function* (dependencies: CloudHttpDependencies, payload: EnvironmentCloudPreferencesRequest) {
13511386
yield* requireEnvironmentScope(AuthRelayWriteScope);
1387+
if (payload.holdWebhooksWhileOffline !== undefined) {
1388+
// The relay decides whether to hold a request, so it is told first; the
1389+
// local copy is only saved once the relay has the same value.
1390+
yield* pushHoldWebhooksWhileOffline(dependencies, payload.holdWebhooksWhileOffline);
1391+
yield* dependencies.secrets.set(
1392+
HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET,
1393+
stringToBytes(String(payload.holdWebhooksWhileOffline)),
1394+
);
1395+
}
13521396
yield* dependencies.secrets.set(
13531397
PUBLISH_AGENT_ACTIVITY_SECRET,
13541398
stringToBytes(String(payload.publishAgentActivity)),
Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
import { assert, it } from "@effect/vitest";
2+
import type { RelayPendingHook } from "@t3tools/contracts/relay";
3+
import * as Effect from "effect/Effect";
4+
import * as Layer from "effect/Layer";
5+
6+
import {
7+
ScheduledTaskService,
8+
type WebhookTriggerRequest,
9+
type WebhookTriggerResult,
10+
} from "../scheduledTasks/ScheduledTaskService.ts";
11+
import { drainOnce, HookMailboxRelay, toTriggerRequest } from "./HookMailboxDrain.ts";
12+
13+
const held = (id: string, overrides: Partial<RelayPendingHook> = {}): RelayPendingHook => ({
14+
id,
15+
receivedAt: "2026-10-04T10:00:00.000Z",
16+
method: "POST",
17+
rawHookId: "scheduled-task%3Ahook",
18+
rawToken: "tok%2Fen",
19+
query: "a=1",
20+
headers: { "Content-Type": "application/json" },
21+
bodyBase64: Buffer.from('{"x":1}').toString("base64"),
22+
...overrides,
23+
});
24+
25+
/** Runs one drain pass against a relay holding `pages` and a service answering `outcome`. */
26+
const drain = (
27+
pages: ReadonlyArray<ReadonlyArray<RelayPendingHook>> | null,
28+
outcome: (request: WebhookTriggerRequest) => WebhookTriggerResult["_tag"],
29+
) =>
30+
Effect.gen(function* () {
31+
const acked: Array<string> = [];
32+
const triggered: Array<WebhookTriggerRequest> = [];
33+
let page = 0;
34+
const relay = Layer.succeed(HookMailboxRelay, {
35+
pending: Effect.sync(() => (pages === null ? null : (pages[page++] ?? []))),
36+
ack: (ids) => Effect.sync(() => void acked.push(...ids)),
37+
});
38+
const service = Layer.mock(ScheduledTaskService)({
39+
triggerWebhook: (request) =>
40+
Effect.sync(() => {
41+
triggered.push(request);
42+
return { _tag: outcome(request) } as WebhookTriggerResult;
43+
}),
44+
});
45+
const delivered = yield* drainOnce().pipe(Effect.provide(Layer.merge(relay, service)));
46+
return { acked, triggered, delivered };
47+
});
48+
49+
it("rebuilds the request the webhook route would have built", () => {
50+
const request = toTriggerRequest(held("d1"));
51+
assert.equal(request?.hookId, "scheduled-task:hook");
52+
assert.equal(request?.token, "tok/en");
53+
assert.equal(request?.headers["content-type"], "application/json");
54+
assert.equal(request?.bodyText, '{"x":1}');
55+
assert.equal(request?.relayDeliveryId, "d1");
56+
assert.equal(request?.receivedAt, "2026-10-04T10:00:00.000Z");
57+
});
58+
59+
it.effect("delivers held requests and acks every final outcome", () =>
60+
Effect.gen(function* () {
61+
const result = yield* drain([[held("ok"), held("gone"), held("bad-sig")]], (request) =>
62+
request.relayDeliveryId === "ok"
63+
? "accepted"
64+
: request.relayDeliveryId === "gone"
65+
? "not_found"
66+
: "rejected_signature",
67+
);
68+
assert.deepEqual(result.acked, ["ok", "gone", "bad-sig"]);
69+
}),
70+
);
71+
72+
it.effect("leaves rate-limited requests held for the next pass", () =>
73+
Effect.gen(function* () {
74+
const result = yield* drain(
75+
[[held("first"), held("limited")], [held("never-pulled")]],
76+
(request) => (request.relayDeliveryId === "limited" ? "rate_limited" : "accepted"),
77+
);
78+
assert.deepEqual(result.acked, ["first"]);
79+
// The pass stops instead of pulling the next page.
80+
assert.isFalse(result.triggered.some((request) => request.relayDeliveryId === "never-pulled"));
81+
}),
82+
);
83+
84+
it.effect("acks and drops a request whose path cannot be decoded", () =>
85+
Effect.gen(function* () {
86+
const result = yield* drain([[held("broken", { rawToken: "%E0" })]], () => "accepted");
87+
assert.deepEqual(result.acked, ["broken"]);
88+
assert.equal(result.triggered.length, 0);
89+
}),
90+
);
91+
92+
it.effect("does nothing when the environment is not linked", () =>
93+
Effect.gen(function* () {
94+
const result = yield* drain(null, () => "accepted");
95+
assert.equal(result.delivered, 0);
96+
assert.equal(result.triggered.length, 0);
97+
}),
98+
);

0 commit comments

Comments
 (0)