Skip to content

Commit 6327a63

Browse files
feat(relay): relay alerts, warnings and health dashboard via Alchemy (#22067)
Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 3284143 commit 6327a63

6 files changed

Lines changed: 360 additions & 0 deletions

File tree

‎.github/workflows/deploy-relay.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@ jobs:
4141
RELAY_TUNNEL_ZONE_NAME: ${{ vars.RELAY_TUNNEL_ZONE_NAME }}
4242
RELAY_TUNNEL_CLEANUP_MODE: ${{ vars.RELAY_TUNNEL_CLEANUP_MODE }}
4343
RELAY_LEGACY_TUNNEL_CLEANUP_MODE: ${{ vars.RELAY_LEGACY_TUNNEL_CLEANUP_MODE }}
44+
RELAY_ALERT_NOTIFIER_ID: ${{ vars.RELAY_ALERT_NOTIFIER_ID }}
45+
RELAY_WARNING_NOTIFIER_ID: ${{ vars.RELAY_WARNING_NOTIFIER_ID }}
4446
CLERK_PUBLISHABLE_KEY: ${{ vars.CLERK_PUBLISHABLE_KEY }}
4547
CLERK_JWT_AUDIENCE: ${{ vars.CLERK_JWT_AUDIENCE }}
4648
APNS_ENVIRONMENT: ${{ vars.APNS_ENVIRONMENT }}

‎docs/operations/relay-observability.md‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,3 +108,39 @@ Questions these answer:
108108
) on trace_id
109109
| summarize runs = count(), failures = max(failures), backlog = max(backlog) by endpoint
110110
```
111+
112+
## Alerts
113+
114+
The stack provisions threshold monitors and a `t3-code-relay-health` dashboard from
115+
[`infra/relay/src/monitoring.ts`](../../infra/relay/src/monitoring.ts) when the deployment sets
116+
`RELAY_ALERT_NOTIFIER_ID`. Paging monitors go to that notifier and need two failing runs in a row.
117+
Warnings go to `RELAY_WARNING_NOTIFIER_ID` when it is set and are otherwise only visible in Axiom.
118+
Each monitor's description gives its healthy baseline.
119+
120+
Monitors count HTTP status codes, not span errors. A request the relay cuts off at its 9 s deadline
121+
ends as an interrupted span without an error flag, but still answers 504. `/connect` 502s are
122+
normal: offline hosts produce most of them, retried by their clients every few seconds.
123+
124+
**Links failing / tunnel recovery failing.** Open the dashboard's "Link failures by reason" table.
125+
`provisioning: ensure-tunnel` or `configure-tunnel` with Cloudflare 429s on the "Cloudflare API calls
126+
by status" chart is the #16810 failure: the account's Cloudflare API rate limit. Ask Cloudflare to
127+
raise it, and check whether something is calling the API in a loop (tunnel recovery from hosts is the
128+
largest caller). Slow Postgres instead points at the database or Hyperdrive.
129+
130+
**Environments unreachable.** Clients reach fewer hosts than usual. Check Cloudflare Tunnel status and
131+
recent relay deploys before suspecting hosts: a host-side problem does not move this many
132+
environments at once.
133+
134+
**Push delivery failing.** Look at the `relay.apns_deliveries.process_signed_job` or `relay.fcm.process`
135+
error messages. An expired APNs key or FCM service account fails every job.
136+
137+
**Cloudflare API rate limited / near its limit.** The warning fires on sustained 429s; the page fires
138+
when they are frequent enough to stall provisioning. Compare the call volume on the dashboard with the
139+
account limit.
140+
141+
**Traffic dropped.** The Worker is down, a deploy broke it, or spans stopped reaching Axiom. Check
142+
`https://relay.t3.codes/health` first.
143+
144+
**Links degraded / database slow.** Warnings for slow degradation. Link failures above 20% over six
145+
hours have so far meant provisioning running into the 9 s deadline, with Postgres latency as the
146+
usual cause.

‎infra/relay/.env.example‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,13 @@ RELAY_TUNNEL_ZONE_NAME=tunnels.example.com
1515
# Ignored on prod.
1616
# RELAY_LEGACY_TUNNEL_GRACE_MINUTES=10
1717

18+
# Optional: relay monitors and health dashboard in Axiom. Set the notifier id
19+
# that pages (incident.io in prod) to create them; unset creates neither, so
20+
# personal stages never page anyone. Warnings go to their own notifier, or
21+
# only show in Axiom when it is unset.
22+
# RELAY_ALERT_NOTIFIER_ID=
23+
# RELAY_WARNING_NOTIFIER_ID=
24+
1825
# Optional: Relay domain override
1926
# Set this only when the derived relay hostname should not be used.
2027
# RELAY_DOMAIN=relay.example.com

‎infra/relay/README.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,10 @@ The `production` GitHub environment must define these Actions variables:
141141
- `RELAY_API_ZONE_NAME`
142142
- `RELAY_TUNNEL_ZONE_NAME`
143143
- `RELAY_DOMAIN` if overriding the derived production relay domain
144+
- `RELAY_ALERT_NOTIFIER_ID`, the Axiom notifier that relay alerts page. Without it the deployment
145+
creates no monitors or health dashboard. See
146+
[Relay alerts](../../docs/operations/relay-observability.md#alerts).
147+
- `RELAY_WARNING_NOTIFIER_ID`, optional, the Axiom notifier for relay warnings
144148
- `CLERK_PUBLISHABLE_KEY`
145149
- `CLERK_JWT_AUDIENCE`
146150
- `CLERK_JWT_TEMPLATE`

‎infra/relay/src/monitoring.ts‎

Lines changed: 308 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,308 @@
1+
import * as Alchemy from "alchemy";
2+
import * as Axiom from "alchemy/Axiom";
3+
import * as Output from "alchemy/Output";
4+
import * as Config from "effect/Config";
5+
import * as Effect from "effect/Effect";
6+
import * as Option from "effect/Option";
7+
8+
import { relayResourceNameForStage } from "./deploymentConfig.ts";
9+
10+
const RUNBOOK_URL =
11+
"https://github.com/pingdotgg/t3code/blob/main/docs/operations/relay-observability.md#alerts";
12+
13+
const LINK_ROUTE = `"/v1/client/environment-links"`;
14+
const TUNNEL_ROUTES = `"/v1/environments/:environmentId/tunnel", "/v1/environments/:environmentId/tunnel/recovery"`;
15+
const CLOUDFLARE_API = `['attributes.url.full'] startswith "https://api.cloudflare.com"`;
16+
const STATUS = `toint(['attributes.http.response.status_code'])`;
17+
18+
// Status codes, not span errors: a request cut off by the relay deadline ends
19+
// as an interrupted span with no error flag, but still answers 504.
20+
// Below `minimum` requests the rate reads 0, so a quiet window cannot page.
21+
const serverFailurePercent = (dataset: string, routes: string, minimum: number) =>
22+
[
23+
`['${dataset}']`,
24+
`| where kind == "server" and ['attributes.http.request.method'] == "POST" and ['attributes.http.route'] in (${routes})`,
25+
`| summarize total = count(), failed = countif(${STATUS} >= 500)`,
26+
`| project failure_pct = iff(total >= ${minimum}, 100.0 * failed / total, 0.0)`,
27+
].join("\n");
28+
29+
type Severity = "page" | "warn";
30+
31+
interface RelayMonitor {
32+
readonly id: string;
33+
readonly severity: Severity;
34+
readonly name: string;
35+
/** Why it fires and the healthy baseline. The runbook link is appended. */
36+
readonly description: string;
37+
readonly query: (dataset: string) => string;
38+
readonly operator: "Above" | "AboveOrEqual" | "Below";
39+
readonly threshold: number;
40+
readonly rangeMinutes: number;
41+
readonly intervalMinutes?: number;
42+
readonly triggerFromNRuns?: number;
43+
readonly alertOnNoData?: boolean;
44+
}
45+
46+
// Thresholds come from 2026-10-16..19 prod data: the worst healthy window,
47+
// and the #16810 outage (Cloudflare API 429s stalling every provision).
48+
// Paging monitors need two bad runs in a row; warnings catch the slow
49+
// degradations an outage threshold has to ignore.
50+
const relayMonitors: ReadonlyArray<RelayMonitor> = [
51+
{
52+
id: "RelayLinkFailureRateMonitor",
53+
severity: "page",
54+
name: "T3 Connect relay: links failing",
55+
description:
56+
"Share of POST /v1/client/environment-links (turning on T3 Connect) answering 5xx over 15 min. Healthy: median 11%, worst 75%. #16810: 87-96%.",
57+
query: (dataset) => serverFailurePercent(dataset, LINK_ROUTE, 20),
58+
operator: "Above",
59+
threshold: 80,
60+
rangeMinutes: 15,
61+
triggerFromNRuns: 2,
62+
},
63+
{
64+
id: "RelayTunnelFailureRateMonitor",
65+
severity: "page",
66+
name: "T3 Connect relay: tunnel recovery failing",
67+
description:
68+
"Share of host tunnel registration and recovery answering 5xx over 15 min. Hosts whose tunnel drops cannot come back. Healthy: median 4%, worst 52%. #16810: 83-94%.",
69+
query: (dataset) => serverFailurePercent(dataset, TUNNEL_ROUTES, 200),
70+
operator: "Above",
71+
threshold: 60,
72+
rangeMinutes: 15,
73+
triggerFromNRuns: 2,
74+
},
75+
{
76+
id: "RelayConnectReachabilityMonitor",
77+
severity: "page",
78+
name: "T3 Connect relay: environments unreachable",
79+
description:
80+
"Share of environments clients tried to reach that connected at least once in 30 min. Offline hosts keep this below 100%, and retry storms against them do not move it. Healthy: 65-72%.",
81+
query: (dataset) =>
82+
[
83+
`['${dataset}']`,
84+
`| where name == "relay.environment_connector.connect"`,
85+
`| extend environment = tostring(['attributes.custom']['relay.environment_id'])`,
86+
`| summarize environments = dcount(environment), connected = dcountif(environment, error != true)`,
87+
`| project reachable_pct = iff(environments >= 1000, 100.0 * connected / environments, 100.0)`,
88+
].join("\n"),
89+
operator: "Below",
90+
threshold: 55,
91+
rangeMinutes: 30,
92+
triggerFromNRuns: 2,
93+
},
94+
{
95+
id: "RelayPushFailureRateMonitor",
96+
severity: "page",
97+
name: "T3 Connect relay: push delivery failing",
98+
description:
99+
"Share of APNs and FCM delivery jobs failing over 15 min, e.g. an expired APNs key or FCM credential. Healthy: APNs under 0.3%, FCM under 4%.",
100+
query: (dataset) =>
101+
[
102+
`['${dataset}']`,
103+
`| where name in ("relay.apns_deliveries.process_signed_job", "relay.fcm.process")`,
104+
`| summarize total = count(), failed = countif(error == true)`,
105+
`| project failure_pct = iff(total >= 100, 100.0 * failed / total, 0.0)`,
106+
].join("\n"),
107+
operator: "Above",
108+
threshold: 10,
109+
rangeMinutes: 15,
110+
triggerFromNRuns: 2,
111+
},
112+
{
113+
id: "RelayCloudflareRateLimitedMonitor",
114+
severity: "page",
115+
name: "T3 Connect relay: Cloudflare API rate limited",
116+
description:
117+
"Cloudflare API 429s per 10 min from tunnel and DNS provisioning. Sustained at this level, links and recoveries stall. Peak before #16810: 1,286; during it: ~11k.",
118+
query: (dataset) =>
119+
`['${dataset}']\n| where ${CLOUDFLARE_API} and ${STATUS} == 429\n| summarize rate_limited = count()`,
120+
operator: "AboveOrEqual",
121+
threshold: 500,
122+
rangeMinutes: 10,
123+
triggerFromNRuns: 2,
124+
},
125+
{
126+
id: "RelayTrafficDroppedMonitor",
127+
severity: "page",
128+
name: "T3 Connect relay: traffic dropped",
129+
description:
130+
"Relay requests per 10 min collapsed: Worker down, broken deploy, or trace export stopped. Healthy minimum: ~69k.",
131+
query: (dataset) => `['${dataset}']\n| where kind == "server"\n| summarize requests = count()`,
132+
operator: "Below",
133+
threshold: 10_000,
134+
rangeMinutes: 10,
135+
triggerFromNRuns: 2,
136+
alertOnNoData: true,
137+
},
138+
{
139+
id: "RelayCloudflareRateLimitWarningMonitor",
140+
severity: "warn",
141+
name: "T3 Connect relay: Cloudflare API near its limit",
142+
description:
143+
"Any sustained Cloudflare API 429s. These bursts ran for days before #16810; raise the account limit or cut call volume before they grow.",
144+
query: (dataset) =>
145+
`['${dataset}']\n| where ${CLOUDFLARE_API} and ${STATUS} == 429\n| summarize rate_limited = count()`,
146+
operator: "AboveOrEqual",
147+
threshold: 20,
148+
rangeMinutes: 10,
149+
triggerFromNRuns: 2,
150+
},
151+
{
152+
id: "RelayLinkChronicFailureMonitor",
153+
severity: "warn",
154+
name: "T3 Connect relay: links degraded",
155+
description:
156+
"Share of links answering 5xx over 6 h, mostly 9 s deadline 504s from slow provisioning. Healthy days: 10-18%. Degraded days before #16810: 28-33%.",
157+
query: (dataset) => serverFailurePercent(dataset, LINK_ROUTE, 100),
158+
operator: "Above",
159+
threshold: 20,
160+
rangeMinutes: 360,
161+
intervalMinutes: 30,
162+
},
163+
{
164+
id: "RelayDatabaseLatencyMonitor",
165+
severity: "warn",
166+
name: "T3 Connect relay: database slow",
167+
description:
168+
"p90 Postgres query time over 15 min, in ms. Every relay request runs several queries in sequence, so this sets how close provisioning gets to the 9 s deadline. Healthy: ~200 ms.",
169+
query: (dataset) =>
170+
`['${dataset}']\n| where name == "postgres"\n| summarize p90 = percentile(duration, 90)\n| project p90_ms = p90 / 1000000`,
171+
operator: "Above",
172+
threshold: 500,
173+
rangeMinutes: 15,
174+
triggerFromNRuns: 2,
175+
},
176+
];
177+
178+
interface QueryChart {
179+
readonly id: string;
180+
readonly name: string;
181+
readonly type: "TimeSeries" | "Table";
182+
readonly query: { readonly apl: string };
183+
}
184+
185+
const timeSeries = (id: string, name: string, apl: string): QueryChart => ({
186+
id,
187+
name,
188+
type: "TimeSeries",
189+
query: { apl },
190+
});
191+
192+
const relayDashboardCharts = (dataset: string): ReadonlyArray<QueryChart> => {
193+
const source = `['${dataset}']`;
194+
const linkRequests = `${source}\n| where kind == "server" and ['attributes.http.request.method'] == "POST" and ['attributes.http.route'] == ${LINK_ROUTE}`;
195+
const tunnelRequests = `${source}\n| where kind == "server" and ['attributes.http.request.method'] == "POST" and ['attributes.http.route'] in (${TUNNEL_ROUTES})`;
196+
return [
197+
timeSeries(
198+
"links",
199+
"Links by outcome",
200+
`${linkRequests}\n| summarize succeeded = countif(${STATUS} < 400), rejected = countif(${STATUS} >= 400 and ${STATUS} < 500), failed = countif(${STATUS} >= 500) by bin_auto(_time)`,
201+
),
202+
timeSeries(
203+
"link-latency",
204+
"Link latency (ms, 9 s deadline)",
205+
`${linkRequests}\n| summarize p50 = percentile(duration, 50) / 1000000, p90 = percentile(duration, 90) / 1000000 by bin_auto(_time)`,
206+
),
207+
timeSeries(
208+
"reachability",
209+
"Environments reachable per 30 min (%)",
210+
`${source}\n| where name == "relay.environment_connector.connect"\n| extend environment = tostring(['attributes.custom']['relay.environment_id'])\n| summarize environments = dcount(environment), connected = dcountif(environment, error != true) by bin(_time, 30m)\n| project _time, reachable_pct = 100.0 * connected / environments`,
211+
),
212+
timeSeries(
213+
"tunnel",
214+
"Tunnel registration and recovery by outcome",
215+
`${tunnelRequests}\n| summarize succeeded = countif(${STATUS} < 400), rejected = countif(${STATUS} >= 400 and ${STATUS} < 500), failed = countif(${STATUS} >= 500) by bin_auto(_time)`,
216+
),
217+
timeSeries(
218+
"cloudflare-api",
219+
"Cloudflare API calls by status",
220+
`${source}\n| where ${CLOUDFLARE_API}\n| summarize count() by bin_auto(_time), status = ${STATUS}`,
221+
),
222+
timeSeries(
223+
"postgres",
224+
"Postgres query time (ms)",
225+
`${source}\n| where name == "postgres"\n| summarize p50 = percentile(duration, 50) / 1000000, p90 = percentile(duration, 90) / 1000000 by bin_auto(_time)`,
226+
),
227+
timeSeries(
228+
"push",
229+
"Push delivery jobs",
230+
`${source}\n| where name in ("relay.apns_deliveries.process_signed_job", "relay.fcm.process")\n| summarize delivered = countif(error != true), failed = countif(error == true) by bin_auto(_time), service = iff(name == "relay.fcm.process", "fcm", "apns")`,
231+
),
232+
timeSeries(
233+
"server-errors",
234+
"5xx by route",
235+
`${source}\n| where kind == "server" and ${STATUS} >= 500\n| summarize count() by bin_auto(_time), route = tostring(['attributes.http.route']), status = ${STATUS}`,
236+
),
237+
{
238+
id: "link-failures",
239+
name: "Link failures by reason",
240+
type: "Table",
241+
query: {
242+
apl: `${source}\n| where name == "relay.environment_linker.link" and error == true\n| extend message = tostring(['status.message'])\n| summarize failures = count() by reason = case(message contains "failed during '", strcat("provisioning: ", extract("failed during '([^']+)'", 1, message)), message startswith "Managed tunnel limit reached", "managed tunnel limit reached", substring(message, 0, 80))\n| top 10 by failures`,
243+
},
244+
},
245+
];
246+
};
247+
248+
const notifierConfig = (name: string) =>
249+
Config.String(name).pipe(
250+
Config.option,
251+
Config.map(Option.flatMap((value) => Option.liftPredicate(value.trim(), Boolean))),
252+
);
253+
254+
/**
255+
* A deployment opts in by naming the notifier that pages (incident.io for
256+
* prod). Personal stages and forks leave it unset and get no monitors or
257+
* dashboard. Warnings go to their own notifier when one is set, and are only
258+
* visible in Axiom otherwise.
259+
*/
260+
export const RelayMonitoring = Effect.fn("RelayMonitoring")(function* (traces: Axiom.Dataset) {
261+
const { stage } = yield* Alchemy.Stack;
262+
const page = yield* notifierConfig("RELAY_ALERT_NOTIFIER_ID");
263+
if (Option.isNone(page)) return;
264+
const notifiers: Record<Severity, Option.Option<string>> = {
265+
page,
266+
warn: yield* notifierConfig("RELAY_WARNING_NOTIFIER_ID"),
267+
};
268+
269+
for (const monitor of relayMonitors) {
270+
yield* Axiom.Monitor(monitor.id, {
271+
name: stage === "prod" ? monitor.name : `[${stage}] ${monitor.name}`,
272+
description: `${monitor.description} Runbook: ${RUNBOOK_URL}`,
273+
type: "Threshold",
274+
aplQuery: Output.map(traces.name, monitor.query),
275+
operator: monitor.operator,
276+
threshold: monitor.threshold,
277+
intervalMinutes: monitor.intervalMinutes ?? 5,
278+
rangeMinutes: monitor.rangeMinutes,
279+
triggerFromNRuns: monitor.triggerFromNRuns ?? 1,
280+
alertOnNoData: monitor.alertOnNoData ?? false,
281+
notifierIds: Option.toArray(notifiers[monitor.severity]),
282+
});
283+
}
284+
285+
yield* Axiom.Dashboard("RelayHealthDashboard", {
286+
dashboard: Output.map(traces.name, (dataset) => {
287+
const charts = relayDashboardCharts(dataset);
288+
return {
289+
name: relayResourceNameForStage("t3-code-relay-health", stage),
290+
owner: "",
291+
description: `T3 Connect relay health. Alerts and runbook: ${RUNBOOK_URL}`,
292+
refreshTime: 60,
293+
schemaVersion: 2,
294+
timeWindowStart: "qr-now-24h",
295+
timeWindowEnd: "qr-now",
296+
charts: [...charts],
297+
// Two charts per row on Axiom's 12-column grid.
298+
layout: charts.map((chart, index) => ({
299+
i: chart.id,
300+
x: (index % 2) * 6,
301+
y: Math.floor(index / 2) * 6,
302+
w: 6,
303+
h: 6,
304+
})),
305+
};
306+
}),
307+
});
308+
});

‎infra/relay/src/observability.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import * as Schema from "effect/Schema";
99
import * as Tracer from "effect/Tracer";
1010

1111
import { relayResourceNameForStage } from "./deploymentConfig.ts";
12+
import { RelayMonitoring } from "./monitoring.ts";
1213

1314
const relayRecentSpansQuery = (dataset: string) =>
1415
[
@@ -62,6 +63,8 @@ export const RelayObservability = Effect.gen(function* () {
6263
aplQuery: Output.map(traces.name, relayRecentSpansQuery),
6364
});
6465

66+
yield* RelayMonitoring(traces);
67+
6568
return { traces, workerIngestToken, mobileIngestToken, clientIngestToken } as const;
6669
});
6770

0 commit comments

Comments
 (0)