|
| 1 | +import * as Alchemy from "alchemy"; |
| 2 | +import * as Axiom from "alchemy/Axiom"; |
| 3 | +import * as Output from "alchemy/Output"; |
| 4 | +import * as Config from "effect/Config"; |
| 5 | +import * as Effect from "effect/Effect"; |
| 6 | +import * as Option from "effect/Option"; |
| 7 | + |
| 8 | +import { relayResourceNameForStage } from "./deploymentConfig.ts"; |
| 9 | + |
| 10 | +const RUNBOOK_URL = |
| 11 | + "https://github.com/pingdotgg/t3code/blob/main/docs/operations/relay-observability.md#alerts"; |
| 12 | + |
| 13 | +const LINK_ROUTE = `"/v1/client/environment-links"`; |
| 14 | +const TUNNEL_ROUTES = `"/v1/environments/:environmentId/tunnel", "/v1/environments/:environmentId/tunnel/recovery"`; |
| 15 | +const CLOUDFLARE_API = `['attributes.url.full'] startswith "https://api.cloudflare.com"`; |
| 16 | +const STATUS = `toint(['attributes.http.response.status_code'])`; |
| 17 | + |
| 18 | +// Status codes, not span errors: a request cut off by the relay deadline ends |
| 19 | +// as an interrupted span with no error flag, but still answers 504. |
| 20 | +// Below `minimum` requests the rate reads 0, so a quiet window cannot page. |
| 21 | +const serverFailurePercent = (dataset: string, routes: string, minimum: number) => |
| 22 | + [ |
| 23 | + `['${dataset}']`, |
| 24 | + `| where kind == "server" and ['attributes.http.request.method'] == "POST" and ['attributes.http.route'] in (${routes})`, |
| 25 | + `| summarize total = count(), failed = countif(${STATUS} >= 500)`, |
| 26 | + `| project failure_pct = iff(total >= ${minimum}, 100.0 * failed / total, 0.0)`, |
| 27 | + ].join("\n"); |
| 28 | + |
| 29 | +type Severity = "page" | "warn"; |
| 30 | + |
| 31 | +interface RelayMonitor { |
| 32 | + readonly id: string; |
| 33 | + readonly severity: Severity; |
| 34 | + readonly name: string; |
| 35 | + /** Why it fires and the healthy baseline. The runbook link is appended. */ |
| 36 | + readonly description: string; |
| 37 | + readonly query: (dataset: string) => string; |
| 38 | + readonly operator: "Above" | "AboveOrEqual" | "Below"; |
| 39 | + readonly threshold: number; |
| 40 | + readonly rangeMinutes: number; |
| 41 | + readonly intervalMinutes?: number; |
| 42 | + readonly triggerFromNRuns?: number; |
| 43 | + readonly alertOnNoData?: boolean; |
| 44 | +} |
| 45 | + |
| 46 | +// Thresholds come from 2026-10-16..19 prod data: the worst healthy window, |
| 47 | +// and the #16810 outage (Cloudflare API 429s stalling every provision). |
| 48 | +// Paging monitors need two bad runs in a row; warnings catch the slow |
| 49 | +// degradations an outage threshold has to ignore. |
| 50 | +const relayMonitors: ReadonlyArray<RelayMonitor> = [ |
| 51 | + { |
| 52 | + id: "RelayLinkFailureRateMonitor", |
| 53 | + severity: "page", |
| 54 | + name: "T3 Connect relay: links failing", |
| 55 | + description: |
| 56 | + "Share of POST /v1/client/environment-links (turning on T3 Connect) answering 5xx over 15 min. Healthy: median 11%, worst 75%. #16810: 87-96%.", |
| 57 | + query: (dataset) => serverFailurePercent(dataset, LINK_ROUTE, 20), |
| 58 | + operator: "Above", |
| 59 | + threshold: 80, |
| 60 | + rangeMinutes: 15, |
| 61 | + triggerFromNRuns: 2, |
| 62 | + }, |
| 63 | + { |
| 64 | + id: "RelayTunnelFailureRateMonitor", |
| 65 | + severity: "page", |
| 66 | + name: "T3 Connect relay: tunnel recovery failing", |
| 67 | + description: |
| 68 | + "Share of host tunnel registration and recovery answering 5xx over 15 min. Hosts whose tunnel drops cannot come back. Healthy: median 4%, worst 52%. #16810: 83-94%.", |
| 69 | + query: (dataset) => serverFailurePercent(dataset, TUNNEL_ROUTES, 200), |
| 70 | + operator: "Above", |
| 71 | + threshold: 60, |
| 72 | + rangeMinutes: 15, |
| 73 | + triggerFromNRuns: 2, |
| 74 | + }, |
| 75 | + { |
| 76 | + id: "RelayConnectReachabilityMonitor", |
| 77 | + severity: "page", |
| 78 | + name: "T3 Connect relay: environments unreachable", |
| 79 | + description: |
| 80 | + "Share of environments clients tried to reach that connected at least once in 30 min. Offline hosts keep this below 100%, and retry storms against them do not move it. Healthy: 65-72%.", |
| 81 | + query: (dataset) => |
| 82 | + [ |
| 83 | + `['${dataset}']`, |
| 84 | + `| where name == "relay.environment_connector.connect"`, |
| 85 | + `| extend environment = tostring(['attributes.custom']['relay.environment_id'])`, |
| 86 | + `| summarize environments = dcount(environment), connected = dcountif(environment, error != true)`, |
| 87 | + `| project reachable_pct = iff(environments >= 1000, 100.0 * connected / environments, 100.0)`, |
| 88 | + ].join("\n"), |
| 89 | + operator: "Below", |
| 90 | + threshold: 55, |
| 91 | + rangeMinutes: 30, |
| 92 | + triggerFromNRuns: 2, |
| 93 | + }, |
| 94 | + { |
| 95 | + id: "RelayPushFailureRateMonitor", |
| 96 | + severity: "page", |
| 97 | + name: "T3 Connect relay: push delivery failing", |
| 98 | + description: |
| 99 | + "Share of APNs and FCM delivery jobs failing over 15 min, e.g. an expired APNs key or FCM credential. Healthy: APNs under 0.3%, FCM under 4%.", |
| 100 | + query: (dataset) => |
| 101 | + [ |
| 102 | + `['${dataset}']`, |
| 103 | + `| where name in ("relay.apns_deliveries.process_signed_job", "relay.fcm.process")`, |
| 104 | + `| summarize total = count(), failed = countif(error == true)`, |
| 105 | + `| project failure_pct = iff(total >= 100, 100.0 * failed / total, 0.0)`, |
| 106 | + ].join("\n"), |
| 107 | + operator: "Above", |
| 108 | + threshold: 10, |
| 109 | + rangeMinutes: 15, |
| 110 | + triggerFromNRuns: 2, |
| 111 | + }, |
| 112 | + { |
| 113 | + id: "RelayCloudflareRateLimitedMonitor", |
| 114 | + severity: "page", |
| 115 | + name: "T3 Connect relay: Cloudflare API rate limited", |
| 116 | + description: |
| 117 | + "Cloudflare API 429s per 10 min from tunnel and DNS provisioning. Sustained at this level, links and recoveries stall. Peak before #16810: 1,286; during it: ~11k.", |
| 118 | + query: (dataset) => |
| 119 | + `['${dataset}']\n| where ${CLOUDFLARE_API} and ${STATUS} == 429\n| summarize rate_limited = count()`, |
| 120 | + operator: "AboveOrEqual", |
| 121 | + threshold: 500, |
| 122 | + rangeMinutes: 10, |
| 123 | + triggerFromNRuns: 2, |
| 124 | + }, |
| 125 | + { |
| 126 | + id: "RelayTrafficDroppedMonitor", |
| 127 | + severity: "page", |
| 128 | + name: "T3 Connect relay: traffic dropped", |
| 129 | + description: |
| 130 | + "Relay requests per 10 min collapsed: Worker down, broken deploy, or trace export stopped. Healthy minimum: ~69k.", |
| 131 | + query: (dataset) => `['${dataset}']\n| where kind == "server"\n| summarize requests = count()`, |
| 132 | + operator: "Below", |
| 133 | + threshold: 10_000, |
| 134 | + rangeMinutes: 10, |
| 135 | + triggerFromNRuns: 2, |
| 136 | + alertOnNoData: true, |
| 137 | + }, |
| 138 | + { |
| 139 | + id: "RelayCloudflareRateLimitWarningMonitor", |
| 140 | + severity: "warn", |
| 141 | + name: "T3 Connect relay: Cloudflare API near its limit", |
| 142 | + description: |
| 143 | + "Any sustained Cloudflare API 429s. These bursts ran for days before #16810; raise the account limit or cut call volume before they grow.", |
| 144 | + query: (dataset) => |
| 145 | + `['${dataset}']\n| where ${CLOUDFLARE_API} and ${STATUS} == 429\n| summarize rate_limited = count()`, |
| 146 | + operator: "AboveOrEqual", |
| 147 | + threshold: 20, |
| 148 | + rangeMinutes: 10, |
| 149 | + triggerFromNRuns: 2, |
| 150 | + }, |
| 151 | + { |
| 152 | + id: "RelayLinkChronicFailureMonitor", |
| 153 | + severity: "warn", |
| 154 | + name: "T3 Connect relay: links degraded", |
| 155 | + description: |
| 156 | + "Share of links answering 5xx over 6 h, mostly 9 s deadline 504s from slow provisioning. Healthy days: 10-18%. Degraded days before #16810: 28-33%.", |
| 157 | + query: (dataset) => serverFailurePercent(dataset, LINK_ROUTE, 100), |
| 158 | + operator: "Above", |
| 159 | + threshold: 20, |
| 160 | + rangeMinutes: 360, |
| 161 | + intervalMinutes: 30, |
| 162 | + }, |
| 163 | + { |
| 164 | + id: "RelayDatabaseLatencyMonitor", |
| 165 | + severity: "warn", |
| 166 | + name: "T3 Connect relay: database slow", |
| 167 | + description: |
| 168 | + "p90 Postgres query time over 15 min, in ms. Every relay request runs several queries in sequence, so this sets how close provisioning gets to the 9 s deadline. Healthy: ~200 ms.", |
| 169 | + query: (dataset) => |
| 170 | + `['${dataset}']\n| where name == "postgres"\n| summarize p90 = percentile(duration, 90)\n| project p90_ms = p90 / 1000000`, |
| 171 | + operator: "Above", |
| 172 | + threshold: 500, |
| 173 | + rangeMinutes: 15, |
| 174 | + triggerFromNRuns: 2, |
| 175 | + }, |
| 176 | +]; |
| 177 | + |
| 178 | +interface QueryChart { |
| 179 | + readonly id: string; |
| 180 | + readonly name: string; |
| 181 | + readonly type: "TimeSeries" | "Table"; |
| 182 | + readonly query: { readonly apl: string }; |
| 183 | +} |
| 184 | + |
| 185 | +const timeSeries = (id: string, name: string, apl: string): QueryChart => ({ |
| 186 | + id, |
| 187 | + name, |
| 188 | + type: "TimeSeries", |
| 189 | + query: { apl }, |
| 190 | +}); |
| 191 | + |
| 192 | +const relayDashboardCharts = (dataset: string): ReadonlyArray<QueryChart> => { |
| 193 | + const source = `['${dataset}']`; |
| 194 | + const linkRequests = `${source}\n| where kind == "server" and ['attributes.http.request.method'] == "POST" and ['attributes.http.route'] == ${LINK_ROUTE}`; |
| 195 | + const tunnelRequests = `${source}\n| where kind == "server" and ['attributes.http.request.method'] == "POST" and ['attributes.http.route'] in (${TUNNEL_ROUTES})`; |
| 196 | + return [ |
| 197 | + timeSeries( |
| 198 | + "links", |
| 199 | + "Links by outcome", |
| 200 | + `${linkRequests}\n| summarize succeeded = countif(${STATUS} < 400), rejected = countif(${STATUS} >= 400 and ${STATUS} < 500), failed = countif(${STATUS} >= 500) by bin_auto(_time)`, |
| 201 | + ), |
| 202 | + timeSeries( |
| 203 | + "link-latency", |
| 204 | + "Link latency (ms, 9 s deadline)", |
| 205 | + `${linkRequests}\n| summarize p50 = percentile(duration, 50) / 1000000, p90 = percentile(duration, 90) / 1000000 by bin_auto(_time)`, |
| 206 | + ), |
| 207 | + timeSeries( |
| 208 | + "reachability", |
| 209 | + "Environments reachable per 30 min (%)", |
| 210 | + `${source}\n| where name == "relay.environment_connector.connect"\n| extend environment = tostring(['attributes.custom']['relay.environment_id'])\n| summarize environments = dcount(environment), connected = dcountif(environment, error != true) by bin(_time, 30m)\n| project _time, reachable_pct = 100.0 * connected / environments`, |
| 211 | + ), |
| 212 | + timeSeries( |
| 213 | + "tunnel", |
| 214 | + "Tunnel registration and recovery by outcome", |
| 215 | + `${tunnelRequests}\n| summarize succeeded = countif(${STATUS} < 400), rejected = countif(${STATUS} >= 400 and ${STATUS} < 500), failed = countif(${STATUS} >= 500) by bin_auto(_time)`, |
| 216 | + ), |
| 217 | + timeSeries( |
| 218 | + "cloudflare-api", |
| 219 | + "Cloudflare API calls by status", |
| 220 | + `${source}\n| where ${CLOUDFLARE_API}\n| summarize count() by bin_auto(_time), status = ${STATUS}`, |
| 221 | + ), |
| 222 | + timeSeries( |
| 223 | + "postgres", |
| 224 | + "Postgres query time (ms)", |
| 225 | + `${source}\n| where name == "postgres"\n| summarize p50 = percentile(duration, 50) / 1000000, p90 = percentile(duration, 90) / 1000000 by bin_auto(_time)`, |
| 226 | + ), |
| 227 | + timeSeries( |
| 228 | + "push", |
| 229 | + "Push delivery jobs", |
| 230 | + `${source}\n| where name in ("relay.apns_deliveries.process_signed_job", "relay.fcm.process")\n| summarize delivered = countif(error != true), failed = countif(error == true) by bin_auto(_time), service = iff(name == "relay.fcm.process", "fcm", "apns")`, |
| 231 | + ), |
| 232 | + timeSeries( |
| 233 | + "server-errors", |
| 234 | + "5xx by route", |
| 235 | + `${source}\n| where kind == "server" and ${STATUS} >= 500\n| summarize count() by bin_auto(_time), route = tostring(['attributes.http.route']), status = ${STATUS}`, |
| 236 | + ), |
| 237 | + { |
| 238 | + id: "link-failures", |
| 239 | + name: "Link failures by reason", |
| 240 | + type: "Table", |
| 241 | + query: { |
| 242 | + apl: `${source}\n| where name == "relay.environment_linker.link" and error == true\n| extend message = tostring(['status.message'])\n| summarize failures = count() by reason = case(message contains "failed during '", strcat("provisioning: ", extract("failed during '([^']+)'", 1, message)), message startswith "Managed tunnel limit reached", "managed tunnel limit reached", substring(message, 0, 80))\n| top 10 by failures`, |
| 243 | + }, |
| 244 | + }, |
| 245 | + ]; |
| 246 | +}; |
| 247 | + |
| 248 | +const notifierConfig = (name: string) => |
| 249 | + Config.String(name).pipe( |
| 250 | + Config.option, |
| 251 | + Config.map(Option.flatMap((value) => Option.liftPredicate(value.trim(), Boolean))), |
| 252 | + ); |
| 253 | + |
| 254 | +/** |
| 255 | + * A deployment opts in by naming the notifier that pages (incident.io for |
| 256 | + * prod). Personal stages and forks leave it unset and get no monitors or |
| 257 | + * dashboard. Warnings go to their own notifier when one is set, and are only |
| 258 | + * visible in Axiom otherwise. |
| 259 | + */ |
| 260 | +export const RelayMonitoring = Effect.fn("RelayMonitoring")(function* (traces: Axiom.Dataset) { |
| 261 | + const { stage } = yield* Alchemy.Stack; |
| 262 | + const page = yield* notifierConfig("RELAY_ALERT_NOTIFIER_ID"); |
| 263 | + if (Option.isNone(page)) return; |
| 264 | + const notifiers: Record<Severity, Option.Option<string>> = { |
| 265 | + page, |
| 266 | + warn: yield* notifierConfig("RELAY_WARNING_NOTIFIER_ID"), |
| 267 | + }; |
| 268 | + |
| 269 | + for (const monitor of relayMonitors) { |
| 270 | + yield* Axiom.Monitor(monitor.id, { |
| 271 | + name: stage === "prod" ? monitor.name : `[${stage}] ${monitor.name}`, |
| 272 | + description: `${monitor.description} Runbook: ${RUNBOOK_URL}`, |
| 273 | + type: "Threshold", |
| 274 | + aplQuery: Output.map(traces.name, monitor.query), |
| 275 | + operator: monitor.operator, |
| 276 | + threshold: monitor.threshold, |
| 277 | + intervalMinutes: monitor.intervalMinutes ?? 5, |
| 278 | + rangeMinutes: monitor.rangeMinutes, |
| 279 | + triggerFromNRuns: monitor.triggerFromNRuns ?? 1, |
| 280 | + alertOnNoData: monitor.alertOnNoData ?? false, |
| 281 | + notifierIds: Option.toArray(notifiers[monitor.severity]), |
| 282 | + }); |
| 283 | + } |
| 284 | + |
| 285 | + yield* Axiom.Dashboard("RelayHealthDashboard", { |
| 286 | + dashboard: Output.map(traces.name, (dataset) => { |
| 287 | + const charts = relayDashboardCharts(dataset); |
| 288 | + return { |
| 289 | + name: relayResourceNameForStage("t3-code-relay-health", stage), |
| 290 | + owner: "", |
| 291 | + description: `T3 Connect relay health. Alerts and runbook: ${RUNBOOK_URL}`, |
| 292 | + refreshTime: 60, |
| 293 | + schemaVersion: 2, |
| 294 | + timeWindowStart: "qr-now-24h", |
| 295 | + timeWindowEnd: "qr-now", |
| 296 | + charts: [...charts], |
| 297 | + // Two charts per row on Axiom's 12-column grid. |
| 298 | + layout: charts.map((chart, index) => ({ |
| 299 | + i: chart.id, |
| 300 | + x: (index % 2) * 6, |
| 301 | + y: Math.floor(index / 2) * 6, |
| 302 | + w: 6, |
| 303 | + h: 6, |
| 304 | + })), |
| 305 | + }; |
| 306 | + }), |
| 307 | + }); |
| 308 | +}); |
0 commit comments