Skip to content

Commit 4427358

Browse files
fix(relay): export traces through one tracer, one request span each (#16382)
Co-authored-by: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 83b982a commit 4427358

6 files changed

Lines changed: 87 additions & 133 deletions

File tree

‎infra/relay/src/hooks/HookForwarder.test.ts‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -615,15 +615,13 @@ describe("HookForwarder", () => {
615615
});
616616
const harness = makeHarness();
617617
const handler = yield* harness.httpEffect;
618-
// As the worker runtime runs it: its own tracer around ours, off for hook
619-
// paths. This checks the predicate; whether alchemy applies it per event
620-
// is only visible on a deployed worker (see worker.ts).
618+
// As the worker runtime runs it: its own tracer around ours, turned off
619+
// (see worker.ts). Whether alchemy applies the predicate per event is
620+
// only visible on a deployed worker.
621621
yield* HttpMiddleware.tracer(
622622
traceRelayHttpRequestWith(handler, Layer.succeed(Tracer.Tracer, tracer)),
623623
).pipe(
624-
Effect.provideService(HttpMiddleware.TracerDisabledWhen, (request) =>
625-
HookForwarder.isRelayHookPath(request.url),
626-
),
624+
Effect.provideService(HttpMiddleware.TracerDisabledWhen, () => true),
627625
Effect.withTracer(tracer),
628626
Effect.provideService(
629627
HttpServerRequest.HttpServerRequest,

‎infra/relay/src/http/Api.test.ts‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import * as Tracer from "effect/Tracer";
3030
import * as Etag from "effect/http/Etag";
3131
import * as HttpEffect from "effect/http/HttpEffect";
3232
import * as HttpRouter from "effect/http/HttpRouter";
33+
import * as HttpMiddleware from "effect/http/HttpMiddleware";
3334
import * as HttpServerRequest from "effect/http/HttpServerRequest";
3435
import * as HttpServerResponse from "effect/http/HttpServerResponse";
3536
import * as HttpApi from "effect/http-api/HttpApi";
@@ -1195,6 +1196,38 @@ describe("relay request tracing", () => {
11951196
}),
11961197
);
11971198

1199+
it.effect("records one server span inside the worker's disabled HTTP tracer", () =>
1200+
Effect.gen(function* () {
1201+
const spans: Array<Tracer.NativeSpan> = [];
1202+
const tracer = Tracer.make({
1203+
span: (options) => {
1204+
const span = new Tracer.NativeSpan(options);
1205+
spans.push(span);
1206+
return span;
1207+
},
1208+
});
1209+
const request = HttpServerRequest.fromWeb(
1210+
new Request("https://relay.test/v1/mobile/devices", { method: "POST" }),
1211+
);
1212+
1213+
// As the worker runtime runs it: its own tracer around ours, turned off.
1214+
yield* HttpMiddleware.tracer(
1215+
traceRelayHttpRequestWith(
1216+
Effect.succeed(HttpServerResponse.empty({ status: 204 })),
1217+
Layer.empty,
1218+
),
1219+
).pipe(
1220+
Effect.provideService(HttpMiddleware.TracerDisabledWhen, () => true),
1221+
Effect.provideService(HttpServerRequest.HttpServerRequest, request),
1222+
Effect.withTracer(tracer),
1223+
);
1224+
yield* Effect.yieldNow;
1225+
1226+
expect(spans.filter((span) => span.kind === "server")).toHaveLength(1);
1227+
expect(spans[0]?.attributes.get("url.path")).toBe("/v1/mobile/devices");
1228+
}),
1229+
);
1230+
11981231
it.effect("fails hung requests with a 504 before the client's 10s abort", () =>
11991232
Effect.gen(function* () {
12001233
const spans: Array<Tracer.NativeSpan> = [];

‎infra/relay/src/http/Api.ts‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -228,8 +228,12 @@ export const traceRelayHttpRequest = <E, R>(
228228
Effect.andThen(relayRequestDeadline(httpEffect)),
229229
);
230230
if (!isRelayHookPath(request.url)) {
231-
// HttpMiddleware finalizes its span on the dispatcher; do not close a request-scoped exporter first.
232-
return yield* HttpMiddleware.tracer(traced).pipe(Effect.ensuring(Effect.yieldNow));
231+
return yield* HttpMiddleware.tracer(traced).pipe(
232+
// The worker turns its own request span off; this one is ours.
233+
Effect.provideService(HttpMiddleware.TracerDisabledWhen, () => false),
234+
// HttpMiddleware finalizes its span on the dispatcher; do not close a request-scoped exporter first.
235+
Effect.ensuring(Effect.yieldNow),
236+
);
233237
}
234238
// Hook URLs carry a secret token: the tracer and deadline log see a redacted
235239
// request, while the route itself still receives the original. A webhook
@@ -253,7 +257,7 @@ export const traceRelayHttpRequest = <E, R>(
253257
),
254258
).pipe(
255259
Effect.provideService(HttpServerRequest.HttpServerRequest, redacted),
256-
// The worker disables its own span for hook paths; this one is ours.
260+
// The worker turns its own request span off; this one is ours.
257261
Effect.provideService(HttpMiddleware.TracerDisabledWhen, () => false),
258262
Effect.ensuring(Effect.yieldNow),
259263
);
Lines changed: 15 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -1,46 +1,20 @@
1-
import * as NodeHttpServer from "@effect/platform-node/NodeHttpServer";
21
import { expect, it } from "@effect/vitest";
3-
import * as Deferred from "effect/Deferred";
42
import * as Effect from "effect/Effect";
5-
import * as Redacted from "effect/Redacted";
6-
import * as Schema from "effect/Schema";
7-
import * as HttpServer from "effect/http/HttpServer";
8-
import * as HttpServerRequest from "effect/http/HttpServerRequest";
9-
import * as HttpServerResponse from "effect/http/HttpServerResponse";
10-
import type { OtlpTracer } from "effect/observability";
3+
import * as Tracer from "effect/Tracer";
114

125
import * as EnvironmentConnector from "./environments/EnvironmentConnector.ts";
136
import * as Observability from "./observability.ts";
147

15-
interface ExportedRequest {
16-
readonly authorization: string | undefined;
17-
readonly body: string;
18-
readonly dataset: string | undefined;
19-
}
20-
21-
const otlpAttributeValue = (value: {
22-
readonly stringValue?: string | null;
23-
readonly boolValue?: boolean | null;
24-
readonly intValue?: string | number | null;
25-
readonly doubleValue?: number | null;
26-
}) => value.stringValue ?? value.boolValue ?? value.intValue ?? value.doubleValue;
27-
28-
const decodeJson = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Unknown));
29-
30-
it.effect("exports schema error fields as span attributes", () =>
8+
it.effect("adds schema error fields to spans on the current tracer", () =>
319
Effect.gen(function* () {
32-
const exportedRequest = yield* Deferred.make<ExportedRequest>();
33-
yield* HttpServer.serveEffect(
34-
Effect.gen(function* () {
35-
const request = yield* HttpServerRequest.HttpServerRequest;
36-
yield* Deferred.succeed(exportedRequest, {
37-
authorization: request.headers.authorization,
38-
body: yield* request.text,
39-
dataset: request.headers["x-axiom-dataset"],
40-
});
41-
return HttpServerResponse.empty({ status: 204 });
42-
}),
43-
);
10+
const spans: Array<Tracer.NativeSpan> = [];
11+
const tracer = Tracer.make({
12+
span: (options) => {
13+
const span = new Tracer.NativeSpan(options);
14+
spans.push(span);
15+
return span;
16+
},
17+
});
4418

4519
yield* Effect.fail(
4620
new EnvironmentConnector.EnvironmentConnectNotAuthorized({
@@ -51,44 +25,16 @@ it.effect("exports schema error fields as span attributes", () =>
5125
).pipe(
5226
Effect.withSpan("relay.test.schema_error"),
5327
Effect.exit,
54-
Effect.provide(
55-
Observability.layer({
56-
tracesEndpoint: "/v1/traces",
57-
tracesDatasetName: "relay-test-traces",
58-
ingestToken: Redacted.make("test-token"),
59-
}),
60-
),
28+
Observability.withSchemaErrorSpanAttributes,
29+
Effect.withTracer(tracer),
6130
);
6231

63-
const request = yield* Deferred.await(exportedRequest).pipe(Effect.timeout("1 second"));
64-
const payload = (yield* decodeJson(request.body)) as OtlpTracer.TraceData;
65-
const resourceAttributes = Object.fromEntries(
66-
payload.resourceSpans
67-
.flatMap((resourceSpan) => resourceSpan.resource.attributes)
68-
.map((attribute) => [attribute.key, otlpAttributeValue(attribute.value)]),
69-
);
70-
const span = payload.resourceSpans
71-
.flatMap((resourceSpan) => resourceSpan.scopeSpans)
72-
.flatMap((scopeSpan) => scopeSpan.spans)
73-
.find((candidate) => candidate.name === "relay.test.schema_error");
74-
const attributes = Object.fromEntries(
75-
(span?.attributes ?? []).map((attribute) => [
76-
attribute.key,
77-
otlpAttributeValue(attribute.value),
78-
]),
79-
);
80-
81-
expect(request.authorization).toBe("Bearer test-token");
82-
expect(request.dataset).toBe("relay-test-traces");
83-
expect(resourceAttributes).toMatchObject({
84-
"service.name": "t3code-relay",
85-
"service.namespace": "t3code",
86-
});
87-
expect(attributes).toMatchObject({
32+
expect(spans.map((span) => span.name)).toEqual(["relay.test.schema_error"]);
33+
expect(Object.fromEntries(spans[0]!.attributes)).toMatchObject({
8834
"error.type": "EnvironmentConnectNotAuthorized",
8935
"error.environmentId": "environment-1",
9036
"error.operation": "connect",
9137
"error.reason": "managed_endpoint_allocation_not_ready",
9238
});
93-
}).pipe(Effect.provide(NodeHttpServer.layerTest), Effect.scoped),
39+
}),
9440
);

‎infra/relay/src/observability.ts‎

Lines changed: 11 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,9 @@ import * as Output from "alchemy/Output";
44
import * as Cause from "effect/Cause";
55
import * as Effect from "effect/Effect";
66
import * as Exit from "effect/Exit";
7-
import * as Layer from "effect/Layer";
87
import * as Option from "effect/Option";
9-
import * as Redacted from "effect/Redacted";
108
import * as Schema from "effect/Schema";
119
import * as Tracer from "effect/Tracer";
12-
import { OtlpExporter, OtlpSerialization, OtlpTracer } from "effect/observability";
1310

1411
import { relayResourceNameForStage } from "./deploymentConfig.ts";
1512

@@ -212,27 +209,14 @@ const withSchemaErrorAttributes = (delegate: Tracer.Tracer): Tracer.Tracer =>
212209
...(delegate.context ? { context: delegate.context } : {}),
213210
});
214211

215-
export const layer = (input: {
216-
readonly tracesEndpoint: string;
217-
readonly tracesDatasetName: string;
218-
readonly ingestToken: Redacted.Redacted<string>;
219-
}) =>
220-
Layer.effect(
221-
Tracer.Tracer,
222-
OtlpTracer.make({
223-
url: input.tracesEndpoint,
224-
resource: {
225-
serviceName: "t3code-relay",
226-
attributes: {
227-
"service.namespace": "t3code",
228-
"service.runtime": "cloudflare-worker",
229-
"service.component": "relay",
230-
},
231-
},
232-
headers: {
233-
Authorization: `Bearer ${Redacted.value(input.ingestToken)}`,
234-
"X-Axiom-Dataset": input.tracesDatasetName,
235-
},
236-
exportInterval: "1 second",
237-
}).pipe(Effect.map(withSchemaErrorAttributes)),
238-
).pipe(Layer.provideMerge(OtlpExporter.layerFlusher), Layer.provide(OtlpSerialization.layerJson));
212+
/**
213+
* Adds a failed span's schema error fields (`error.type`, `error.<field>`) to
214+
* the span, on whichever tracer is current. Provide it around the work whose
215+
* spans should carry them; the Worker's telemetry still owns export.
216+
*/
217+
export const withSchemaErrorSpanAttributes = <A, E, R>(
218+
effect: Effect.Effect<A, E, R>,
219+
): Effect.Effect<A, E, R> =>
220+
Effect.flatMap(Effect.tracer, (tracer) =>
221+
effect.pipe(Effect.withTracer(withSchemaErrorAttributes(tracer))),
222+
);

‎infra/relay/src/worker.ts‎

Lines changed: 17 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,7 @@ export const layer = Api.make(
134134
const relayApiZone = yield* RelayApiZone;
135135
const managedEndpointZone = yield* ManagedEndpointZone;
136136
const randomApnsDeliveryJobSigningSecret = yield* ApnsDeliveryJobSigningSecret;
137-
const observability = yield* RelayObservability;
137+
yield* RelayObservability;
138138

139139
//
140140
// 2. Create bindings
@@ -159,10 +159,6 @@ export const layer = Api.make(
159159
const apnsDeliveryQueueSender = yield* Cloudflare.Queues.WriteQueue(apnsDeliveryQueue);
160160
const fcmDeliveryQueueSender = yield* Cloudflare.Queues.WriteQueue(fcmDeliveryQueue);
161161

162-
const axiomDatasetName = yield* observability.traces.name;
163-
const axiomIngestToken = yield* observability.workerIngestToken.token;
164-
const axiomTracesEndpoint = yield* observability.traces.otelTracesEndpoint;
165-
166162
const clerkSecretKey = yield* Config.Redacted("CLERK_SECRET_KEY");
167163
const clerkPublishableKey = yield* Config.String("CLERK_PUBLISHABLE_KEY");
168164
const clerkJwtAudience = yield* Config.String("CLERK_JWT_AUDIENCE");
@@ -231,14 +227,6 @@ export const layer = Api.make(
231227
});
232228
});
233229

234-
const layerRelayTrace = Layer.unwrap(
235-
Effect.all({
236-
tracesDatasetName: axiomDatasetName,
237-
tracesEndpoint: axiomTracesEndpoint,
238-
ingestToken: axiomIngestToken,
239-
}).pipe(Effect.map(Observability.layer)),
240-
);
241-
242230
// Each managed endpoint's held webhook requests live in its own Durable Object.
243231
const inboxCall =
244232
<A>(operation: HookInbox.HookInboxError["operation"], endpointKey: string) =>
@@ -442,8 +430,8 @@ export const layer = Api.make(
442430
{ concurrency: 2, discard: true },
443431
).pipe(
444432
Effect.withSpan("relay.cron.prune_expired_state"),
445-
// Export cron spans to Axiom like HTTP spans; the scope flushes them before the run ends.
446-
Effect.provide(Layer.merge(layerRuntime, layerRelayTrace)),
433+
Observability.withSchemaErrorSpanAttributes,
434+
Effect.provide(layerRuntime),
447435
),
448436
);
449437

@@ -462,7 +450,11 @@ export const layer = Api.make(
462450
HttpRouter.toHttpEffect,
463451
Effect.provideService(HttpRouter.RouterConfig, RELAY_HTTP_ROUTER_CONFIG),
464452
withoutCapturedParentSpan,
465-
Effect.flatMap((httpEffect) => traceRelayHttpRequestWith(httpEffect, layerRelayTrace)),
453+
Effect.map((httpEffect) =>
454+
traceRelayHttpRequestWith(httpEffect, Layer.empty).pipe(
455+
Observability.withSchemaErrorSpanAttributes,
456+
),
457+
),
466458
);
467459

468460
return { fetch };
@@ -477,20 +469,17 @@ export const layer = Api.make(
477469
Layer.provideMerge(Cloudflare.DNS.ReadWriteDnsHttp),
478470
Layer.provideMerge(Cloudflare.Workers.RateLimitBinding),
479471
Layer.provideMerge(HookInboxObject.layer),
480-
// The worker runtime opens its own HTTP span around ours. For webhook
481-
// paths it would record the raw URL, token included, and adopt the
482-
// sender's traceparent, so only our redacted span covers those.
483-
// Registered as telemetry: request-time context is assembled per
484-
// event, and only these layers are built into it.
472+
// The worker runtime opens its own HTTP span around ours. Ours carries
473+
// the route, header redaction and webhook URL redaction, and drops a
474+
// webhook sender's traceparent, so the runtime's span is off for every
475+
// request rather than duplicating it. Registered as telemetry:
476+
// request-time context is assembled per event, and only these layers
477+
// are built into it.
485478
Layer.provideMerge(
486-
Alchemy.Telemetry.layer(
487-
Layer.succeed(HttpMiddleware.TracerDisabledWhen)((request) =>
488-
HookForwarder.isRelayHookPath(request.url),
489-
),
490-
),
479+
Alchemy.Telemetry.layer(Layer.succeed(HttpMiddleware.TracerDisabledWhen)(() => true)),
491480
),
492-
// Exports spans from events the HTTP tracer does not wrap, notably
493-
// HookInboxObject calls and alarms, to the same Axiom dataset.
481+
// The Worker's only trace exporter: every event (fetch, queue, cron,
482+
// HookInboxObject calls and alarms) exports through it to Axiom.
494483
Layer.provideMerge(
495484
Layer.unwrap(
496485
Effect.map(RelayObservability, (observability) =>

0 commit comments

Comments
 (0)