Skip to content

Commit fcf1604

Browse files
fix(relay): export traces through one tracer, one request span each
The relay Worker exported traces twice. Alongside Alchemy's Axiom.Telemetry it built its own OTLP tracer per request, and Alchemy's runtime opened its own http.server span around ours, so every request produced two server spans in Axiom: ours with the route and redaction, and an empty sibling. Axiom.Telemetry is now the only exporter. The runtime's request span is off for every path, leaving ours, and the schema error attributes our tracer added are applied to the current tracer instead. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 8f75697 commit fcf1604

6 files changed

Lines changed: 87 additions & 133 deletions

File tree

‎infra/relay/src/hooks/HookForwarder.test.ts‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -615,15 +615,13 @@ describe("HookForwarder", () => {
615615
});
616616
const harness = makeHarness();
617617
const handler = yield* harness.httpEffect;
618-
// As the worker runtime runs it: its own tracer around ours, off for hook
619-
// paths. This checks the predicate; whether alchemy applies it per event
620-
// is only visible on a deployed worker (see worker.ts).
618+
// As the worker runtime runs it: its own tracer around ours, turned off
619+
// (see worker.ts). Whether alchemy applies the predicate per event is
620+
// only visible on a deployed worker.
621621
yield* HttpMiddleware.tracer(
622622
traceRelayHttpRequestWith(handler, Layer.succeed(Tracer.Tracer, tracer)),
623623
).pipe(
624-
Effect.provideService(HttpMiddleware.TracerDisabledWhen, (request) =>
625-
HookForwarder.isRelayHookPath(request.url),
626-
),
624+
Effect.provideService(HttpMiddleware.TracerDisabledWhen, () => true),
627625
Effect.withTracer(tracer),
628626
Effect.provideService(
629627
HttpServerRequest.HttpServerRequest,

‎infra/relay/src/http/Api.test.ts‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import * as Tracer from "effect/Tracer";
3030
import * as Etag from "effect/http/Etag";
3131
import * as HttpEffect from "effect/http/HttpEffect";
3232
import * as HttpRouter from "effect/http/HttpRouter";
33+
import * as HttpMiddleware from "effect/http/HttpMiddleware";
3334
import * as HttpServerRequest from "effect/http/HttpServerRequest";
3435
import * as HttpServerResponse from "effect/http/HttpServerResponse";
3536
import * as HttpApi from "effect/http-api/HttpApi";
@@ -1195,6 +1196,38 @@ describe("relay request tracing", () => {
11951196
}),
11961197
);
11971198

1199+
it.effect("records one server span inside the worker's disabled HTTP tracer", () =>
1200+
Effect.gen(function* () {
1201+
const spans: Array<Tracer.NativeSpan> = [];
1202+
const tracer = Tracer.make({
1203+
span: (options) => {
1204+
const span = new Tracer.NativeSpan(options);
1205+
spans.push(span);
1206+
return span;
1207+
},
1208+
});
1209+
const request = HttpServerRequest.fromWeb(
1210+
new Request("https://relay.test/v1/mobile/devices", { method: "POST" }),
1211+
);
1212+
1213+
// As the worker runtime runs it: its own tracer around ours, turned off.
1214+
yield* HttpMiddleware.tracer(
1215+
traceRelayHttpRequestWith(
1216+
Effect.succeed(HttpServerResponse.empty({ status: 204 })),
1217+
Layer.empty,
1218+
),
1219+
).pipe(
1220+
Effect.provideService(HttpMiddleware.TracerDisabledWhen, () => true),
1221+
Effect.provideService(HttpServerRequest.HttpServerRequest, request),
1222+
Effect.withTracer(tracer),
1223+
);
1224+
yield* Effect.yieldNow;
1225+
1226+
expect(spans.filter((span) => span.kind === "server")).toHaveLength(1);
1227+
expect(spans[0]?.attributes.get("url.path")).toBe("/v1/mobile/devices");
1228+
}),
1229+
);
1230+
11981231
it.effect("fails hung requests with a 504 before the client's 10s abort", () =>
11991232
Effect.gen(function* () {
12001233
const spans: Array<Tracer.NativeSpan> = [];

‎infra/relay/src/http/Api.ts‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -228,8 +228,12 @@ export const traceRelayHttpRequest = <E, R>(
228228
Effect.andThen(relayRequestDeadline(httpEffect)),
229229
);
230230
if (!isRelayHookPath(request.url)) {
231-
// HttpMiddleware finalizes its span on the dispatcher; do not close a request-scoped exporter first.
232-
return yield* HttpMiddleware.tracer(traced).pipe(Effect.ensuring(Effect.yieldNow));
231+
return yield* HttpMiddleware.tracer(traced).pipe(
232+
// The worker turns its own request span off; this one is ours.
233+
Effect.provideService(HttpMiddleware.TracerDisabledWhen, () => false),
234+
// HttpMiddleware finalizes its span on the dispatcher; do not close a request-scoped exporter first.
235+
Effect.ensuring(Effect.yieldNow),
236+
);
233237
}
234238
// Hook URLs carry a secret token: the tracer and deadline log see a redacted
235239
// request, while the route itself still receives the original. A webhook
@@ -253,7 +257,7 @@ export const traceRelayHttpRequest = <E, R>(
253257
),
254258
).pipe(
255259
Effect.provideService(HttpServerRequest.HttpServerRequest, redacted),
256-
// The worker disables its own span for hook paths; this one is ours.
260+
// The worker turns its own request span off; this one is ours.
257261
Effect.provideService(HttpMiddleware.TracerDisabledWhen, () => false),
258262
Effect.ensuring(Effect.yieldNow),
259263
);
Lines changed: 15 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -1,46 +1,20 @@
1-
import * as NodeHttpServer from "@effect/platform-node/NodeHttpServer";
21
import { expect, it } from "@effect/vitest";
3-
import * as Deferred from "effect/Deferred";
42
import * as Effect from "effect/Effect";
5-
import * as Redacted from "effect/Redacted";
6-
import * as Schema from "effect/Schema";
7-
import * as HttpServer from "effect/http/HttpServer";
8-
import * as HttpServerRequest from "effect/http/HttpServerRequest";
9-
import * as HttpServerResponse from "effect/http/HttpServerResponse";
10-
import type { OtlpTracer } from "effect/observability";
3+
import * as Tracer from "effect/Tracer";
114

125
import * as EnvironmentConnector from "./environments/EnvironmentConnector.ts";
136
import * as Observability from "./observability.ts";
147

15-
interface ExportedRequest {
16-
readonly authorization: string | undefined;
17-
readonly body: string;
18-
readonly dataset: string | undefined;
19-
}
20-
21-
const otlpAttributeValue = (value: {
22-
readonly stringValue?: string | null;
23-
readonly boolValue?: boolean | null;
24-
readonly intValue?: string | number | null;
25-
readonly doubleValue?: number | null;
26-
}) => value.stringValue ?? value.boolValue ?? value.intValue ?? value.doubleValue;
27-
28-
const decodeJson = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Unknown));
29-
30-
it.effect("exports schema error fields as span attributes", () =>
8+
it.effect("adds schema error fields to spans on the current tracer", () =>
319
Effect.gen(function* () {
32-
const exportedRequest = yield* Deferred.make<ExportedRequest>();
33-
yield* HttpServer.serveEffect(
34-
Effect.gen(function* () {
35-
const request = yield* HttpServerRequest.HttpServerRequest;
36-
yield* Deferred.succeed(exportedRequest, {
37-
authorization: request.headers.authorization,
38-
body: yield* request.text,
39-
dataset: request.headers["x-axiom-dataset"],
40-
});
41-
return HttpServerResponse.empty({ status: 204 });
42-
}),
43-
);
10+
const spans: Array<Tracer.NativeSpan> = [];
11+
const tracer = Tracer.make({
12+
span: (options) => {
13+
const span = new Tracer.NativeSpan(options);
14+
spans.push(span);
15+
return span;
16+
},
17+
});
4418

4519
yield* Effect.fail(
4620
new EnvironmentConnector.EnvironmentConnectNotAuthorized({
@@ -51,44 +25,16 @@ it.effect("exports schema error fields as span attributes", () =>
5125
).pipe(
5226
Effect.withSpan("relay.test.schema_error"),
5327
Effect.exit,
54-
Effect.provide(
55-
Observability.layer({
56-
tracesEndpoint: "/v1/traces",
57-
tracesDatasetName: "relay-test-traces",
58-
ingestToken: Redacted.make("test-token"),
59-
}),
60-
),
28+
Observability.withSchemaErrorSpanAttributes,
29+
Effect.withTracer(tracer),
6130
);
6231

63-
const request = yield* Deferred.await(exportedRequest).pipe(Effect.timeout("1 second"));
64-
const payload = (yield* decodeJson(request.body)) as OtlpTracer.TraceData;
65-
const resourceAttributes = Object.fromEntries(
66-
payload.resourceSpans
67-
.flatMap((resourceSpan) => resourceSpan.resource.attributes)
68-
.map((attribute) => [attribute.key, otlpAttributeValue(attribute.value)]),
69-
);
70-
const span = payload.resourceSpans
71-
.flatMap((resourceSpan) => resourceSpan.scopeSpans)
72-
.flatMap((scopeSpan) => scopeSpan.spans)
73-
.find((candidate) => candidate.name === "relay.test.schema_error");
74-
const attributes = Object.fromEntries(
75-
(span?.attributes ?? []).map((attribute) => [
76-
attribute.key,
77-
otlpAttributeValue(attribute.value),
78-
]),
79-
);
80-
81-
expect(request.authorization).toBe("Bearer test-token");
82-
expect(request.dataset).toBe("relay-test-traces");
83-
expect(resourceAttributes).toMatchObject({
84-
"service.name": "t3code-relay",
85-
"service.namespace": "t3code",
86-
});
87-
expect(attributes).toMatchObject({
32+
expect(spans.map((span) => span.name)).toEqual(["relay.test.schema_error"]);
33+
expect(Object.fromEntries(spans[0]!.attributes)).toMatchObject({
8834
"error.type": "EnvironmentConnectNotAuthorized",
8935
"error.environmentId": "environment-1",
9036
"error.operation": "connect",
9137
"error.reason": "managed_endpoint_allocation_not_ready",
9238
});
93-
}).pipe(Effect.provide(NodeHttpServer.layerTest), Effect.scoped),
39+
}),
9440
);

‎infra/relay/src/observability.ts‎

Lines changed: 11 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,9 @@ import * as Output from "alchemy/Output";
44
import * as Cause from "effect/Cause";
55
import * as Effect from "effect/Effect";
66
import * as Exit from "effect/Exit";
7-
import * as Layer from "effect/Layer";
87
import * as Option from "effect/Option";
9-
import * as Redacted from "effect/Redacted";
108
import * as Schema from "effect/Schema";
119
import * as Tracer from "effect/Tracer";
12-
import { OtlpExporter, OtlpSerialization, OtlpTracer } from "effect/observability";
1310

1411
import { relayResourceNameForStage } from "./deploymentConfig.ts";
1512

@@ -212,27 +209,14 @@ const withSchemaErrorAttributes = (delegate: Tracer.Tracer): Tracer.Tracer =>
212209
...(delegate.context ? { context: delegate.context } : {}),
213210
});
214211

215-
export const layer = (input: {
216-
readonly tracesEndpoint: string;
217-
readonly tracesDatasetName: string;
218-
readonly ingestToken: Redacted.Redacted<string>;
219-
}) =>
220-
Layer.effect(
221-
Tracer.Tracer,
222-
OtlpTracer.make({
223-
url: input.tracesEndpoint,
224-
resource: {
225-
serviceName: "t3code-relay",
226-
attributes: {
227-
"service.namespace": "t3code",
228-
"service.runtime": "cloudflare-worker",
229-
"service.component": "relay",
230-
},
231-
},
232-
headers: {
233-
Authorization: `Bearer ${Redacted.value(input.ingestToken)}`,
234-
"X-Axiom-Dataset": input.tracesDatasetName,
235-
},
236-
exportInterval: "1 second",
237-
}).pipe(Effect.map(withSchemaErrorAttributes)),
238-
).pipe(Layer.provideMerge(OtlpExporter.layerFlusher), Layer.provide(OtlpSerialization.layerJson));
212+
/**
213+
* Adds a failed span's schema error fields (`error.type`, `error.<field>`) to
214+
* the span, on whichever tracer is current. Provide it around the work whose
215+
* spans should carry them; the Worker's telemetry still owns export.
216+
*/
217+
export const withSchemaErrorSpanAttributes = <A, E, R>(
218+
effect: Effect.Effect<A, E, R>,
219+
): Effect.Effect<A, E, R> =>
220+
Effect.flatMap(Effect.tracer, (tracer) =>
221+
effect.pipe(Effect.withTracer(withSchemaErrorAttributes(tracer))),
222+
);

‎infra/relay/src/worker.ts‎

Lines changed: 17 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,7 @@ export const layer = Api.make(
134134
const relayApiZone = yield* RelayApiZone;
135135
const managedEndpointZone = yield* ManagedEndpointZone;
136136
const randomApnsDeliveryJobSigningSecret = yield* ApnsDeliveryJobSigningSecret;
137-
const observability = yield* RelayObservability;
137+
yield* RelayObservability;
138138

139139
//
140140
// 2. Create bindings
@@ -159,10 +159,6 @@ export const layer = Api.make(
159159
const apnsDeliveryQueueSender = yield* Cloudflare.Queues.WriteQueue(apnsDeliveryQueue);
160160
const fcmDeliveryQueueSender = yield* Cloudflare.Queues.WriteQueue(fcmDeliveryQueue);
161161

162-
const axiomDatasetName = yield* observability.traces.name;
163-
const axiomIngestToken = yield* observability.workerIngestToken.token;
164-
const axiomTracesEndpoint = yield* observability.traces.otelTracesEndpoint;
165-
166162
const clerkSecretKey = yield* Config.Redacted("CLERK_SECRET_KEY");
167163
const clerkPublishableKey = yield* Config.String("CLERK_PUBLISHABLE_KEY");
168164
const clerkJwtAudience = yield* Config.String("CLERK_JWT_AUDIENCE");
@@ -231,14 +227,6 @@ export const layer = Api.make(
231227
});
232228
});
233229

234-
const layerRelayTrace = Layer.unwrap(
235-
Effect.all({
236-
tracesDatasetName: axiomDatasetName,
237-
tracesEndpoint: axiomTracesEndpoint,
238-
ingestToken: axiomIngestToken,
239-
}).pipe(Effect.map(Observability.layer)),
240-
);
241-
242230
// Each managed endpoint's held webhook requests live in its own Durable Object.
243231
const inboxCall =
244232
<A>(operation: HookInbox.HookInboxError["operation"], endpointKey: string) =>
@@ -442,8 +430,8 @@ export const layer = Api.make(
442430
{ concurrency: 2, discard: true },
443431
).pipe(
444432
Effect.withSpan("relay.cron.prune_expired_state"),
445-
// Export cron spans to Axiom like HTTP spans; the scope flushes them before the run ends.
446-
Effect.provide(Layer.merge(layerRuntime, layerRelayTrace)),
433+
Observability.withSchemaErrorSpanAttributes,
434+
Effect.provide(layerRuntime),
447435
),
448436
);
449437

@@ -462,7 +450,11 @@ export const layer = Api.make(
462450
HttpRouter.toHttpEffect,
463451
Effect.provideService(HttpRouter.RouterConfig, RELAY_HTTP_ROUTER_CONFIG),
464452
withoutCapturedParentSpan,
465-
Effect.flatMap((httpEffect) => traceRelayHttpRequestWith(httpEffect, layerRelayTrace)),
453+
Effect.map((httpEffect) =>
454+
traceRelayHttpRequestWith(httpEffect, Layer.empty).pipe(
455+
Observability.withSchemaErrorSpanAttributes,
456+
),
457+
),
466458
);
467459

468460
return { fetch };
@@ -477,20 +469,17 @@ export const layer = Api.make(
477469
Layer.provideMerge(Cloudflare.DNS.ReadWriteDnsHttp),
478470
Layer.provideMerge(Cloudflare.Workers.RateLimitBinding),
479471
Layer.provideMerge(HookInboxObject.layer),
480-
// The worker runtime opens its own HTTP span around ours. For webhook
481-
// paths it would record the raw URL, token included, and adopt the
482-
// sender's traceparent, so only our redacted span covers those.
483-
// Registered as telemetry: request-time context is assembled per
484-
// event, and only these layers are built into it.
472+
// The worker runtime opens its own HTTP span around ours. Ours carries
473+
// the route, header redaction and webhook URL redaction, and drops a
474+
// webhook sender's traceparent, so the runtime's span is off for every
475+
// request rather than duplicating it. Registered as telemetry:
476+
// request-time context is assembled per event, and only these layers
477+
// are built into it.
485478
Layer.provideMerge(
486-
Alchemy.Telemetry.layer(
487-
Layer.succeed(HttpMiddleware.TracerDisabledWhen)((request) =>
488-
HookForwarder.isRelayHookPath(request.url),
489-
),
490-
),
479+
Alchemy.Telemetry.layer(Layer.succeed(HttpMiddleware.TracerDisabledWhen)(() => true)),
491480
),
492-
// Exports spans from events the HTTP tracer does not wrap, notably
493-
// HookInboxObject calls and alarms, to the same Axiom dataset.
481+
// The Worker's only trace exporter: every event (fetch, queue, cron,
482+
// HookInboxObject calls and alarms) exports through it to Axiom.
494483
Layer.provideMerge(
495484
Layer.unwrap(
496485
Effect.map(RelayObservability, (observability) =>

0 commit comments

Comments
 (0)