Skip to content

Commit bc81b81

Browse files
committed
Add HttpApi redacted names for client and server span attributes
1 parent 72c3fb4 commit bc81b81

8 files changed

Lines changed: 316 additions & 7 deletions

File tree

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"effect": patch
3+
---
4+
5+
Add the `HttpApi.RedactedNames` annotation for declaring query parameter and header names of an API, group or endpoint to redact in the URL and header attributes of `HttpApiClient` and `HttpApiBuilder` spans, for values that are only sometimes secret. `addHttpApi` treats it like any other annotation, and `user_agent.original` is recorded as is.

‎packages/effect/src/http-api/HttpApi.ts‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import * as InternalRecord from "../internal/record.ts"
1616
import { type Pipeable, pipeArguments } from "../Pipeable.ts"
1717
import * as Predicate from "../Predicate.ts"
1818
import * as Record from "../Record.ts"
19+
import * as RegExp_ from "../RegExp.ts"
1920
import type * as Schema from "../Schema.ts"
2021
import type * as SchemaAST from "../SchemaAST.ts"
2122
import * as HttpApiEndpoint from "./HttpApiEndpoint.ts"
@@ -484,6 +485,52 @@ export class ErrorParseOptions extends Context.Service<
484485
SchemaAST.ParseOptions
485486
>()("effect/http-api/HttpApi/ErrorParseOptions") {}
486487

488+
/**
489+
* Declares query parameters and headers whose values are only sometimes secret,
490+
* so they are redacted in spans without being modelled as `Redacted`.
491+
*
492+
* **Details**
493+
*
494+
* Names are wire names after encoding, matched like `Headers.isRedactedName`:
495+
* strings case-insensitively, regular expressions by test. A string query name
496+
* also matches the bracketed keys of a nested parameter, such as
497+
* `buildargs[NPM_TOKEN]` for `buildargs`. For the endpoint's client request and
498+
* server span, they extend `UrlParams.CurrentRedactedNames` and
499+
* `Headers.CurrentRedactedNames`. Names from the API, its groups and its
500+
* endpoints are combined. Values sent and decoded are unchanged.
501+
*
502+
* **Gotchas**
503+
*
504+
* - `addHttpApi` copies the added API's annotations onto its groups like any
505+
* other annotation, so a group's own `RedactedNames` replaces them. Repeat
506+
* the names on the group to keep both.
507+
* - Only URL and header attributes use these names. `user_agent.original` and
508+
* the messages of span errors are recorded as is.
509+
*
510+
* **Example** (Redacting Docker build arguments)
511+
*
512+
* ```ts
513+
* import { Schema } from "effect"
514+
* import { HttpApi, HttpApiEndpoint } from "effect/http-api"
515+
*
516+
* const Build = HttpApiEndpoint.post("build", "/build", {
517+
* query: { buildargs: Schema.optional(Schema.Record(Schema.String, Schema.String)) },
518+
* headers: { "x-registry-config": Schema.optional(Schema.String) }
519+
* }).annotate(HttpApi.RedactedNames, {
520+
* query: ["buildargs"],
521+
* headers: ["x-registry-config"]
522+
* })
523+
* ```
524+
*
525+
* @stability unstable
526+
* @category services
527+
* @since 4.0.0
528+
*/
529+
export class RedactedNames extends Context.Service<RedactedNames, {
530+
readonly query?: ReadonlyArray<string | RegExp> | undefined
531+
readonly headers?: ReadonlyArray<string | RegExp> | undefined
532+
}>()("effect/http-api/HttpApi/RedactedNames") {}
533+
487534
/**
488535
* @internal
489536
*/
@@ -510,3 +557,36 @@ export const getSlotParseOptions = (annotations: Context.Context<never>): SlotPa
510557
error: Context.getOrUndefined(annotations, ErrorParseOptions) ?? fallback
511558
}
512559
}
560+
561+
/**
562+
* @internal
563+
*/
564+
export interface EndpointRedactedNames {
565+
readonly query: ReadonlyArray<string | RegExp>
566+
readonly headers: ReadonlyArray<string | RegExp>
567+
}
568+
569+
/**
570+
* Combines the `RedactedNames` of an API, group and endpoint. A string query
571+
* name also matches the bracketed keys of a nested parameter.
572+
*
573+
* @internal
574+
*/
575+
export const getEndpointRedactedNames = (
576+
api: Pick<Top, "annotations">,
577+
group: HttpApiGroup.Top,
578+
endpoint: HttpApiEndpoint.Top
579+
): EndpointRedactedNames | undefined => {
580+
const query: Array<string | RegExp> = []
581+
const headers: Array<string | RegExp> = []
582+
for (const annotations of [api.annotations, group.annotations, endpoint.annotations]) {
583+
const options = Context.getOrUndefined(annotations, RedactedNames)
584+
if (options === undefined) continue
585+
for (const name of options.query ?? []) {
586+
query.push(name)
587+
if (typeof name === "string") query.push(new RegExp(`^${RegExp_.escape(name)}\\[`, "i"))
588+
}
589+
if (options.headers !== undefined) headers.push(...options.headers)
590+
}
591+
return query.length === 0 && headers.length === 0 ? undefined : { query, headers }
592+
}

‎packages/effect/src/http-api/HttpApiBuilder.ts‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ import { HttpServerRequest } from "../http/HttpServerRequest.ts"
3131
import * as Respondable from "../http/HttpServerRespondable.ts"
3232
import * as Response from "../http/HttpServerResponse.ts"
3333
import type { HttpServerResponse } from "../http/HttpServerResponse.ts"
34+
import { addRedactedNamesUnsafe } from "../http/internal/requestState.ts"
3435
import * as Multipart from "../http/Multipart.ts"
3536
import * as UrlParams from "../http/UrlParams.ts"
3637
import { stringOrRedacted } from "../internal/redacted.ts"
@@ -966,8 +967,9 @@ function handlerToHttpEffect(
966967

967968
const shouldParsePayload = endpoint.payload.size > 0 && !isRaw
968969
const payloadBy = shouldParsePayload ? buildPayloadDecoders(endpoint.payload, options.payload) : undefined
970+
const redactedNames = HttpApi.getEndpointRedactedNames(api, group, endpoint)
969971

970-
return applyMiddleware(
972+
const effect = applyMiddleware(
971973
group,
972974
endpoint,
973975
context,
@@ -1057,6 +1059,12 @@ function handlerToHttpEffect(
10571059
}),
10581060
Effect.provideContext(context)
10591061
)
1062+
if (redactedNames === undefined) return effect
1063+
// Added before middleware and decoding run, so their failures are redacted too.
1064+
return Effect.withFiber((fiber) => {
1065+
addRedactedNamesUnsafe(fiber.context, redactedNames.query, redactedNames.headers)
1066+
return effect
1067+
})
10601068
}
10611069

10621070
/**

‎packages/effect/src/http-api/HttpApiClient.ts‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ import * as Effect from "../Effect.ts"
1919
import * as Ndjson from "../encoding/Ndjson.ts"
2020
import * as Sse from "../encoding/Sse.ts"
2121
import { identity } from "../Function.ts"
22-
import type * as Headers from "../http/Headers.ts"
22+
import * as Headers from "../http/Headers.ts"
2323
import * as HttpBody from "../http/HttpBody.ts"
2424
import * as HttpClient from "../http/HttpClient.ts"
2525
import * as HttpClientError from "../http/HttpClientError.ts"
@@ -507,6 +507,7 @@ export const makeClient = <ApiId extends string, Groups extends HttpApiGroup.Con
507507
)
508508

509509
const middlewareKeys = Array.from(onEndpointOptions.middleware, (tag) => `${tag.key}/Client`)
510+
const redactedNames = HttpApi.getEndpointRedactedNames(api, group, endpoint)
510511

511512
const endpointFn = Effect.fnUntraced(function*(
512513
request: {
@@ -575,13 +576,14 @@ export const makeClient = <ApiId extends string, Groups extends HttpApiGroup.Con
575576
httpRequest = HttpClientRequest.upgrade(httpRequest, upgrade.protocol)
576577
}
577578

578-
const response = yield* executeMiddleware(
579+
const execute = executeMiddleware(
579580
group,
580581
endpoint,
581582
httpRequest,
582583
middlewareKeys,
583584
middlewareKeys.length - 1
584585
)
586+
const response = yield* (redactedNames === undefined ? execute : withRedactedNames(execute, redactedNames))
585587

586588
if (request?.responseMode === "response-only") {
587589
return response
@@ -603,6 +605,15 @@ export const makeClient = <ApiId extends string, Groups extends HttpApiGroup.Con
603605
})
604606
})
605607

608+
const withRedactedNames = <A, E, R>(
609+
effect: Effect.Effect<A, E, R>,
610+
redactedNames: HttpApi.EndpointRedactedNames
611+
): Effect.Effect<A, E, R> =>
612+
effect.pipe(
613+
Effect.updateService(UrlParams.CurrentRedactedNames, (names) => [...names, ...redactedNames.query]),
614+
Effect.updateService(Headers.CurrentRedactedNames, (names) => [...names, ...redactedNames.headers])
615+
)
616+
606617
/**
607618
* Constructs a type-safe client for an HTTP API using the `HttpClient` service,
608619
* endpoint schemas, middleware, and optional client or response transformations.

‎packages/effect/src/http/HttpMiddleware.ts‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -288,6 +288,12 @@ const responseSpanExit = (
288288
}
289289
}
290290

291+
const withStateNames = (
292+
names: ReadonlyArray<string | RegExp>,
293+
stateNames: ReadonlyArray<string | RegExp> | undefined
294+
): ReadonlyArray<string | RegExp> =>
295+
stateNames === undefined || stateNames.length === 0 ? names : [...names, ...stateNames]
296+
291297
/**
292298
* Middleware that creates a server trace span for each request and records request and response HTTP attributes.
293299
*
@@ -317,7 +323,8 @@ export const tracer: <E, R>(
317323
span.end(endTime, responseSpanExit(fiber, request, exit.value))
318324
return undefined
319325
}
320-
const redactedHeaderNames = fiber.getRef(Headers.CurrentRedactedNames)
326+
const state = Context.getOrUndefined(prevServices, CurrentRequestState)
327+
const redactedHeaderNames = withStateNames(fiber.getRef(Headers.CurrentRedactedNames), state?.redactedHeaderNames)
321328
const headerFilter = fiber.getRef(TracerHeaderFilter)
322329
// Complete the span before the request fiber exits so shutdown can flush it.
323330
let response: HttpServerResponse
@@ -337,7 +344,11 @@ export const tracer: <E, R>(
337344
span.attribute("http.request.method", request.method)
338345
const url = Request.toURL(request)
339346
if (Option.isSome(url)) {
340-
tracing.addUrlAttributes(span, url.value, fiber.getRef(UrlParams.CurrentRedactedNames))
347+
tracing.addUrlAttributes(
348+
span,
349+
url.value,
350+
withStateNames(fiber.getRef(UrlParams.CurrentRedactedNames), state?.redactedUrlParamNames)
351+
)
341352
}
342353
if (request.headers["user-agent"] !== undefined) {
343354
span.attribute("user_agent.original", request.headers["user-agent"])

‎packages/effect/src/http/internal/requestState.ts‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@ import type { PreResponseHandler } from "../HttpEffect.ts"
1010
export class RequestState {
1111
preResponseHandler: PreResponseHandler | undefined = undefined
1212
loggerDisabled = false
13+
redactedUrlParamNames: ReadonlyArray<string | RegExp> = []
14+
redactedHeaderNames: ReadonlyArray<string | RegExp> = []
1315
}
1416

1517
/**
@@ -31,3 +33,21 @@ export const appendPreResponseHandlerUnsafe = (context: Context.Context<never>,
3133
(request, response) => Effect.flatMap(prev(request, response), (response) => handler(request, response))
3234
: handler
3335
}
36+
37+
/**
38+
* Adds names to redact in the server span of the current request. The span
39+
* is recorded after the handler restores its context, so scoped updates of
40+
* `CurrentRedactedNames` do not reach it.
41+
*
42+
* @internal
43+
*/
44+
export const addRedactedNamesUnsafe = (
45+
context: Context.Context<never>,
46+
urlParamNames: ReadonlyArray<string | RegExp>,
47+
headerNames: ReadonlyArray<string | RegExp>
48+
): void => {
49+
const state = Context.getOrUndefined(context, CurrentRequestState)
50+
if (state === undefined) return
51+
if (urlParamNames.length > 0) state.redactedUrlParamNames = [...state.redactedUrlParamNames, ...urlParamNames]
52+
if (headerNames.length > 0) state.redactedHeaderNames = [...state.redactedHeaderNames, ...headerNames]
53+
}

‎packages/effect/test/httpapi/HttpApiBuilder.test.ts‎

Lines changed: 77 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { assert, describe, it, vi } from "@effect/vitest"
22
import {
33
Cause,
4+
Context,
45
DateTime,
56
Effect,
67
type Exit,
@@ -11,14 +12,16 @@ import {
1112
Redacted,
1213
Schema,
1314
SchemaTransformation,
14-
Stream
15+
Stream,
16+
Tracer
1517
} from "effect"
1618
import {
1719
Etag,
1820
HttpClient,
1921
HttpClientRequest,
2022
HttpClientResponse,
2123
HttpEffect,
24+
HttpMiddleware,
2225
HttpPlatform,
2326
HttpRouter,
2427
HttpServerRequest,
@@ -2083,6 +2086,79 @@ it.layer(TestServices)("HttpApiBuilder middleware error responses", (it) => {
20832086
}))
20842087
})
20852088

2089+
describe("HttpApiBuilder RedactedNames", () => {
2090+
it("redacts declared query and header names in server spans", async () => {
2091+
const Api = HttpApi.make("Api").add(
2092+
HttpApiGroup.make("images").add(
2093+
HttpApiEndpoint.post("build", "/build", {
2094+
query: { remote: Schema.String, t: Schema.String },
2095+
headers: { "x-registry-config": Schema.String },
2096+
success: Schema.String
2097+
}).annotate(HttpApi.RedactedNames, {
2098+
query: ["remote"],
2099+
headers: ["x-registry-config"]
2100+
})
2101+
)
2102+
)
2103+
let received: unknown
2104+
const GroupLayer = HttpApiBuilder.group(
2105+
Api,
2106+
"images",
2107+
(handlers) =>
2108+
handlers.handle("build", ({ headers, query }) =>
2109+
Effect.sync(() => {
2110+
received = { remote: query.remote, t: query.t, config: headers["x-registry-config"] }
2111+
return "ok"
2112+
}))
2113+
)
2114+
const spans: Array<Tracer.NativeSpan> = []
2115+
const tracer = Tracer.make({
2116+
span(options) {
2117+
const span = new Tracer.NativeSpan(options)
2118+
spans.push(span)
2119+
return span
2120+
}
2121+
})
2122+
const { dispose, handler } = HttpRouter.toWebHandler(
2123+
HttpApiBuilder.layer(Api).pipe(Layer.provide(GroupLayer), Layer.provide(TestServices)),
2124+
{ disableLogger: true }
2125+
)
2126+
const context = Context.make(Tracer.Tracer, tracer).pipe(Context.add(HttpMiddleware.TracerHeaderFilter, () => true))
2127+
const statuses: Array<number> = []
2128+
try {
2129+
for (
2130+
const url of [
2131+
"/build?remote=https%3A%2F%2Fuser%3Apass%40host%2Frepo.git&t=app",
2132+
// Fails query decoding, as `t` is missing: names apply before decoding.
2133+
"/build?remote=https%3A%2F%2Fuser%3Apass%40host%2Frepo.git"
2134+
]
2135+
) {
2136+
const response = await handler(
2137+
new Request(`http://localhost${url}`, {
2138+
method: "POST",
2139+
headers: { "x-registry-config": "cfg-s3cret", "x-page": "1" }
2140+
}),
2141+
context
2142+
)
2143+
statuses.push(response.status)
2144+
}
2145+
} finally {
2146+
await dispose()
2147+
}
2148+
2149+
assert.deepStrictEqual(statuses, [200, 400])
2150+
const [success, invalid] = spans
2151+
assert.strictEqual(success.attributes.get("url.full"), "http://localhost/build?remote=REDACTED&t=app")
2152+
assert.strictEqual(success.attributes.get("url.query"), "remote=REDACTED&t=app")
2153+
assert.strictEqual(success.attributes.get("http.request.header.x-registry-config"), "<redacted>")
2154+
assert.strictEqual(success.attributes.get("http.request.header.x-page"), "1")
2155+
// The handler sees the values unchanged.
2156+
assert.deepStrictEqual(received, { remote: "https://user:pass@host/repo.git", t: "app", config: "cfg-s3cret" })
2157+
assert.strictEqual(invalid.attributes.get("url.full"), "http://localhost/build?remote=REDACTED")
2158+
assert.strictEqual(invalid.attributes.get("http.request.header.x-registry-config"), "<redacted>")
2159+
})
2160+
})
2161+
20862162
describe("HttpApiBuilder streaming payloads", () => {
20872163
const Item = Schema.Struct({ n: Schema.Number })
20882164
const encoder = new TextEncoder()

0 commit comments

Comments
 (0)