Skip to content

Commit c0a1534

Browse files
tim-smartclaude
andauthored
Add HttpServer response compression middleware (#6898)
Co-authored-by: Claude Fable 5 <[email protected]>
1 parent e02fbb6 commit c0a1534

18 files changed

Lines changed: 1643 additions & 12 deletions

File tree

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
---
2+
"effect": patch
3+
"@effect/platform-node": patch
4+
"@effect/platform-bun": patch
5+
"@effect/platform-deno": patch
6+
---
7+
8+
Add HTTP response compression support. Node.js, Bun, and Deno use asynchronous
9+
`node:zlib` one-shot compression for byte-array bodies, preserving an exact
10+
`Content-Length`; stream and raw bodies remain streaming transforms.

‎.github/workflows/check.yml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,21 @@ jobs:
148148
if: matrix.runtime == 'Deno'
149149
run: deno task test --shard ${{ matrix.shard }}
150150

151+
test-bun:
152+
name: Test on Bun
153+
runs-on: ubuntu-latest
154+
permissions:
155+
contents: read
156+
timeout-minutes: 10
157+
steps:
158+
- uses: actions/checkout@v6
159+
- name: Install dependencies
160+
uses: ./.github/actions/setup
161+
with:
162+
bun-version: 1.3.13
163+
- name: Test
164+
run: bun node_modules/vitest/vitest.mjs run --project @effect/platform-bun
165+
151166
doctest:
152167
name: Documentation Tests
153168
runs-on: ubuntu-latest

‎packages/effect/src/unstable/http/HttpMiddleware.ts‎

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,15 @@ import type { ReadonlyRecord } from "../../Record.ts"
2323
import { TracerEnabled } from "../../References.ts"
2424
import { ParentSpan } from "../../Tracer.ts"
2525
import * as Headers from "./Headers.ts"
26+
import type { CompressionAlgorithm } from "./HttpPlatform.ts"
27+
import { HttpPlatform } from "./HttpPlatform.ts"
2628
import { causeResponseStripped } from "./HttpServerError.ts"
2729
import { HttpServerRequest } from "./HttpServerRequest.ts"
2830
import * as Request from "./HttpServerRequest.ts"
2931
import * as Response from "./HttpServerResponse.ts"
3032
import type { HttpServerResponse } from "./HttpServerResponse.ts"
3133
import * as TraceContext from "./HttpTraceContext.ts"
34+
import * as compressionInternal from "./internal/compression.ts"
3235
import { appendPreResponseHandlerUnsafe } from "./internal/preResponseHandler.ts"
3336

3437
/**
@@ -403,3 +406,161 @@ export const cors = (options?: {
403406
return httpApp
404407
})
405408
}
409+
410+
/**
411+
* Middleware that compresses HTTP response bodies based on the request's
412+
* `Accept-Encoding` header.
413+
*
414+
* **Details**
415+
*
416+
* Content negotiation follows RFC 9110: the first algorithm in server
417+
* preference order that the client accepts with a positive q-value and the
418+
* platform supports is used. When no algorithm is acceptable the response is
419+
* sent uncompressed.
420+
*
421+
* The body transform is performed by the `HttpPlatform` service.
422+
*
423+
* Responses are skipped when the status is 1xx, 204, 206, or 304, when a
424+
* `Content-Encoding` is already present, when `Cache-Control: no-transform`
425+
* is set, when the content type is absent or not compressible, when a known
426+
* body length is below `minSize`, or when the body is empty or `FormData`. A
427+
* response carrying `Content-Encoding: identity` opts out of compression and
428+
* has the header stripped before sending.
429+
*
430+
* `Vary: Accept-Encoding` is set on every response that was eligible by
431+
* status and content type, including ones skipped by negotiation or
432+
* `minSize`.
433+
*
434+
* Do not combine this middleware with Deno's automatic response compression.
435+
* On edge runtimes that already apply automatic compression, the middleware
436+
* is unnecessary. Platforms backed by Web `CompressionStream` also cannot
437+
* explicitly flush each input chunk, so incremental delivery depends on the
438+
* runtime's implementation.
439+
*
440+
* **Security**
441+
*
442+
* Compression can expose secrets through BREACH-style attacks when one
443+
* response contains both secret data and attacker-controlled input and an
444+
* attacker can observe the compressed response length. For affected routes,
445+
* disable compression with `Content-Encoding: identity` or
446+
* `Cache-Control: no-transform`, or use `compressible` to restrict which
447+
* response content types can be compressed.
448+
*
449+
* @category compression
450+
* @since 4.0.0
451+
*/
452+
export const compression = (
453+
options?: {
454+
/**
455+
* Server preference order. Negotiation picks the first accepted algorithm
456+
* supported by the platform. Defaults to `["br", "gzip", "deflate"]`;
457+
* `zstd` must be explicitly opted into.
458+
*/
459+
readonly algorithms?: ReadonlyArray<CompressionAlgorithm> | undefined
460+
/**
461+
* Minimum body size in bytes when the length is known. Unknown-length bodies
462+
* are always compressed. Defaults to `1024`.
463+
*/
464+
readonly minSize?: number | undefined
465+
/** Replaces the default content-type predicate. */
466+
readonly compressible?: ((contentType: string) => boolean) | undefined
467+
/** Per-algorithm levels. Platforms without a level knob ignore them. */
468+
readonly levels?: {
469+
readonly gzip?: number | undefined
470+
readonly deflate?: number | undefined
471+
readonly br?: number | undefined
472+
readonly zstd?: number | undefined
473+
} | undefined
474+
} | undefined
475+
): <E, R>(
476+
httpApp: Effect.Effect<HttpServerResponse, E, R>
477+
) => Effect.Effect<HttpServerResponse, E, R | HttpServerRequest | HttpPlatform> => {
478+
const preferred = options?.algorithms ?? defaultAlgorithms
479+
const minSize = options?.minSize ?? 1024
480+
const compressible = options?.compressible ?? compressionInternal.defaultCompressible
481+
const levels = { ...defaultLevels, ...options?.levels }
482+
const levelOptions: Record<CompressionAlgorithm, { readonly level: number | undefined }> = {
483+
gzip: { level: levels.gzip },
484+
deflate: { level: levels.deflate },
485+
br: { level: levels.br },
486+
zstd: { level: levels.zstd }
487+
}
488+
const transform = (
489+
compression: HttpPlatform["Service"]["compression"],
490+
acceptEncoding: string | undefined,
491+
response: HttpServerResponse
492+
): Effect.Effect<HttpServerResponse> => {
493+
if (
494+
response.status < 200 ||
495+
response.status === 204 ||
496+
response.status === 206 ||
497+
response.status === 304
498+
) {
499+
return Effect.succeed(response)
500+
}
501+
const currentEncoding = response.headers["content-encoding"]
502+
if (currentEncoding !== undefined) {
503+
return Effect.succeed(
504+
currentEncoding.trim().toLowerCase() === "identity"
505+
? Response.removeHeader(response, "content-encoding")
506+
: response
507+
)
508+
}
509+
const body = response.body
510+
if (body._tag === "Empty" || body._tag === "FormData") {
511+
return Effect.succeed(response)
512+
}
513+
const cacheControl = response.headers["cache-control"]
514+
if (cacheControl !== undefined && noTransformRegex.test(cacheControl)) {
515+
return Effect.succeed(response)
516+
}
517+
const contentType = response.headers["content-type"] ?? body.contentType
518+
if (contentType === undefined || !compressible(contentType)) {
519+
return Effect.succeed(response)
520+
}
521+
const algorithm = compressionInternal.negotiate(acceptEncoding, preferred, compression.algorithms)
522+
if (algorithm === undefined) {
523+
return Effect.succeed(withVary(response))
524+
}
525+
const contentLength = body.contentLength ?? contentLengthHeader(response.headers)
526+
if (contentLength !== undefined && contentLength < minSize) {
527+
return Effect.succeed(withVary(response))
528+
}
529+
return compression.compressResponse(response, algorithm, levelOptions[algorithm])
530+
}
531+
return <E, R>(
532+
httpApp: Effect.Effect<HttpServerResponse, E, R>
533+
): Effect.Effect<HttpServerResponse, E, R | HttpServerRequest | HttpPlatform> =>
534+
Effect.withFiber((fiber) => {
535+
const request = Context.getUnsafe(fiber.context, HttpServerRequest)
536+
const compression = Context.getUnsafe(fiber.context, HttpPlatform).compression
537+
appendPreResponseHandlerUnsafe(request, (request, response) =>
538+
transform(compression, request.headers["accept-encoding"], response))
539+
return httpApp
540+
})
541+
}
542+
543+
const withVary = (response: HttpServerResponse): HttpServerResponse => {
544+
const vary = compressionInternal.varyAcceptEncoding(response.headers)
545+
return vary === undefined ? response : Response.setHeader(response, "vary", vary)
546+
}
547+
548+
const defaultAlgorithms: ReadonlyArray<CompressionAlgorithm> = ["br", "gzip", "deflate"]
549+
550+
const defaultLevels = {
551+
gzip: 6,
552+
deflate: 6,
553+
br: 4,
554+
zstd: 3
555+
} as const
556+
557+
const noTransformRegex = /(?:^|[\s,])no-transform(?:$|[\s,;])/i
558+
559+
const contentLengthHeader = (headers: Headers.Headers): number | undefined => {
560+
const value = headers["content-length"]
561+
if (value === undefined) {
562+
return undefined
563+
}
564+
const parsed = Number(value)
565+
return Number.isFinite(parsed) && parsed >= 0 ? parsed : undefined
566+
}

‎packages/effect/src/unstable/http/HttpPlatform.ts‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import * as Etag from "./Etag.ts"
2121
import * as Headers from "./Headers.ts"
2222
import type * as Body from "./HttpBody.ts"
2323
import * as Response from "./HttpServerResponse.ts"
24+
import * as internal from "./internal/compression.ts"
2425

2526
/**
2627
* Service for platform-specific HTTP response helpers, including file-backed server responses.
@@ -30,6 +31,7 @@ import * as Response from "./HttpServerResponse.ts"
3031
*/
3132
export class HttpPlatform extends Context.Service<HttpPlatform, {
3233
readonly platform: "deno" | "node" | "bun" | "web"
34+
readonly compression: Compression
3335
readonly fileResponse: (
3436
path: string,
3537
options?: Response.Options.WithContent & {
@@ -56,6 +58,7 @@ export class HttpPlatform extends Context.Service<HttpPlatform, {
5658
*/
5759
export const make: (impl: {
5860
readonly platform: "deno" | "node" | "bun" | "web"
61+
readonly compression: Compression
5962
readonly fileResponse: (
6063
path: string,
6164
status: number,
@@ -86,6 +89,7 @@ export const make: (impl: {
8689

8790
return HttpPlatform.of({
8891
platform: impl.platform,
92+
compression: internal.wrapCompression(impl.compression),
8993
fileResponse: Effect.fnUntraced(function*(path, options) {
9094
const info = yield* fs.stat(path)
9195
const etag = yield* etagGen.fromFileInfo(info)
@@ -147,6 +151,7 @@ export const layer = Layer.effect(HttpPlatform)(
147151
Effect.flatMap(FileSystem.FileSystem, (fs) =>
148152
make({
149153
platform: "web",
154+
compression: internal.compressionWeb,
150155
fileResponse(path, status, statusText, headers, start, end, contentLength) {
151156
return Response.stream(
152157
fs.stream(path, {
@@ -167,3 +172,89 @@ export const layer = Layer.effect(HttpPlatform)(
167172
}
168173
}))
169174
).pipe(Layer.provide(Etag.layerWeak))
175+
176+
/**
177+
* Content codings that HTTP response compression can apply.
178+
*
179+
* @category compression
180+
* @since 4.0.0
181+
*/
182+
export type CompressionAlgorithm = "gzip" | "deflate" | "br" | "zstd"
183+
184+
/**
185+
* Options passed to a platform when compressing a response body.
186+
*
187+
* **Details**
188+
*
189+
* The `level` scale depends on the algorithm. Platforms without a level knob,
190+
* such as the Web `CompressionStream` implementation, ignore it.
191+
*
192+
* @category compression
193+
* @since 4.0.0
194+
*/
195+
export interface CompressionOptions {
196+
readonly level?: number | undefined
197+
}
198+
199+
/**
200+
* Platform primitive for HTTP response compression.
201+
*
202+
* **Details**
203+
*
204+
* `algorithms` advertises what the platform can encode; content negotiation
205+
* happens in the shared `HttpMiddleware.compression` middleware.
206+
*
207+
* `compressResponse` is only called when compression is definitely happening —
208+
* all skip logic runs in the shared middleware first. The platform owns the
209+
* body transform and removes `Content-Length` when the compressed size is not
210+
* known in advance. The `make` wrapper owns the `Content-Encoding` and `Vary`
211+
* headers.
212+
*
213+
* @category compression
214+
* @since 4.0.0
215+
*/
216+
export interface Compression {
217+
readonly algorithms: ReadonlySet<CompressionAlgorithm>
218+
readonly compressResponse: (
219+
response: Response.HttpServerResponse,
220+
algorithm: CompressionAlgorithm,
221+
options?: CompressionOptions | undefined
222+
) => Effect.Effect<Response.HttpServerResponse>
223+
}
224+
225+
/**
226+
* Creates a compression body transform backed by the Web `CompressionStream`
227+
* API, for use with `makeCompressionWeb`.
228+
*
229+
* **Details**
230+
*
231+
* The format string is passed through to the runtime, so runtime-specific
232+
* formats such as Bun's `"brotli"` and `"zstd"` are usable. `CompressionStream`
233+
* has no compression level knob, so `CompressionOptions.level` does not apply.
234+
*
235+
* @category compression
236+
* @since 4.0.0
237+
*/
238+
export const compressionTransformWeb: (
239+
format: string
240+
) => (stream: ReadableStream<Uint8Array>) => ReadableStream<Uint8Array> = internal.compressionTransformWeb
241+
242+
/**
243+
* Creates a `Compression` implementation from Web `ReadableStream`
244+
* transforms.
245+
*
246+
* **Details**
247+
*
248+
* All supported bodies are transformed as streams. The `Content-Length`
249+
* header is dropped in every case.
250+
*
251+
* @category compression
252+
* @since 4.0.0
253+
*/
254+
export const makeCompressionWeb: (options: {
255+
readonly algorithms: Iterable<CompressionAlgorithm>
256+
readonly transform: (
257+
algorithm: CompressionAlgorithm,
258+
options?: CompressionOptions | undefined
259+
) => (stream: ReadableStream<Uint8Array>) => ReadableStream<Uint8Array>
260+
}) => Compression = internal.makeCompressionWeb

‎packages/effect/src/unstable/http/HttpServerResponse.ts‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -536,6 +536,24 @@ export const setHeader: {
536536
})
537537
)
538538

539+
/**
540+
* Returns a response with the specified header removed.
541+
*
542+
* @category combinators
543+
* @since 4.0.0
544+
*/
545+
export const removeHeader: {
546+
(key: string): (self: HttpServerResponse) => HttpServerResponse
547+
(self: HttpServerResponse, key: string): HttpServerResponse
548+
} = dual(
549+
2,
550+
(self: HttpServerResponse, key: string): HttpServerResponse =>
551+
makeResponse({
552+
...self,
553+
headers: Headers.remove(self.headers, key)
554+
})
555+
)
556+
539557
/**
540558
* Returns a response with all supplied headers set on the existing header map.
541559
*

0 commit comments

Comments
 (0)