Skip to content

Commit d1201f0

Browse files
Release NodeSink writables on upstream failure and interruption (#9402)
Co-authored-by: Tim Smart <[email protected]>
1 parent 8e9616a commit d1201f0

6 files changed

Lines changed: 132 additions & 40 deletions

File tree

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
---
2+
"@effect/platform-node-shared": patch
3+
"@effect/platform-node": patch
4+
---
5+
6+
Release writables created by `NodeSink.fromWritable` and `fromWritableChannel` when upstream fails or the sink is interrupted, preserving the original exit cause. Writables using `endOnDone: false` remain open. Preserve stream body encoding errors in the Node HTTP client instead of reporting transport errors caused by request cleanup.

‎packages/platform/node-shared/src/NodeSink.ts‎

Lines changed: 30 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,15 +15,17 @@ import * as Cause from "effect/Cause"
1515
import * as Channel from "effect/Channel"
1616
import * as Deferred from "effect/Deferred"
1717
import * as Effect from "effect/Effect"
18-
import { identity, type LazyArg } from "effect/Function"
18+
import { constVoid, identity, type LazyArg } from "effect/Function"
1919
import * as Pull from "effect/Pull"
2020
import * as Sink from "effect/Sink"
2121
import { finished, type Writable } from "node:stream"
22+
import { destroyStream } from "./internal/stream.ts"
2223

2324
/**
2425
* Creates a `Sink` that writes chunks to a Node writable stream, respecting
25-
* backpressure, mapping writable errors with `onError`, and ending the stream
26-
* on completion unless `endOnDone` is `false`.
26+
* backpressure and mapping writable errors with `onError`. Unless `endOnDone`
27+
* is `false`, the writable is ended on completion and destroyed on upstream
28+
* failure or interruption.
2729
*
2830
* @stability unstable
2931
* @category constructors
@@ -41,8 +43,9 @@ export const fromWritable = <E, A = Uint8Array | string>(
4143

4244
/**
4345
* Creates a `Channel` that pulls chunks from upstream and writes them to a
44-
* Node writable stream, respecting backpressure and optionally ending the
45-
* writable when upstream is done.
46+
* Node writable stream, respecting backpressure. Unless `endOnDone` is
47+
* `false`, the writable is ended when upstream is done and destroyed on
48+
* upstream failure or interruption.
4649
*
4750
* @stability unstable
4851
* @category constructors
@@ -58,7 +61,28 @@ export const fromWritableChannel = <IE, E, A = Uint8Array | string>(
5861
): Channel.Channel<never, IE | E, void, NonEmptyReadonlyArray<A>, IE> =>
5962
Channel.fromTransform((pull: Pull.Pull<NonEmptyReadonlyArray<A>, IE, unknown>) => {
6063
const writable = options.evaluate() as Writable
61-
return Effect.succeed(pullIntoWritable({ ...options, writable, pull }))
64+
const write = pullIntoWritable({ ...options, writable, pull })
65+
return Effect.succeed(
66+
options.endOnDone === false
67+
? write
68+
: Effect.onError(write, (cause) => Pull.isDoneCause(cause) ? Effect.void : releaseWritable(writable))
69+
)
70+
})
71+
72+
// Destroy the writable and wait for destruction to settle, keeping any
73+
// destruction error handled.
74+
const releaseWritable = (writable: Writable): Effect.Effect<void> =>
75+
Effect.callback<void>((resume) => {
76+
const onDestroyed = () => resume(Effect.void)
77+
if (writable.closed) {
78+
// Destruction may be complete with its error still queued.
79+
const cleanup = finished(writable, { readable: false }, () => {
80+
cleanup()
81+
onDestroyed()
82+
})
83+
} else {
84+
destroyStream(writable, constVoid, onDestroyed)
85+
}
6286
})
6387

6488
/**

‎packages/platform/node-shared/src/NodeStream.ts‎

Lines changed: 1 addition & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import * as Scope from "effect/Scope"
2626
import * as Stream from "effect/Stream"
2727
import type { Duplex } from "node:stream"
2828
import { finished, Readable } from "node:stream"
29+
import { destroyStream as destroyReadable } from "./internal/stream.ts"
2930
import { pullIntoWritable } from "./NodeSink.ts"
3031

3132
/**
@@ -339,39 +340,6 @@ export const toUint8Array = <E = Cause.UnknownError>(
339340
// internal
340341
// ----------------------------------------------------------------------------
341342

342-
// Keep teardown errors handled after removing the consumer's lifecycle listeners.
343-
// Node emits the destruction error (if any) only once _destroy has completed.
344-
const destroyReadable = (readable: Readable, cleanup: () => void): void => {
345-
if (!("closed" in readable)) {
346-
cleanup()
347-
return
348-
}
349-
const onDone = () => {
350-
clearImmediate(immediate)
351-
readable.off("error", onDone)
352-
readable.off("close", onDone)
353-
}
354-
// With emitClose: false, successful destruction emits no event. Check the
355-
// public closed flag on a later turn, after any queued destruction error.
356-
// This also protects streams already closed but with an error still queued.
357-
const onClosed = () => {
358-
if (readable.closed) {
359-
onDone()
360-
} else {
361-
immediate = setImmediate(onClosed)
362-
immediate.unref()
363-
}
364-
}
365-
let immediate = setImmediate(onClosed)
366-
immediate.unref()
367-
readable.once("error", onDone)
368-
readable.once("close", onDone)
369-
cleanup()
370-
if (!readable.closed) {
371-
readable.destroy()
372-
}
373-
}
374-
375343
const readableToPullUnsafe = <A, E>(options: {
376344
readonly scope: Scope.Scope
377345
readonly exit?: MutableRef.MutableRef<Exit.Exit<never, E | Cause.Done> | undefined> | undefined
Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
import type { Readable, Writable } from "node:stream"
2+
3+
// Keep teardown errors handled after removing the consumer's lifecycle listeners.
4+
// Node emits the destruction error (if any) only once _destroy has completed.
5+
// When `onDestroyed` is given, it is called once destruction has settled and
6+
// the wait keeps the event loop alive.
7+
export const destroyStream = (
8+
stream: Readable | Writable,
9+
cleanup: () => void,
10+
onDestroyed?: () => void
11+
): void => {
12+
if (!("closed" in stream)) {
13+
cleanup()
14+
onDestroyed?.()
15+
return
16+
}
17+
const onDone = () => {
18+
clearImmediate(immediate)
19+
stream.off("error", onDone)
20+
stream.off("close", onDone)
21+
onDestroyed?.()
22+
}
23+
// With emitClose: false, successful destruction emits no event. Check the
24+
// public closed flag on a later turn, after any queued destruction error.
25+
// This also protects streams already closed but with an error still queued.
26+
const onClosed = () => {
27+
if (stream.closed) {
28+
onDone()
29+
} else {
30+
immediate = setImmediate(onClosed)
31+
if (!onDestroyed) immediate.unref()
32+
}
33+
}
34+
let immediate = setImmediate(onClosed)
35+
if (!onDestroyed) immediate.unref()
36+
stream.once("error", onDone)
37+
stream.once("close", onDone)
38+
cleanup()
39+
if (!stream.closed) {
40+
stream.destroy()
41+
}
42+
}

‎packages/platform/node-shared/test/NodeSink.test.ts‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,52 @@ import * as Tar from "tar"
1515
const TEST_TARBALL = join(__dirname, "fixtures", "helloworld.tar.gz")
1616

1717
describe("Sink", () => {
18+
it.effect.each(
19+
[
20+
{ mode: "failure", endOnDone: undefined },
21+
{ mode: "interrupt", endOnDone: undefined },
22+
{ mode: "failure", endOnDone: false },
23+
{ mode: "interrupt", endOnDone: false }
24+
] as const
25+
)("releases owned writables on $mode (endOnDone: $endOnDone)", ({ mode, endOnDone }) =>
26+
Effect.gen(function*() {
27+
const started = yield* Deferred.make<void>()
28+
const failure = new Error("upstream failed")
29+
const items: Array<string> = []
30+
const writable = new Writable({
31+
write(chunk, _encoding, callback) {
32+
items.push(chunk.toString())
33+
callback()
34+
}
35+
})
36+
yield* Effect.addFinalizer(() => Effect.sync(() => writable.destroy()))
37+
const fiber = yield* Stream.make("a").pipe(
38+
Stream.concat(Stream.fromEffect(Effect.gen(function*() {
39+
yield* Deferred.succeed(started, undefined)
40+
return yield* (mode === "failure" ? Effect.fail(failure) : Effect.never)
41+
}))),
42+
Stream.run(NodeSink.fromWritable({
43+
evaluate: () => writable,
44+
onError: (error) => error,
45+
endOnDone
46+
})),
47+
Effect.forkScoped
48+
)
49+
yield* Deferred.await(started)
50+
if (mode === "interrupt") {
51+
yield* Fiber.interrupt(fiber)
52+
}
53+
const exit = yield* Fiber.await(fiber)
54+
if (mode === "failure") {
55+
assert.deepEqual(exit, Exit.fail(failure))
56+
} else {
57+
assert.isTrue(Exit.hasInterrupts(exit))
58+
}
59+
assert.deepEqual(items, ["a"])
60+
// Assert before test-scope cleanup can release the writable for the sink.
61+
assert.strictEqual(writable.writableEnded || writable.destroyed, endOnDone !== false)
62+
}))
63+
1864
it.effect.each(["already_finished", "finished_while_pulling", "finished_while_draining", "empty"] as const)(
1965
"settles after normal writable finish: %s",
2066
(phase) =>

‎packages/platform/node/src/NodeHttpClient.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -618,6 +618,9 @@ const sendBody = (
618618
})),
619619
NodeSink.fromWritable({
620620
evaluate: () => nodeRequest,
621+
// The request race owns destruction, so a body failure must reach
622+
// it before destroying the request can produce a transport error.
623+
endOnDone: false,
621624
onError: (cause) =>
622625
new Error.HttpClientError({
623626
reason: new Error.TransportError({
@@ -626,7 +629,10 @@ const sendBody = (
626629
})
627630
})
628631
})
629-
)
632+
).pipe(Effect.andThen(() => {
633+
nodeRequest.end()
634+
return waitForFinish(nodeRequest, request)
635+
}))
630636
}
631637
}
632638
})

0 commit comments

Comments
 (0)