Skip to content

Commit 919e61f

Browse files
committed
stream: port remaining SonicBoom tests for Utf8Stream
Port the SonicBoom tests that could not be imported directly because they depend on the third-party proxyquire module, along with the maxWriteRetries option they exercise, into the Utf8Stream module. Adds maxWriteRetries support, which bounds consecutive EAGAIN/EBUSY retry attempts and resets the counter on forward progress, plus drop-event and flush-callback edge-case coverage. Fixes: nodejs#58955 Assisted-by: pi Signed-off-by: Matteo Collina <[email protected]>
1 parent ebef774 commit 919e61f

4 files changed

Lines changed: 321 additions & 3 deletions

File tree

‎lib/internal/streams/fast-utf8-stream.js‎

Lines changed: 31 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,8 @@ class Utf8Stream extends EventEmitter {
8383
#mode = 0o666;
8484
#retryEAGAIN = () => true;
8585
#mkdir = false;
86+
#maxWriteRetries = 0;
87+
#writeRetries = 0;
8688
#writingBuf = '';
8789
#write;
8890
#flush;
@@ -128,6 +130,7 @@ class Utf8Stream extends EventEmitter {
128130
append = true,
129131
mkdir,
130132
retryEAGAIN,
133+
maxWriteRetries,
131134
fsync,
132135
contentMode = kContentModeUtf8,
133136
mode,
@@ -161,12 +164,14 @@ class Utf8Stream extends EventEmitter {
161164
this.#mode = mode;
162165
this.#retryEAGAIN = retryEAGAIN || (() => true);
163166
this.#mkdir = mkdir || false;
167+
this.#maxWriteRetries = maxWriteRetries || 0;
164168

165169
validateUint32(this.#hwm, 'options.hwm');
166170
validateUint32(this.#minLength, 'options.minLength');
167171
validateUint32(this.#maxLength, 'options.maxLength');
168172
validateUint32(this.#maxWrite, 'options.maxWrite');
169173
validateUint32(this.#periodicFlush, 'options.periodicFlush');
174+
validateUint32(this.#maxWriteRetries, 'options.maxWriteRetries');
170175
validateBoolean(this.#sync, 'options.sync');
171176
validateBoolean(this.#fsync, 'options.fsync');
172177
validateBoolean(this.#append, 'options.append');
@@ -373,7 +378,13 @@ class Utf8Stream extends EventEmitter {
373378

374379
#release(err, n) {
375380
if (err) {
376-
if ((err.code === 'EAGAIN' || err.code === 'EBUSY') &&
381+
const isRetryableErr = (err.code === 'EAGAIN' || err.code === 'EBUSY');
382+
if (isRetryableErr) {
383+
this.#writeRetries++;
384+
}
385+
const retriesExhausted = this.#maxWriteRetries > 0 && this.#writeRetries > this.#maxWriteRetries;
386+
387+
if (isRetryableErr && !retriesExhausted &&
377388
this.#retryEAGAIN(err, this.#writingBuf.length, this.#len - this.#writingBuf.length)) {
378389
if (this.#sync) {
379390
// This error code should not happen in sync mode, because it is
@@ -398,6 +409,13 @@ class Utf8Stream extends EventEmitter {
398409
return;
399410
}
400411

412+
// Reset the retry counter only once real forward progress (n > 0) is
413+
// confirmed. Otherwise a stuck destination would never accumulate past
414+
// one retry and `maxWriteRetries` could never trigger.
415+
if (n > 0) {
416+
this.#writeRetries = 0;
417+
}
418+
401419
this.emit('write', n);
402420
const releasedBufObj = releaseWritingBuf(this.#writingBuf, this.#len, n);
403421
this.#len = releasedBufObj.len;
@@ -635,6 +653,7 @@ class Utf8Stream extends EventEmitter {
635653
}
636654
try {
637655
const n = this.#fs.writeSync(this.#fd, buf);
656+
this.#writeRetries = 0;
638657
buf = buf.subarray(n);
639658
this.#len = MathMax(this.#len - n, 0);
640659
if (buf.length <= 0) {
@@ -643,7 +662,11 @@ class Utf8Stream extends EventEmitter {
643662
}
644663
} catch (err) {
645664
const shouldRetry = err.code === 'EAGAIN' || err.code === 'EBUSY';
646-
if (shouldRetry && !this.#retryEAGAIN(err, buf.length, this.#len - buf.length)) {
665+
if (shouldRetry) {
666+
this.#writeRetries++;
667+
}
668+
const retriesExhausted = this.#maxWriteRetries > 0 && this.#writeRetries > this.#maxWriteRetries;
669+
if (!shouldRetry || retriesExhausted || !this.#retryEAGAIN(err, buf.length, this.#len - buf.length)) {
647670
throw err;
648671
}
649672

@@ -673,6 +696,7 @@ class Utf8Stream extends EventEmitter {
673696
}
674697
try {
675698
const n = this.#fs.writeSync(this.#fd, buf, 'utf8');
699+
this.#writeRetries = 0;
676700
const releasedBufObj = releaseWritingBuf(buf, this.#len, n);
677701
buf = releasedBufObj.writingBuf;
678702
this.#len = releasedBufObj.len;
@@ -681,7 +705,11 @@ class Utf8Stream extends EventEmitter {
681705
}
682706
} catch (err) {
683707
const shouldRetry = err.code === 'EAGAIN' || err.code === 'EBUSY';
684-
if (shouldRetry && !this.#retryEAGAIN(err, buf.length, this.#len - buf.length)) {
708+
if (shouldRetry) {
709+
this.#writeRetries++;
710+
}
711+
const retriesExhausted = this.#maxWriteRetries > 0 && this.#writeRetries > this.#maxWriteRetries;
712+
if (!shouldRetry || retriesExhausted || !this.#retryEAGAIN(err, buf.length, this.#len - buf.length)) {
685713
throw err;
686714
}
687715

‎test/parallel/test-fastutf8stream-flush.js‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,12 @@ const common = require('../common');
44
const tmpdir = require('../common/tmpdir');
55
const assert = require('node:assert');
66
const {
7+
open,
78
openSync,
89
readFile,
910
writeFileSync,
11+
write,
12+
writeSync,
1013
} = require('node:fs');
1114
const { join } = require('node:path');
1215
const { Utf8Stream } = require('node:fs');
@@ -25,6 +28,32 @@ function getTempFile() {
2528
runTests(false);
2629
runTests(true);
2730

31+
// flush cb is invoked when flushing before 'ready' while the stream is
32+
// still opening (async mode only; sync mode writes synchronously).
33+
{
34+
const dest = getTempFile();
35+
36+
const stream = new Utf8Stream({
37+
dest,
38+
minLength: 4096,
39+
sync: false,
40+
fs: {
41+
open(file, flags, mode, cb) {
42+
process.nextTick(() => {
43+
assert.ok(stream.write('hello world\n'));
44+
stream.flush(common.mustCall((e) => {
45+
assert.ifError(e);
46+
stream.destroy();
47+
}));
48+
open(file, flags, mode, cb);
49+
});
50+
},
51+
},
52+
});
53+
54+
stream.on('ready', common.mustCall(() => {}));
55+
}
56+
2857
function runTests(sync) {
2958
{
3059
const dest = getTempFile();
@@ -136,4 +165,87 @@ function runTests(sync) {
136165
stream.destroy();
137166
stream.flush(common.mustCall(assert.ok));
138167
}
168+
169+
{
170+
// flush cb is invoked with the error when the underlying write fails.
171+
const dest = getTempFile();
172+
const fd = openSync(dest, 'w');
173+
174+
const err = new Error('other');
175+
err.code = 'other';
176+
let first = true;
177+
178+
const fsOverride = {};
179+
if (sync) {
180+
fsOverride.writeSync = common.mustCallAtLeast((...args) => {
181+
if (first) {
182+
first = false;
183+
throw err;
184+
}
185+
return writeSync(...args);
186+
}, 1);
187+
} else {
188+
fsOverride.write = common.mustCallAtLeast((...args) => {
189+
const callback = args[args.length - 1];
190+
if (first) {
191+
first = false;
192+
process.nextTick(callback, err);
193+
return;
194+
}
195+
return write(...args);
196+
}, 1);
197+
}
198+
199+
const stream = new Utf8Stream({
200+
fd,
201+
sync,
202+
minLength: 4096,
203+
fs: fsOverride,
204+
});
205+
206+
stream.on('ready', common.mustCall(() => {
207+
assert.ok(stream.write('hello world\n'));
208+
stream.flush(common.mustCall((e) => {
209+
assert.strictEqual(e.code, 'other');
210+
stream.destroy();
211+
}));
212+
}));
213+
}
214+
215+
{
216+
// flush cb is invoked once the in-flight write completes.
217+
const dest = getTempFile();
218+
const fd = openSync(dest, 'w');
219+
220+
const fsOverride = {};
221+
if (sync) {
222+
fsOverride.writeSync = common.mustCallAtLeast((...args) => {
223+
stream.flush(common.mustCall((e) => {
224+
assert.ifError(e);
225+
stream.destroy();
226+
}));
227+
return writeSync(...args);
228+
}, 1);
229+
} else {
230+
fsOverride.write = common.mustCallAtLeast((...args) => {
231+
const callback = args[args.length - 1];
232+
stream.flush(common.mustCall((e) => {
233+
assert.ifError(e);
234+
stream.destroy();
235+
}));
236+
return write(...args);
237+
}, 1);
238+
}
239+
240+
const stream = new Utf8Stream({
241+
fd,
242+
sync,
243+
minLength: 1,
244+
fs: fsOverride,
245+
});
246+
247+
stream.on('ready', common.mustCall(() => {
248+
assert.ok(stream.write('hello world\n'));
249+
}));
250+
}
139251
}

‎test/parallel/test-fastutf8stream-retry.js‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,3 +211,117 @@ function runTests(sync) {
211211
}));
212212
}));
213213
}
214+
215+
{
216+
const dest = getTempFile();
217+
const fd = openSync(dest, 'w');
218+
219+
const err = new Error('EAGAIN');
220+
err.code = 'EAGAIN';
221+
let attempts = 0;
222+
223+
const stream = new Utf8Stream({
224+
fd,
225+
sync: false,
226+
minLength: 0,
227+
maxWriteRetries: 3,
228+
// retryEAGAIN always returns true ("keep going"), so maxWriteRetries must
229+
// itself cap the number of attempts instead of retrying forever.
230+
retryEAGAIN: () => true,
231+
fs: {
232+
write: common.mustCall((...args) => {
233+
attempts++;
234+
const callback = args[args.length - 1];
235+
process.nextTick(callback, err);
236+
}, 4),
237+
}
238+
});
239+
240+
stream.on('ready', common.mustCall(() => {
241+
assert.ok(stream.write('hello world\n'));
242+
}));
243+
244+
stream.once('error', common.mustCall((err) => {
245+
assert.strictEqual(err.code, 'EAGAIN');
246+
// 1 initial attempt + 3 retries = 4 total fs.write calls, then give up.
247+
assert.strictEqual(attempts, 4);
248+
assert.strictEqual(stream.writing, false);
249+
stream.destroy();
250+
}));
251+
}
252+
253+
{
254+
const dest = getTempFile();
255+
const fd = openSync(dest, 'w');
256+
257+
const err = new Error('EAGAIN');
258+
err.code = 'EAGAIN';
259+
260+
const stream = new Utf8Stream({
261+
fd,
262+
sync: true,
263+
minLength: 0,
264+
maxWriteRetries: 3,
265+
retryEAGAIN: () => true,
266+
fs: {
267+
writeSync: common.mustCall((...args) => {
268+
throw err;
269+
}, 4),
270+
}
271+
});
272+
273+
stream.on('ready', common.mustCall(() => {
274+
// Once retries are exhausted, write() must surface the error instead of
275+
// spinning forever on EAGAIN.
276+
assert.throws(() => {
277+
stream.write('hello world\n');
278+
}, (e) => e.code === 'EAGAIN');
279+
stream.destroy();
280+
}));
281+
}
282+
283+
{
284+
const dest = getTempFile();
285+
const fd = openSync(dest, 'w');
286+
287+
const err = new Error('EAGAIN');
288+
err.code = 'EAGAIN';
289+
let call = 0;
290+
291+
const stream = new Utf8Stream({
292+
fd,
293+
sync: false,
294+
minLength: 0,
295+
maxWriteRetries: 3,
296+
retryEAGAIN: () => true,
297+
fs: {
298+
write: common.mustCallAtLeast((...args) => {
299+
call++;
300+
const callback = args[args.length - 1];
301+
if (call % 3 === 0) {
302+
return write(...args);
303+
}
304+
process.nextTick(callback, err);
305+
}, 5),
306+
}
307+
});
308+
309+
stream.on('error', common.mustNotCall());
310+
311+
stream.on('ready', common.mustCall(() => {
312+
// First burst: calls 1-2 fail with EAGAIN, call 3 succeeds.
313+
assert.ok(stream.write('hello world\n'));
314+
stream.once('drain', common.mustCall(() => {
315+
// Second burst: calls 4-5 fail again. The counter must have been reset
316+
// by the successful write at call 3, so this burst succeeds too.
317+
stream.write('sonic boom\n');
318+
stream.end();
319+
}));
320+
}));
321+
322+
stream.on('finish', common.mustCall(() => {
323+
readFile(dest, 'utf8', common.mustSucceed((data) => {
324+
assert.strictEqual(data, 'hello world\nsonic boom\n');
325+
}));
326+
}));
327+
}

0 commit comments

Comments
 (0)