Bug Description
client.request throws an error when body.json() & body.text() are called with setEncoding.
Reproducible By
import { createServer } from 'http'
import { Client } from 'undici'
import { once } from 'events'
import assert from 'assert'
const data = 'a'.repeat(100)
const server = createServer((request, response) => {
response.end(JSON.stringify({ data }))
}).listen()
await once(server, 'listening')
const client = new Client(`http://localhost:${server.address().port}`)
try {
const { body, headers, statusCode } = await client.request({
path: '/',
method: 'GET'
})
console.log(`response received ${statusCode}`)
console.log('headers', headers)
body.setEncoding('utf8')
const json = await body.json()
assert.strictEqual(json.data, data, 'multi byte')
} catch (error) {
console.error('error!!:', error)
} finally {
client.close()
server.close()
}$ node index.mjs
error!!: TypeError [ERR_INVALID_ARG_TYPE]: The "list[0]" argument must be an instance of Buffer or Uint8Array. Received type string ('{"data":"aaaaaaaaaaaaaaa...)
at new NodeError (node:internal/errors:371:5)
at Function.concat (node:buffer:559:13)
at consumeEnd (/Users/kohtaito/dev/tmp/test_undici/node_modules/undici/lib/api/readable.js:238:33)
at BodyReadable.<anonymous> (/Users/kohtaito/dev/tmp/test_undici/node_modules/undici/lib/api/readable.js:220:7)
at BodyReadable.emit (node:events:390:28)
at BodyReadable.emit (/Users/kohtaito/dev/tmp/test_undici/node_modules/undici/lib/api/readable.js:66:18)
at endReadableNT (node:internal/streams/readable:1343:12)
at processTicksAndRejections (node:internal/process/task_queues:83:21) {
code: 'ERR_INVALID_ARG_TYPE'
}Expected Behavior
Logs & Screenshots
Environment
Node.js v16.13.1
undici v4.11.0
Additional context
chunk is converted to a string by StringDecoder only when consumeStart is executed.
$ git diff
diff --git a/lib/api/readable.js b/lib/api/readable.js
index 3bb454e..8ea758e 100644
--- a/lib/api/readable.js
+++ b/lib/api/readable.js
@@ -94,6 +94,7 @@ module.exports = class BodyReadable extends Readable {
push (chunk) {
if (this[kConsume] && chunk !== null && !this[kReading]) {
+ console.log('BodyReadable: push', typeof chunk, this.readableEncoding)
consumePush(this[kConsume], chunk)
return true
} else {
@@ -196,6 +197,7 @@ function consumeStart (consume) {
const { _readableState: state } = consume.stream
for (const chunk of state.buffer) {
+ console.log('consumeStart', typeof chunk, consume.stream.readableEncoding)
consumePush(consume, chunk)
}$ node index.mjs
response received 200
headers {
date: 'Mon, 06 Dec 2021 09:08:48 GMT',
connection: 'keep-alive',
'keep-alive': 'timeout=5',
'content-length': '10000011'
}
consumeStart string utf8
BodyReadable: push object utf8
BodyReadable: push object utf8
BodyReadable: push object utf8
BodyReadable: push object utf8
BodyReadable: push object utf8I think that this code needs to use consume.stream._readableState.decoder.end() in consumeEnd instead of Buffer.concat(body) if the value is in readableEncoding.
Is this the right way to fix it?
I also considered a method to overwrite StringDecoder so that it is not created in setEncoding.
However, it seems that it will be difficult to upgrade because it depends too much on the structure of Stream.
Ref: #1119
CC: @mcollina
Bug Description
client.requestthrows an error whenbody.json()&body.text()are called withsetEncoding.Reproducible By
Expected Behavior
Logs & Screenshots
Environment
Node.js v16.13.1
undici v4.11.0
Additional context
chunkis converted to a string by StringDecoder only when consumeStart is executed.$ node index.mjs response received 200 headers { date: 'Mon, 06 Dec 2021 09:08:48 GMT', connection: 'keep-alive', 'keep-alive': 'timeout=5', 'content-length': '10000011' } consumeStart string utf8 BodyReadable: push object utf8 BodyReadable: push object utf8 BodyReadable: push object utf8 BodyReadable: push object utf8 BodyReadable: push object utf8I think that this code needs to use
consume.stream._readableState.decoder.end()inconsumeEndinstead ofBuffer.concat(body)if the value is inreadableEncoding.Is this the right way to fix it?
I also considered a method to overwrite StringDecoder so that it is not created in setEncoding.
However, it seems that it will be difficult to upgrade because it depends too much on the structure of Stream.
Ref: #1119
CC: @mcollina