Skip to content

Commit be7b802

Browse files
Vadim Demedessindresorhus
Vadim Demedes
authored and
sindresorhus
committed
Add progress events (#322)
1 parent a4eb37b commit be7b802

4 files changed

Lines changed: 389 additions & 8 deletions

File tree

‎index.js‎

Lines changed: 165 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,27 +3,64 @@ const EventEmitter = require('events');
33
const http = require('http');
44
const https = require('https');
55
const PassThrough = require('stream').PassThrough;
6+
const Transform = require('stream').Transform;
67
const urlLib = require('url');
8+
const fs = require('fs');
79
const querystring = require('querystring');
810
const duplexer3 = require('duplexer3');
11+
const intoStream = require('into-stream');
912
const isStream = require('is-stream');
1013
const getStream = require('get-stream');
1114
const timedOut = require('timed-out');
1215
const urlParseLax = require('url-parse-lax');
1316
const urlToOptions = require('url-to-options');
1417
const lowercaseKeys = require('lowercase-keys');
1518
const decompressResponse = require('decompress-response');
19+
const mimicResponse = require('mimic-response');
1620
const isRetryAllowed = require('is-retry-allowed');
1721
const Buffer = require('safe-buffer').Buffer;
1822
const isURL = require('isurl');
1923
const isPlainObj = require('is-plain-obj');
2024
const PCancelable = require('p-cancelable');
2125
const pTimeout = require('p-timeout');
26+
const pify = require('pify');
2227
const pkg = require('./package');
2328

2429
const getMethodRedirectCodes = new Set([300, 301, 302, 303, 304, 305, 307, 308]);
2530
const allMethodRedirectCodes = new Set([300, 303, 307, 308]);
2631

32+
const isFormData = body => isStream(body) && typeof body.getBoundary === 'function';
33+
34+
const getBodySize = opts => {
35+
const body = opts.body;
36+
37+
if (opts.headers['content-length']) {
38+
return Number(opts.headers['content-length']);
39+
}
40+
41+
if (!body && !opts.stream) {
42+
return 0;
43+
}
44+
45+
if (typeof body === 'string') {
46+
return Buffer.byteLength(body);
47+
}
48+
49+
if (isFormData(body)) {
50+
return pify(body.getLength.bind(body))();
51+
}
52+
53+
if (body instanceof fs.ReadStream) {
54+
return pify(fs.stat)(body.path).then(stat => stat.size);
55+
}
56+
57+
if (isStream(body) && Buffer.isBuffer(body._buffer)) {
58+
return body._buffer.length;
59+
}
60+
61+
return null;
62+
};
63+
2764
function requestAsEventEmitter(opts) {
2865
opts = opts || {};
2966

@@ -32,6 +69,8 @@ function requestAsEventEmitter(opts) {
3269
const redirects = [];
3370
let retryCount = 0;
3471
let redirectUrl;
72+
let uploadBodySize;
73+
let uploaded = 0;
3574

3675
const get = opts => {
3776
if (opts.protocol !== 'http:' && opts.protocol !== 'https:') {
@@ -46,7 +85,17 @@ function requestAsEventEmitter(opts) {
4685
fn = electron.net || electron.remote.net;
4786
}
4887

88+
let progressInterval;
89+
4990
const req = fn.request(opts, res => {
91+
clearInterval(progressInterval);
92+
93+
ee.emit('uploadProgress', {
94+
percent: 1,
95+
transferred: uploaded,
96+
total: uploadBodySize
97+
});
98+
5099
const statusCode = res.statusCode;
51100

52101
res.url = redirectUrl || requestUrl;
@@ -85,22 +134,65 @@ function requestAsEventEmitter(opts) {
85134
return;
86135
}
87136

137+
const downloadBodySize = Number(res.headers['content-length']) || null;
138+
let downloaded = 0;
139+
88140
setImmediate(() => {
141+
const progressStream = new Transform({
142+
transform(chunk, encoding, callback) {
143+
downloaded += chunk.length;
144+
145+
const percent = downloadBodySize ? downloaded / downloadBodySize : 0;
146+
147+
// Let flush() be responsible for emitting the last event
148+
if (percent < 1) {
149+
ee.emit('downloadProgress', {
150+
percent,
151+
transferred: downloaded,
152+
total: downloadBodySize
153+
});
154+
}
155+
156+
callback(null, chunk);
157+
},
158+
159+
flush(callback) {
160+
ee.emit('downloadProgress', {
161+
percent: 1,
162+
transferred: downloaded,
163+
total: downloadBodySize
164+
});
165+
166+
callback();
167+
}
168+
});
169+
170+
mimicResponse(res, progressStream);
171+
progressStream.redirectUrls = redirects;
172+
89173
const response = opts.decompress === true &&
90174
typeof decompressResponse === 'function' &&
91-
req.method !== 'HEAD' ? decompressResponse(res) : res;
175+
req.method !== 'HEAD' ? decompressResponse(progressStream) : progressStream;
92176

93177
if (!opts.decompress && ['gzip', 'deflate'].indexOf(res.headers['content-encoding']) !== -1) {
94178
opts.encoding = null;
95179
}
96180

97-
response.redirectUrls = redirects;
98-
99181
ee.emit('response', response);
182+
183+
ee.emit('downloadProgress', {
184+
percent: 0,
185+
transferred: 0,
186+
total: downloadBodySize
187+
});
188+
189+
res.pipe(progressStream);
100190
});
101191
});
102192

103193
req.once('error', err => {
194+
clearInterval(progressInterval);
195+
104196
const backoff = opts.retries(++retryCount, err);
105197

106198
if (backoff) {
@@ -111,7 +203,44 @@ function requestAsEventEmitter(opts) {
111203
ee.emit('error', new got.RequestError(err, opts));
112204
});
113205

206+
ee.on('request', req => {
207+
ee.emit('uploadProgress', {
208+
percent: 0,
209+
transferred: 0,
210+
total: uploadBodySize
211+
});
212+
213+
req.connection.on('connect', () => {
214+
const uploadEventFrequency = 150;
215+
216+
progressInterval = setInterval(() => {
217+
const lastUploaded = uploaded;
218+
const headersSize = Buffer.byteLength(req._header);
219+
uploaded = req.connection.bytesWritten - headersSize;
220+
221+
// Prevent the known issue of `bytesWritten` being larger than body size
222+
if (uploadBodySize && uploaded > uploadBodySize) {
223+
uploaded = uploadBodySize;
224+
}
225+
226+
// Don't emit events with unchanged progress and
227+
// prevent last event from being emitted, because
228+
// it's emitted when `response` is emitted
229+
if (uploaded === lastUploaded || uploaded === uploadBodySize) {
230+
return;
231+
}
232+
233+
ee.emit('uploadProgress', {
234+
percent: uploadBodySize ? uploaded / uploadBodySize : 0,
235+
transferred: uploaded,
236+
total: uploadBodySize
237+
});
238+
}, uploadEventFrequency);
239+
});
240+
});
241+
114242
if (opts.gotTimeout) {
243+
clearInterval(progressInterval);
115244
timedOut(req, opts.gotTimeout);
116245
}
117246

@@ -121,8 +250,16 @@ function requestAsEventEmitter(opts) {
121250
};
122251

123252
setImmediate(() => {
124-
get(opts);
253+
Promise.resolve(getBodySize(opts))
254+
.then(size => {
255+
uploadBodySize = size;
256+
get(opts);
257+
})
258+
.catch(err => {
259+
ee.emit('error', err);
260+
});
125261
});
262+
126263
return ee;
127264
}
128265

@@ -131,7 +268,9 @@ function asPromise(opts) {
131268
pTimeout(requestPromise, opts.gotTimeout.request, new got.RequestError({message: 'Request timed out', code: 'ETIMEDOUT'}, opts)) :
132269
requestPromise;
133270

134-
return timeoutFn(new PCancelable((onCancel, resolve, reject) => {
271+
const proxy = new EventEmitter();
272+
273+
const promise = timeoutFn(new PCancelable((onCancel, resolve, reject) => {
135274
const ee = requestAsEventEmitter(opts);
136275
let cancelOnRequest = false;
137276

@@ -191,10 +330,21 @@ function asPromise(opts) {
191330
});
192331

193332
ee.on('error', reject);
333+
ee.on('uploadProgress', proxy.emit.bind(proxy, 'uploadProgress'));
334+
ee.on('downloadProgress', proxy.emit.bind(proxy, 'downloadProgress'));
194335
}));
336+
337+
promise.on = (name, fn) => {
338+
proxy.on(name, fn);
339+
return promise;
340+
};
341+
342+
return promise;
195343
}
196344

197345
function asStream(opts) {
346+
opts.stream = true;
347+
198348
const input = new PassThrough();
199349
const output = new PassThrough();
200350
const proxy = duplexer3(input, output);
@@ -256,6 +406,8 @@ function asStream(opts) {
256406

257407
ee.on('redirect', proxy.emit.bind(proxy, 'redirect'));
258408
ee.on('error', proxy.emit.bind(proxy, 'error'));
409+
ee.on('uploadProgress', proxy.emit.bind(proxy, 'uploadProgress'));
410+
ee.on('downloadProgress', proxy.emit.bind(proxy, 'downloadProgress'));
259411

260412
return proxy;
261413
}
@@ -320,7 +472,7 @@ function normalizeArguments(url, opts) {
320472
throw new TypeError('options.body must be a plain Object or Array when options.form or options.json is used');
321473
}
322474

323-
if (isStream(body) && typeof body.getBoundary === 'function') {
475+
if (isFormData(body)) {
324476
// Special case for https://github.com/form-data/form-data
325477
headers['content-type'] = headers['content-type'] || `multipart/form-data; boundary=${body.getBoundary()}`;
326478
} else if (opts.form && canBodyBeStringified) {
@@ -336,6 +488,13 @@ function normalizeArguments(url, opts) {
336488
headers['content-length'] = length;
337489
}
338490

491+
// Convert buffer to stream to receive upload progress events
492+
// see https://github.com/sindresorhus/got/pull/322
493+
if (Buffer.isBuffer(body)) {
494+
opts.body = intoStream(body);
495+
opts.body._buffer = body;
496+
}
497+
339498
opts.method = (opts.method || 'POST').toUpperCase();
340499
} else {
341500
opts.method = (opts.method || 'GET').toUpperCase();

‎package.json‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,13 +53,16 @@
5353
"decompress-response": "^3.2.0",
5454
"duplexer3": "^0.1.4",
5555
"get-stream": "^3.0.0",
56+
"into-stream": "^3.1.0",
5657
"is-plain-obj": "^1.1.0",
5758
"is-retry-allowed": "^1.0.0",
5859
"is-stream": "^1.0.0",
5960
"isurl": "^1.0.0-alpha5",
6061
"lowercase-keys": "^1.0.0",
62+
"mimic-response": "^1.0.0",
6163
"p-cancelable": "^0.3.0",
6264
"p-timeout": "^1.1.1",
65+
"pify": "^3.0.0",
6366
"safe-buffer": "^5.0.1",
6467
"timed-out": "^4.0.0",
6568
"url-parse-lax": "^1.0.0",
@@ -70,10 +73,9 @@
7073
"coveralls": "^2.11.4",
7174
"form-data": "^2.1.1",
7275
"get-port": "^3.0.0",
73-
"into-stream": "^3.0.0",
7476
"nyc": "^11.0.2",
7577
"pem": "^1.4.4",
76-
"pify": "^3.0.0",
78+
"slow-stream": "0.0.4",
7779
"tempfile": "^2.0.0",
7880
"tempy": "^0.1.0",
7981
"universal-url": "1.0.0-alpha",

‎readme.md‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ Created because [`request`](https://github.com/request/request) is bloated *(sev
2121
- [Request cancelation](#aborting-the-request)
2222
- [Follows redirects](#followredirect)
2323
- [Retries on network failure](#retries)
24+
- [Progress events](#onuploadprogress-progress)
2425
- [Handles gzip/deflate](#decompress)
2526
- [Timeout handling](#timeout)
2627
- [Errors with metadata](#errors)
@@ -202,6 +203,36 @@ got.stream('github.com')
202203

203204
`redirect` event to get the response object of a redirect. The second argument is options for the next request to the redirect location.
204205

206+
##### .on('uploadProgress', progress)
207+
##### .on('downloadProgress', progress)
208+
209+
Progress events for uploading (sending request) and downloading (receiving response). The `progress` argument is an object like:
210+
211+
```js
212+
{
213+
percent: 0.1,
214+
transferred: 1024,
215+
total: 10240
216+
}
217+
```
218+
219+
If it's not possible to retrieve the body size (can happen when streaming), `total` will be `null`.
220+
221+
**Note**: Progress events can also be used with promises.
222+
223+
```js
224+
got('todomvc.com')
225+
.on('downloadProgress', progress => {
226+
// Report download progress
227+
})
228+
.on('uploadProgress', progress => {
229+
// Report upload progress
230+
})
231+
.then(response => {
232+
// Done
233+
});
234+
```
235+
205236
##### .on('error', error, body, response)
206237

207238
`error` event emitted in case of protocol error (like `ENOTFOUND` etc.) or status error (4xx or 5xx). The second argument is the body of the server response in case of status error. The third argument is response object.

0 commit comments

Comments
 (0)