Skip to content
This repository was archived by the owner on Mar 3, 2026. It is now read-only.

Commit f666842

Browse files
committed
refactor: Remove pumpify dependency
1 parent 64bf0c9 commit f666842

1 file changed

Lines changed: 29 additions & 57 deletions

File tree

‎src/gcs-resumable-upload.ts‎

Lines changed: 29 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,7 @@ import {
2424
} from 'gaxios';
2525
import * as gaxios from 'gaxios';
2626
import {GoogleAuth, GoogleAuthOptions} from 'google-auth-library';
27-
import * as Pumpify from 'pumpify';
28-
import {Duplex, PassThrough, Readable} from 'stream';
29-
import * as streamEvents from 'stream-events';
27+
import {Readable, Writable} from 'stream';
3028
import retry = require('async-retry');
3129
import {RetryOptions, PreconditionOptions} from './storage';
3230

@@ -110,7 +108,7 @@ export interface UploadConfig {
110108
/**
111109
* Create a separate request per chunk.
112110
*
113-
* Should be a multiple of 256 KiB (2^18).
111+
* This value is in bytes and should be a multiple of 256 KiB (2^18).
114112
* We recommend using at least 8 MiB for the chunk size.
115113
*
116114
* @link https://cloud.google.com/storage/docs/performing-resumable-uploads#chunked-upload
@@ -226,7 +224,7 @@ export interface ApiError extends Error {
226224
errors?: GoogleInnerError[];
227225
}
228226

229-
export class Upload extends Pumpify {
227+
export class Upload extends Writable {
230228
bucket: string;
231229
file: string;
232230
apiEndpoint: string;
@@ -277,7 +275,6 @@ export class Upload extends Pumpify {
277275

278276
constructor(cfg: UploadConfig) {
279277
super();
280-
streamEvents(this);
281278

282279
cfg = cfg || {};
283280

@@ -322,12 +319,7 @@ export class Upload extends Pumpify {
322319
this.retryOptions = cfg.retryOptions;
323320

324321
if (cfg.key) {
325-
/**
326-
* NOTE: This is `as string` because there appears to be some weird kind
327-
* of TypeScript bug as 2.8. Tracking the issue here:
328-
* https://github.com/Microsoft/TypeScript/issues/23155
329-
*/
330-
const base64Key = Buffer.from(cfg.key as string).toString('base64');
322+
const base64Key = Buffer.from(cfg.key).toString('base64');
331323
this.encryption = {
332324
key: base64Key,
333325
hash: createHash('sha256').update(cfg.key).digest('base64'),
@@ -347,7 +339,7 @@ export class Upload extends Pumpify {
347339
this.uriProvidedManually = !!cfg.uri;
348340
this.uri = cfg.uri || this.get('uri');
349341
this.numBytesWritten = 0;
350-
this.numRetries = 0; //counter for number of retries currently executed
342+
this.numRetries = 0; // counter for number of retries currently executed
351343
if (!autoRetry) {
352344
cfg.retryOptions.maxRetries = 0;
353345
}
@@ -359,23 +351,7 @@ export class Upload extends Pumpify {
359351
: NaN;
360352
this.contentLength = isNaN(contentLength) ? '*' : contentLength;
361353

362-
this.upstream.on('end', () => {
363-
this.upstreamEnded = true;
364-
});
365-
366-
this.on('prefinish', () => {
367-
this.upstreamEnded = true;
368-
});
369-
370354
this.once('writing', () => {
371-
// Now that someone is writing to this object, let's attach
372-
// some duplexes. These duplexes enable this object to be
373-
// better managed in terms of 'end'/'finish' control and
374-
// buffering writes downstream if someone enables multi-
375-
// chunk upload support (`chunkSize`) w/o adding too much into
376-
// memory.
377-
this.setPipeline(this.upstream, new PassThrough());
378-
379355
if (this.uri) {
380356
this.continueUploading();
381357
} else {
@@ -391,36 +367,42 @@ export class Upload extends Pumpify {
391367
});
392368
}
393369

394-
/** A stream representing the incoming data to upload */
395-
private readonly upstream = new Duplex({
396-
read: async () => {
397-
this.once('prepareFinish', () => {
398-
// Allows this (`Upload`) to finish/end once the upload has succeeded.
399-
this.upstream.push(null);
400-
});
401-
},
402-
write: this.writeToChunkBuffer.bind(this),
403-
});
370+
/**
371+
* Prevent 'finish' event until the upload has succeeded.
372+
*
373+
* @param fireFinishEvent The finish callback
374+
*/
375+
_final(fireFinishEvent = () => {}) {
376+
this.upstreamEnded = true;
377+
378+
this.once('uploadFinished', fireFinishEvent);
379+
380+
process.nextTick(() => this.emit('upstreamFinished'));
381+
}
404382

405383
/**
406-
* A handler for `upstream` to write and buffer its data.
384+
* Handles incoming data from upstream
407385
*
408386
* @param chunk The chunk to append to the buffer
409387
* @param encoding The encoding of the chunk
410388
* @param readCallback A callback for when the buffer has been read downstream
411389
*/
412-
private writeToChunkBuffer(
390+
_write(
413391
chunk: Buffer | string,
414392
encoding: BufferEncoding,
415-
readCallback: () => void
393+
readCallback = () => {}
416394
) {
395+
// Backwards-compatible event
396+
this.emit('writing');
397+
417398
this.upstreamChunkBuffer = Buffer.concat([
418399
this.upstreamChunkBuffer,
419400
typeof chunk === 'string' ? Buffer.from(chunk, encoding) : chunk,
420401
]);
421402
this.chunkBufferEncoding = encoding;
422403

423404
this.once('readFromChunkBuffer', readCallback);
405+
424406
process.nextTick(() => this.emit('wroteToChunkBuffer'));
425407
}
426408

@@ -485,22 +467,16 @@ export class Upload extends Pumpify {
485467
};
486468

487469
// Remove listeners when we're ready to callback.
488-
// It's important to clean-up listeners as Node has a default max number of
489-
// event listeners. Notably, The number of requests can be greater than the
490-
// number of potential listeners.
491-
// - https://nodejs.org/api/events.html#eventsdefaultmaxlisteners
492470
const removeListeners = () => {
493471
this.removeListener('wroteToChunkBuffer', wroteToChunkBufferCallback);
494-
this.upstream.removeListener('finish', upstreamFinishedCallback);
495-
this.removeListener('prefinish', upstreamFinishedCallback);
472+
this.removeListener('upstreamFinished', upstreamFinishedCallback);
496473
};
497474

498475
// If there's data recently written it should be digested
499476
this.once('wroteToChunkBuffer', wroteToChunkBufferCallback);
500477

501478
// If the upstream finishes let's see if there's anything to grab
502-
this.upstream.once('finish', upstreamFinishedCallback);
503-
this.once('prefinish', upstreamFinishedCallback);
479+
this.once('upstreamFinished', upstreamFinishedCallback);
504480
});
505481

506482
return willBeMoreChunks;
@@ -662,7 +638,7 @@ export class Upload extends Pumpify {
662638
// Check if the offset (server) is too far behind the current stream
663639
if (this.offset < this.numBytesWritten) {
664640
const delta = this.numBytesWritten - this.offset;
665-
const message = `The offset is lower than the number of bytes written. The server has ${this.offset} bytes and while ${this.numBytesWritten} bytes has been uploaded - thus ${delta} bytes are missing. Stopping as this could result in data loss. Initiate a new upload to continue.`;
641+
const message = `The offset is lower than the number of bytes written. The server has ${this.offset} bytes and while ${this.numBytesWritten} bytes has been uploaded - thus ${delta} bytes are missing. Stopping as this could result in data loss. Initiate a new upload to continue.`;
666642

667643
this.emit('error', new RangeError(message));
668644
return;
@@ -811,7 +787,7 @@ export class Upload extends Pumpify {
811787
};
812788
this.destroy(err);
813789
} else {
814-
// remove the last chunk sent
790+
// remove the last chunk sent to free memory
815791
this.lastChunkSent = Buffer.alloc(0);
816792

817793
if (resp && resp.data) {
@@ -822,7 +798,7 @@ export class Upload extends Pumpify {
822798

823799
// Allow the object (Upload) to continue naturally so the user's
824800
// "finish" event fires.
825-
this.emit('prepareFinish');
801+
this.emit('uploadFinished');
826802
}
827803
}
828804

@@ -902,10 +878,6 @@ export class Upload extends Pumpify {
902878
}
903879

904880
// this resumable upload is unrecoverable (bad data or service error).
905-
// -
906-
// https://github.com/googleapis/gcs-resumable-upload/issues/15
907-
// -
908-
// https://github.com/googleapis/gcs-resumable-upload/pull/16#discussion_r80363774
909881
if (resp && resp.status === TERMINATED_UPLOAD_STATUS_CODE) {
910882
this.restart();
911883
return;

0 commit comments

Comments
 (0)