Skip to content

Commit 52ddf23

Browse files
jesecjesec
committed
Squashed revert "allow to compress files in virtual file system (vercel#1115)"
This reverts commit 82ce625. This reverts commit ea23196. This reverts commit ea7d72b. Bug: vercel#1191, vercel#1192
1 parent e8a2c63 commit 52ddf23

33 files changed

Lines changed: 437 additions & 1258 deletions

File tree

‎.eslintrc‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
{
2-
// note you must disable the base rule as it can report incorrect errors
32
"extends": ["airbnb-base", "prettier"],
43
"rules": {
54
"no-bitwise": "off",
@@ -11,10 +10,8 @@
1110
"consistent-return": "off",
1211
"no-restricted-syntax": "off",
1312
"import/prefer-default-export": "off",
14-
"camelcase": "off",
15-
"no-shadow": "off",
16-
"@typescript-eslint/no-shadow": ["error"]
17-
},
13+
"camelcase": "off"
14+
},
1815
"overrides": [
1916
{
2017
"files": ["*.ts"],

‎README.md‎

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -209,16 +209,6 @@ requirements to compile original Node.js:
209209

210210
See [pkg-fetch](https://github.com/vercel/pkg-fetch) for more info.
211211

212-
### Compression
213-
214-
Pass `--compress Brotli` or `--compress GZip` to `pkg` to compress further the content of the files store in the exectable.
215-
216-
This option can reduce the size of the embedded file system by up to 60%.
217-
218-
The startup time of the application might be reduced slightly.
219-
220-
`-C` can be used as a shortcut for `--compress `.
221-
222212
### Environment
223213

224214
| Var | Description |
@@ -368,7 +358,7 @@ and check that all the required files for your application are properly
368358
incorporated to the final executable.
369359

370360
$ pkg --debug app.js -o output
371-
$ DEBUG_PKG=1 output
361+
$ DEBUG_PKG output
372362

373363
or
374364

‎lib/compress_type.ts‎

Lines changed: 0 additions & 6 deletions
This file was deleted.

‎lib/help.ts‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ export default function help() {
1919
--public speed up and disclose the sources of top-level project
2020
--public-packages force specified packages to be considered public
2121
--no-bytecode skip bytecode generation and include source files as plain js
22-
-C, --compress [default=None] compression algorithm = Brotli or GZip
2322
2423
${chalk.dim('Examples:')}
2524
@@ -37,11 +36,5 @@ export default function help() {
3736
${chalk.cyan('$ pkg --public-packages "packageA,packageB" index.js')}
3837
${chalk.gray('–')} Consider all packages to be public
3938
${chalk.cyan('$ pkg --public-packages "*" index.js')}
40-
${chalk.gray('–')} Bakes '--expose-gc' into executable
41-
${chalk.cyan('$ pkg --options expose-gc index.js')}
42-
${chalk.gray(
43-
'–'
44-
)} reduce size of the data packed inside the executable with GZip
45-
${chalk.cyan('$ pkg --compress GZip index.js')}
4639
`);
4740
}

‎lib/index.ts‎

Lines changed: 2 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@ import refine from './refiner';
2626
import { shutdown } from './fabricator';
2727
import walk, { Marker, WalkerParams } from './walker';
2828
import { Target, NodeTarget, SymLinks } from './types';
29-
import { CompressType } from './compress_type';
3029
import { patchMachOExecutable } from './mach-o';
3130

3231
const { version } = JSON.parse(
@@ -248,8 +247,6 @@ export async function exec(argv2: string[]) {
248247
't',
249248
'target',
250249
'targets',
251-
'C',
252-
'compress',
253250
],
254251
default: { bytecode: true },
255252
});
@@ -277,31 +274,7 @@ export async function exec(argv2: string[]) {
277274

278275
const forceBuild = argv.b || argv.build;
279276

280-
// doCompress
281-
const algo = argv.C || argv.compress || 'None';
282-
283-
let doCompress: CompressType = CompressType.None;
284-
switch (algo.toLowerCase()) {
285-
case 'brotli':
286-
case 'br':
287-
doCompress = CompressType.Brotli;
288-
break;
289-
case 'gzip':
290-
case 'gz':
291-
doCompress = CompressType.GZip;
292-
break;
293-
case 'none':
294-
break;
295-
default:
296-
// eslint-disable-next-line no-console
297-
throw wasReported(
298-
`Invalid compression algorithm ${algo} ( should be None, Brotli or Gzip)`
299-
);
300-
}
301-
if (doCompress !== CompressType.None) {
302-
// eslint-disable-next-line no-console
303-
console.log('compression: ', CompressType[doCompress]);
304-
}
277+
// _
305278

306279
if (!argv._.length) {
307280
throw wasReported('Entry file/directory is expected', [
@@ -652,14 +625,12 @@ export async function exec(argv2: string[]) {
652625
await mkdirp(path.dirname(target.output));
653626
}
654627

655-
const slash = target.platform === 'win' ? '\\' : '/';
656628
await producer({
657629
backpack,
658630
bakes,
659-
slash,
631+
slash: target.platform === 'win' ? '\\' : '/',
660632
target: target as Target,
661633
symLinks,
662-
doCompress,
663634
});
664635

665636
if (target.platform !== 'win' && target.output) {

‎lib/packer.ts‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
/* eslint-disable complexity */
22

33
import assert from 'assert';
4-
import * as fs from 'fs-extra';
5-
import * as path from 'path';
4+
import fs from 'fs-extra';
5+
import path from 'path';
66

77
import {
88
STORE_BLOB,
@@ -157,7 +157,7 @@ export default function packer({
157157
}
158158
}
159159
const prelude =
160-
`return (function (REQUIRE_COMMON, VIRTUAL_FILESYSTEM, DEFAULT_ENTRYPOINT, SYMLINKS, DICT, DOCOMPRESS) {
160+
`return (function (REQUIRE_COMMON, VIRTUAL_FILESYSTEM, DEFAULT_ENTRYPOINT, SYMLINKS) {
161161
${bootstrapText}${
162162
log.debugMode ? diagnosticText : ''
163163
}\n})(function (exports) {\n${commonText}\n},\n` +
@@ -166,10 +166,6 @@ export default function packer({
166166
`%DEFAULT_ENTRYPOINT%` +
167167
`\n,\n` +
168168
`%SYMLINKS%` +
169-
'\n,\n' +
170-
'%DICT%' +
171-
'\n,\n' +
172-
'%DOCOMPRESS%' +
173169
`\n);`;
174170

175171
return { prelude, entrypoint, stripes };

‎lib/producer.ts‎

Lines changed: 24 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
import { createBrotliCompress, createGzip } from 'zlib';
21
import Multistream from 'multistream';
32
import assert from 'assert';
43
import { execSync } from 'child_process';
@@ -13,7 +12,6 @@ import { log, wasReported } from './log';
1312
import { fabricateTwice } from './fabricator';
1413
import { platform, SymLinks, Target } from './types';
1514
import { Stripe } from './packer';
16-
import { CompressType } from './compress_type';
1715

1816
interface NotFound {
1917
notFound: true;
@@ -227,7 +225,8 @@ function nativePrebuildInstall(target: Target, nodeFile: string) {
227225
}
228226

229227
execSync(
230-
`${prebuild} -t ${nodeVersion} --platform ${platform[target.platform]
228+
`${prebuild} -t ${nodeVersion} --platform ${
229+
platform[target.platform]
231230
} --arch ${target.arch}`,
232231
{ cwd: dir }
233232
);
@@ -243,34 +242,14 @@ interface ProducerOptions {
243242
slash: string;
244243
target: Target;
245244
symLinks: SymLinks;
246-
doCompress: CompressType;
247245
}
248246

249-
const fileDictionary: { [key: string]: string } = {};
250-
let counter = 0;
251-
function replace(k: string) {
252-
let existingKey = fileDictionary[k];
253-
if (!existingKey) {
254-
const newkey = counter;
255-
counter += 1;
256-
existingKey = newkey.toString(36);
257-
fileDictionary[k] = existingKey;
258-
}
259-
return existingKey;
260-
}
261-
const separator = '$';
262-
263-
function makeKey(filename: string, slash: string): string {
264-
const a = filename.split(slash).map(replace).join(separator);
265-
return a;
266-
}
267247
export default function producer({
268248
backpack,
269249
bakes,
270250
slash,
271251
target,
272252
symLinks,
273-
doCompress
274253
}: ProducerOptions) {
275254
return new Promise<void>((resolve, reject) => {
276255
if (!Buffer.alloc) {
@@ -289,17 +268,18 @@ export default function producer({
289268
for (const stripe of stripes) {
290269
let { snap } = stripe;
291270
snap = snapshotify(snap, slash);
292-
const vfsKey = makeKey(snap, slash);
293-
if (!vfs[vfsKey]) vfs[vfsKey] = {};
271+
272+
if (!vfs[snap]) {
273+
vfs[snap] = {};
274+
}
294275
}
295276

296277
const snapshotSymLinks: SymLinks = {};
297278

298279
for (const [key, value] of Object.entries(symLinks)) {
299280
const k = snapshotify(key, slash);
300281
const v = snapshotify(value, slash);
301-
const vfsKey = makeKey(k, slash);
302-
snapshotSymLinks[vfsKey] = makeKey(v, slash);
282+
snapshotSymLinks[k] = v;
303283
}
304284

305285
let meter: streamMeter.StreamMeter;
@@ -309,15 +289,6 @@ export default function producer({
309289
meter = streamMeter();
310290
return s.pipe(meter);
311291
}
312-
function pipeMayCompressToNewMeter(s: Readable): streamMeter.StreamMeter {
313-
if (doCompress === CompressType.GZip) {
314-
return pipeToNewMeter(s.pipe(createGzip()));
315-
}
316-
if (doCompress === CompressType.Brotli) {
317-
return pipeToNewMeter(s.pipe(createBrotliCompress()));
318-
}
319-
return pipeToNewMeter(s);
320-
}
321292

322293
function next(s: Readable) {
323294
count += 1;
@@ -350,8 +321,7 @@ export default function producer({
350321
const { store } = prevStripe;
351322
let { snap } = prevStripe;
352323
snap = snapshotify(snap, slash);
353-
const vfsKey = makeKey(snap, slash);
354-
vfs[vfsKey][store] = [track, meter.bytes];
324+
vfs[snap][store] = [track, meter.bytes];
355325
track += meter.bytes;
356326
}
357327

@@ -377,14 +347,15 @@ export default function producer({
377347
return cb(null, intoStream(Buffer.alloc(0)));
378348
}
379349

380-
cb(null, pipeMayCompressToNewMeter(intoStream(buffer || Buffer.from(''))));
350+
cb(
351+
null,
352+
pipeToNewMeter(intoStream(buffer || Buffer.from('')))
353+
);
381354
}
382355
);
383356
}
384-
return cb(
385-
null,
386-
pipeMayCompressToNewMeter(intoStream(stripe.buffer))
387-
);
357+
358+
return cb(null, pipeToNewMeter(intoStream(stripe.buffer)));
388359
}
389360

390361
if (stripe.file) {
@@ -407,17 +378,15 @@ export default function producer({
407378
if (fs.existsSync(platformFile)) {
408379
return cb(
409380
null,
410-
pipeMayCompressToNewMeter(fs.createReadStream(platformFile))
381+
pipeToNewMeter(fs.createReadStream(platformFile))
411382
);
412383
}
413384
} catch (err) {
414385
log.debug(`prebuild-install failed[${stripe.file}]:`, err);
415386
}
416387
}
417-
return cb(
418-
null,
419-
pipeMayCompressToNewMeter(fs.createReadStream(stripe.file))
420-
);
388+
389+
return cb(null, pipeToNewMeter(fs.createReadStream(stripe.file)));
421390
}
422391

423392
assert(false, 'producer: bad stripe');
@@ -432,23 +401,15 @@ export default function producer({
432401
replaceDollarWise(
433402
replaceDollarWise(
434403
replaceDollarWise(
435-
replaceDollarWise(
436-
replaceDollarWise(
437-
prelude,
438-
'%VIRTUAL_FILESYSTEM%',
439-
JSON.stringify(vfs)
440-
),
441-
'%DEFAULT_ENTRYPOINT%',
442-
JSON.stringify(entrypoint)
443-
),
444-
'%SYMLINKS%',
445-
JSON.stringify(snapshotSymLinks)
404+
prelude,
405+
'%VIRTUAL_FILESYSTEM%',
406+
JSON.stringify(vfs)
446407
),
447-
'%DICT%',
448-
JSON.stringify(fileDictionary)
408+
'%DEFAULT_ENTRYPOINT%',
409+
JSON.stringify(entrypoint)
449410
),
450-
'%DOCOMPRESS%',
451-
JSON.stringify(doCompress)
411+
'%SYMLINKS%',
412+
JSON.stringify(snapshotSymLinks)
452413
)
453414
)
454415
)

‎lib/walker.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -497,7 +497,6 @@ class Walker {
497497
assets = expandFiles(assets, base);
498498

499499
for (const asset of assets) {
500-
log.debug(' Adding asset : .... ', asset);
501500
const stat = await fs.stat(asset);
502501

503502
if (stat.isFile()) {

‎package.json‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,6 @@
7272
"lint": "eslint lib prelude test",
7373
"lint:fix": "npm run lint -- --fix",
7474
"prepare": "npm run build",
75-
"prepublishOnly": "npm run lint",
7675
"test": "npm run build && node test/test.js node14 no-npm && node test/test.js node12 no-npm && node test/test.js node10 no-npm && node test/test.js host only-npm"
7776
},
7877
"greenkeeper": {

0 commit comments

Comments
 (0)