Skip to content
This repository was archived by the owner on Jan 13, 2024. It is now read-only.

Commit ea23196

Browse files
erossignonerossignon
authored
fs.createReadStream and --compress (#1158)
1 parent c2b7d6c commit ea23196

9 files changed

Lines changed: 360 additions & 12 deletions

File tree

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -348,7 +348,7 @@ and check that all the required files for your application are properly
348348
incorporated to the final executable.
349349

350350
$ pkg --debug app.js -o output
351-
$ DEBUG_PKG output
351+
$ DEBUG_PKG=1 output
352352

353353
or
354354

‎prelude/bootstrap.js‎

Lines changed: 133 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,10 @@ const {
2828
gunzipSync,
2929
brotliDecompressSync,
3030
brotliDecompress,
31+
createBrotliDecompress,
32+
createGunzip,
3133
} = require('zlib');
34+
const { createHash } = require('crypto');
3235

3336
const common = {};
3437
REQUIRE_COMMON(common);
@@ -510,6 +513,7 @@ function payloadFileSync(pointer) {
510513
ancestor.access = fs.access;
511514
ancestor.mkdirSync = fs.mkdirSync;
512515
ancestor.mkdir = fs.mkdir;
516+
ancestor.createReadStream = fs.createReadStream;
513517

514518
const windows = process.platform === 'win32';
515519

@@ -567,10 +571,53 @@ function payloadFileSync(pointer) {
567571
// open //////////////////////////////////////////////////////////
568572
// ///////////////////////////////////////////////////////////////
569573

574+
function removeTemporaryFolderAndContent(folder) {
575+
if (NODE_VERSION_MAJOR <= 14) {
576+
if (NODE_VERSION_MAJOR <= 10) {
577+
// folder must be empty
578+
for (const f of fs.readdirSync(folder)) {
579+
fs.unlinkSync(path.join(folder, f));
580+
}
581+
fs.rmdirSync(folder);
582+
} else {
583+
fs.rmdirSync(folder, { recursive: true });
584+
}
585+
} else {
586+
fs.rmSync(folder, { recursive: true });
587+
}
588+
}
589+
const temporaryFiles = {};
590+
const os = require('os');
591+
const tmpFolder = fs.mkdtempSync(path.join(os.tmpdir(), 'pkg-'));
592+
process.on('beforeExit', () => {
593+
removeTemporaryFolderAndContent(tmpFolder);
594+
});
595+
function deflateSync(snapshotFilename) {
596+
const content = fs.readFileSync(snapshotFilename, { encoding: 'binary' });
597+
// content is already unziped !
598+
const hash = createHash('sha256').update(content).digest('hex');
599+
const fName = path.join(tmpFolder, hash);
600+
fs.writeFileSync(fName, content);
601+
return fName;
602+
}
603+
function uncompressExternally(snapshotFilename) {
604+
let t = temporaryFiles[snapshotFilename];
605+
if (!t) {
606+
const tmpFile = deflateSync(snapshotFilename);
607+
t = { tmpFile };
608+
temporaryFiles[snapshotFilename] = t;
609+
}
610+
return t.tmpFile;
611+
}
612+
function uncompressExternallyAndOpen(snapshotFilename) {
613+
const externalFile = uncompressExternally(snapshotFilename);
614+
const fd = fs.openSync(externalFile, 'r');
615+
return fd;
616+
}
617+
570618
function openFromSnapshot(f, cb) {
571619
const cb2 = cb || rethrow;
572620
const entity = findVirtualFileSystemEntry(f);
573-
574621
if (!entity) return cb2(error_ENOENT('File or directory', f));
575622
const dock = { path: f, entity, position: 0 };
576623
const nullDevice = windows ? '\\\\.\\NUL' : '/dev/null';
@@ -587,13 +634,37 @@ function payloadFileSync(pointer) {
587634
}
588635
}
589636

637+
let bypassCompressCheckWhenCallbyCreateReadStream = false;
638+
639+
fs.createReadStream = function createReadStream(f) {
640+
if (!insideSnapshot(f)) {
641+
return ancestor.createReadStream.apply(fs, arguments);
642+
}
643+
if (insideMountpoint(f)) {
644+
return ancestor.createReadStream.apply(fs, translateNth(arguments, 0, f));
645+
}
646+
bypassCompressCheckWhenCallbyCreateReadStream = true;
647+
const stream = ancestor.createReadStream.apply(fs, arguments);
648+
bypassCompressCheckWhenCallbyCreateReadStream = false;
649+
650+
if (DOCOMPRESS === GZIP) {
651+
return stream.pipe(createGunzip());
652+
}
653+
if (DOCOMPRESS === BROTLI) {
654+
return stream.pipe(createBrotliDecompress());
655+
}
656+
return stream;
657+
};
590658
fs.openSync = function openSync(f) {
591659
if (!insideSnapshot(f)) {
592660
return ancestor.openSync.apply(fs, arguments);
593661
}
594662
if (insideMountpoint(f)) {
595663
return ancestor.openSync.apply(fs, translateNth(arguments, 0, f));
596664
}
665+
if (DOCOMPRESS && !bypassCompressCheckWhenCallbyCreateReadStream) {
666+
return uncompressExternallyAndOpen(f);
667+
}
597668
return openFromSnapshot(f);
598669
};
599670

@@ -604,8 +675,11 @@ function payloadFileSync(pointer) {
604675
if (insideMountpoint(f)) {
605676
return ancestor.open.apply(fs, translateNth(arguments, 0, f));
606677
}
607-
608678
const callback = dezalgo(maybeCallback(arguments));
679+
if (DOCOMPRESS && !bypassCompressCheckWhenCallbyCreateReadStream) {
680+
const fd = uncompressExternallyAndOpen(f);
681+
return callback(null, fd);
682+
}
609683
openFromSnapshot(f, callback);
610684
};
611685

@@ -1344,18 +1418,70 @@ function payloadFileSync(pointer) {
13441418
// promises ////////////////////////////////////////////////////////
13451419
// ///////////////////////////////////////////////////////////////
13461420

1421+
const ancestor_promises = {};
13471422
if (fs.promises !== undefined) {
13481423
var util = require('util');
1349-
fs.promises.open = util.promisify(fs.open);
1350-
fs.promises.read = util.promisify(fs.read);
1351-
fs.promises.write = util.promisify(fs.write);
1352-
fs.promises.readFile = util.promisify(fs.readFile);
1424+
ancestor_promises.open = fs.promises.open;
1425+
ancestor_promises.read = fs.promises.read;
1426+
ancestor_promises.write = fs.promises.write;
1427+
ancestor_promises.readFile = fs.promises.readFile;
1428+
ancestor_promises.readdir = fs.promises.readdir;
1429+
ancestor_promises.realpath = fs.promises.realpath;
1430+
ancestor_promises.stat = fs.promises.stat;
1431+
ancestor_promises.lstat = fs.promises.lstat;
1432+
ancestor_promises.fstat = fs.promises.fstat;
1433+
ancestor_promises.access = fs.promises.access;
1434+
1435+
fs.promises.open = async function open(f) {
1436+
if (!insideSnapshot(f)) {
1437+
return ancestor_promises.open.apply(this, arguments);
1438+
}
1439+
if (insideMountpoint(f)) {
1440+
return ancestor_promises.open.apply(
1441+
this,
1442+
translateNth(arguments, 0, f)
1443+
);
1444+
}
1445+
const externalFile = uncompressExternally(f);
1446+
arguments[0] = externalFile;
1447+
const fd = await ancestor_promises.open.apply(this, arguments);
1448+
if (typeof fd === 'object') {
1449+
fd._pkg = { externalFile, file: f };
1450+
}
1451+
return fd;
1452+
};
1453+
fs.promises.readFile = async function readFile(f) {
1454+
if (!insideSnapshot(f)) {
1455+
return ancestor_promises.readFile.apply(this, arguments);
1456+
}
1457+
if (insideMountpoint(f)) {
1458+
return ancestor_promises.readFile.apply(
1459+
this,
1460+
translateNth(arguments, 0, f)
1461+
);
1462+
}
1463+
const externalFile = uncompressExternally(f);
1464+
arguments[0] = externalFile;
1465+
return ancestor_promises.readFile.apply(this, arguments);
1466+
};
1467+
fs.promises.write = async function write(fd) {
1468+
if (fd._pkg) {
1469+
throw new Error(
1470+
`[PKG] Cannot write into Snapshot file : ${fd._pkg.file}`
1471+
);
1472+
}
1473+
return ancestor_promises.write.apply(this, arguments);
1474+
};
13531475
fs.promises.readdir = util.promisify(fs.readdir);
1476+
1477+
/*
1478+
fs.promises.read = util.promisify(fs.read);
13541479
fs.promises.realpath = util.promisify(fs.realpath);
13551480
fs.promises.stat = util.promisify(fs.stat);
13561481
fs.promises.lstat = util.promisify(fs.lstat);
13571482
fs.promises.fstat = util.promisify(fs.fstat);
13581483
fs.promises.access = util.promisify(fs.access);
1484+
*/
13591485
}
13601486

13611487
// ///////////////////////////////////////////////////////////////
@@ -1897,10 +2023,7 @@ function payloadFileSync(pointer) {
18972023

18982024
// Node addon files and .so cannot be read with fs directly, they are loaded with process.dlopen which needs a filesystem path
18992025
// we need to write the file somewhere on disk first and then load it
1900-
const hash = require('crypto')
1901-
.createHash('sha256')
1902-
.update(moduleContent)
1903-
.digest('hex');
2026+
const hash = createHash('sha256').update(moduleContent).digest('hex');
19042027

19052028
const tmpFolder = path.join(require('os').tmpdir(), hash);
19062029
if (!fs.existsSync(tmpFolder)) {

‎test/test-12-compression-node-opcua/main.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ assert(
4747

4848
/* eslint-disable no-unused-vars */
4949
const input = 'package.json';
50-
const target = 'host';
50+
const target = process.argv[2] || 'host';
5151
const ext = process.platform === 'win32' ? '.exe' : '';
5252
const outputRef = 'test-output-empty' + ext;
5353
const outputNone = 'test-output-None' + ext;

‎test/test-12-compression-various-file-access/main.js‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
#!/usr/bin/env node
2+
3+
'use strict';
4+
5+
const path = require('path');
6+
const assert = require('assert');
7+
const utils = require('../utils.js');
8+
9+
assert(!module.parent);
10+
assert(__dirname === process.cwd());
11+
12+
/* eslint-disable no-unused-vars */
13+
const target = process.argv[2] || 'host';
14+
const ext = process.platform === 'win32' ? '.exe' : '';
15+
const output = 'output' + ext;
16+
17+
async function runTest(input) {
18+
const logPkgNone = utils.pkg.sync(
19+
['--target', target, '--compress', 'None', '--output', output, input],
20+
{ expect: 0 }
21+
);
22+
const logPkgGZip = utils.pkg.sync(
23+
[
24+
'--target',
25+
target,
26+
'--compress',
27+
'GZIP',
28+
'--output',
29+
'gzip_' + output,
30+
input,
31+
],
32+
{ expect: 0 }
33+
);
34+
35+
// -----------------------------------------------------------------------
36+
// Execute programm outside pjg
37+
const logRef = utils.spawn.sync('node', [path.join(__dirname, input)], {
38+
cwd: __dirname,
39+
expect: 0,
40+
});
41+
42+
const logNone = utils.spawn.sync(path.join(__dirname, output), [], {
43+
cwd: __dirname,
44+
expect: 0,
45+
});
46+
47+
const logGZip = utils.spawn.sync(path.join(__dirname, 'gzip_' + output), [], {
48+
cwd: __dirname,
49+
expect: 0,
50+
});
51+
52+
if (logRef !== logNone) {
53+
console.log(
54+
" uncompress pkg doesn't produce same result as running with node"
55+
);
56+
}
57+
if (logRef !== logGZip) {
58+
console.log(
59+
" GZIP compress pkg doesn't produce same result as running with node"
60+
);
61+
}
62+
63+
if (logRef !== logNone || logRef !== logGZip) {
64+
console.log(' Reference:');
65+
console.log(logRef);
66+
console.log(' Uncompress:');
67+
console.log(logNone);
68+
console.log(' GZIPed:');
69+
console.log(logGZip);
70+
71+
process.exit(1);
72+
}
73+
utils.vacuum.sync(output);
74+
utils.vacuum.sync('gzip_' + output);
75+
}
76+
77+
const input1 = 'test.js';
78+
79+
console.log(' now testing with fs callback');
80+
runTest(input1);
81+
82+
console.log(' now testing with fs.promises');
83+
const input2 = 'test_with_new_fs_promises.js';
84+
runTest(input2);
85+
console.log('Done');

‎test/test-12-compression-various-file-access/myfile.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ
2+
123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ

‎test/test-12-compression-various-file-access/myfile.txt1.gz‎

75 Bytes
Binary file not shown.

‎test/test-12-compression-various-file-access/package.json‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
{
2+
"name": "test-12-compression-with-asset",
3+
"version": "1.0.0",
4+
"description": "",
5+
"main": "index.js",
6+
"scripts": {
7+
"test": "echo \"Error: no test specified\" && exit 1"
8+
},
9+
"keywords": [],
10+
"author": "",
11+
"license": "ISC"
12+
}

‎test/test-12-compression-various-file-access/test.js‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
'use strict';
2+
3+
const path = require('path');
4+
const fs = require('fs');
5+
6+
const file = path.join(__dirname, 'myfile.txt');
7+
8+
function withReadFileSync() {
9+
const wholeFile = fs.readFileSync(file, 'ascii');
10+
console.log(wholeFile);
11+
console.log('withReadFileSync done !');
12+
}
13+
14+
async function withDirectAccess() {
15+
try {
16+
const buffer = Buffer.alloc(1000);
17+
const fd = fs.openSync(file);
18+
fs.readSync(fd, buffer, 0, buffer.length, 10);
19+
fs.closeSync(fd);
20+
console.log(buffer.toString('ascii'));
21+
} catch (err) {
22+
console.log(err.message);
23+
console.log(err);
24+
}
25+
console.log('withDirectAccess ! done!');
26+
}
27+
28+
async function withReadStream() {
29+
const stream = fs.createReadStream(
30+
file /* { start: 10, encoding: "ascii" } */
31+
);
32+
stream.on('data', (data) => {
33+
console.log(data.toString());
34+
});
35+
await new Promise((resolve) => {
36+
stream.on('end', resolve);
37+
});
38+
console.log('withReadStream done !');
39+
}
40+
41+
(async () => {
42+
await withReadFileSync();
43+
await withReadStream();
44+
await withDirectAccess();
45+
})();

0 commit comments

Comments
 (0)