Skip to content

Commit 1e7a181

Browse files
bestanderbestander
committed
Merge pull request #60 from bestander/npm-offline-mirror-support
Npm offline mirror support
2 parents 90aa0e7 + 72c4b43 commit 1e7a181

File tree

19 files changed

+211
-31
lines changed

19 files changed

+211
-31
lines changed

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
"cmd-shim": "^2.0.1",
1212
"commander": "^2.9.0",
1313
"diff": "^2.2.1",
14+
"ini": "^1.3.4",
1415
"invariant": "^2.2.0",
1516
"is-builtin-module": "^1.0.0",
1617
"kreporters": "^1.0.2",

‎src/cli/commands/install.js‎

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -93,8 +93,6 @@ export class Install {
9393
let patterns = [];
9494
let deps = [];
9595

96-
let foundConfig = false;
97-
9896
for (let registry of Object.keys(registries)) {
9997
let filenames = registries[registry].filenames;
10098

@@ -103,7 +101,6 @@ export class Install {
103101
if (!(await fs.exists(loc))) continue;
104102

105103
this.registries.push(registry);
106-
foundConfig = true;
107104

108105
let json = await fs.readJson(loc);
109106
Object.assign(this.resolutions, json.resolutions);
@@ -159,15 +156,13 @@ export class Install {
159156
let totalStepsThis = totalSteps + 2;
160157

161158
this.reporter.step(1, totalStepsThis, "Rehydrating dependency graph", emoji.get("fast_forward"));
162-
await this.resolver.init(requests, true);
163-
164-
patterns = await this.flatten(patterns);
159+
await this.resolver.init(requests);
165160

166161
this.reporter.step(2, totalStepsThis, "Fetching packages", emoji.get("package"));
167162
await this.resolver.fetcher.init();
168163

169164
return {
170-
patterns,
165+
patterns: await this.flatten(patterns),
171166
total: totalStepsThis,
172167
step: 2
173168
};
@@ -231,7 +226,7 @@ export class Install {
231226
* Save added packages to `package.json` if any of the --save flags were used
232227
*/
233228

234-
async savePackages(patterns: Array<string>) {
229+
async savePackages(patterns: Array<string>): Promise<void> {
235230
if (!this.args.length) return;
236231

237232
let { save, saveDev, saveExact, saveOptional } = this.flags;
@@ -417,7 +412,8 @@ export async function run(
417412
throw new MessageError("Missing package names for --save flags");
418413
}
419414

420-
let lockfile = await Lockfile.fromDirectory(config.cwd, reporter, isStrictLockfile(flags, args));
415+
let lockfile = await Lockfile.fromDirectory(config.cwd, reporter, isStrictLockfile(flags, args),
416+
hasSaveFlags(flags));
421417
let install = new Install("install", flags, args, config, reporter, lockfile);
422418
return install.init();
423419
}

‎src/config.js‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import map from "./util/map.js";
2424
let invariant = require("invariant");
2525
let userHome = require("user-home");
2626
let path = require("path");
27+
let url = require("url");
2728

2829
type ConfigOptions = {
2930
cwd?: string,
@@ -140,6 +141,27 @@ export default class Config {
140141
return path.join(this.tempFolder, filename);
141142
}
142143

144+
/**
145+
* Remote packages may be cached in a file system to be available for offline installation
146+
* Second time the same package needs to be installed it will be loaded from there
147+
*/
148+
149+
getOfflineMirrorPath(registry: RegistryNames, tarUrl: ?string): string {
150+
let mirrorPath = this.registries[registry] && this.registries[registry]
151+
.config["kpm-offline-mirror"];
152+
if (!mirrorPath) {
153+
return "";
154+
}
155+
if (!tarUrl) {
156+
return mirrorPath;
157+
}
158+
let parsed = url.parse(tarUrl);
159+
if (!parsed || !parsed.pathname) {
160+
return mirrorPath;
161+
}
162+
return path.join(mirrorPath, path.basename(parsed.pathname));
163+
}
164+
143165
/**
144166
* Find temporary folder.
145167
*/
@@ -160,7 +182,7 @@ export default class Config {
160182
return opts.packagesRoot;
161183
}
162184

163-
// walk up from current directory looking for fbkpm_modules folders
185+
// walk up from current directory looking for .fbkpm folders
164186
let parts = this.cwd.split(path.sep);
165187
for (let i = parts.length; i > 0; i--) {
166188
let loc = parts.slice(0, i).concat(constants.MODULE_CACHE_DIRECTORY).join(path.sep);

‎src/fetchers/_base.js‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,17 +20,19 @@ import * as fs from "../util/fs.js";
2020
let path = require("path");
2121

2222
export default class BaseFetcher {
23-
constructor(remote: PackageRemote, config: Config) {
24-
this.reference = remote.reference;
25-
this.registry = remote.registry;
26-
this.hash = remote.hash;
27-
this.config = config;
23+
constructor(remote: PackageRemote, config: Config, saveForOffline: boolean) {
24+
this.reference = remote.reference;
25+
this.registry = remote.registry;
26+
this.hash = remote.hash;
27+
this.config = config;
28+
this.saveForOffline = saveForOffline;
2829
}
2930

3031
registry: RegistryNames;
3132
reference: string;
3233
config: Config;
3334
hash: ?string;
35+
saveForOffline: boolean;
3436

3537
async _fetch(dest: string): Promise<string> {
3638
throw new Error("Not implemented");

‎src/fetchers/tarball.js‎

Lines changed: 54 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,24 +9,61 @@
99
* @flow
1010
*/
1111

12-
import { SecurityError } from "../errors.js";
12+
import { SecurityError, MessageError } from "../errors.js";
1313
import * as crypto from "../util/crypto.js";
1414
import BaseFetcher from "./_base.js";
15+
import * as fsUtil from "../util/fs.js";
1516

1617
let zlib = require("zlib");
1718
let tar = require("tar");
1819
let url = require("url");
20+
let through = require("through2");
21+
let fs = require("fs");
22+
let path = require("path");
1923

2024
export default class TarballFetcher extends BaseFetcher {
25+
2126
async _fetch(dest: string): Promise<string> {
22-
let { reference: ref, hash } = this;
27+
let { reference: ref, hash, config, saveForOffline, registry } = this;
2328

29+
let parts = url.parse(ref);
2430
if (!hash) {
25-
let parts = url.parse(ref);
2631
if (parts.protocol === "http:") {
2732
throw new SecurityError(`${ref}: Refusing to fetch tarball over plain HTTP without a hash`);
2833
}
2934
}
35+
if (parts.protocol === null) {
36+
let localTarball = path.resolve(
37+
this.config.getOfflineMirrorPath(registry, null),
38+
ref);
39+
if (!await fsUtil.exists(localTarball)) {
40+
throw new MessageError(`${ref}: Tarball is not in network and can't be located in cache`);
41+
}
42+
return new Promise((resolve, reject) => {
43+
let validateStream = crypto.hashStreamValidation();
44+
45+
let extractor = tar.Extract({ path: dest, strip: 1 })
46+
.on("error", reject)
47+
.on("end", function () {
48+
let expectHash = hash;
49+
let actualHash = validateStream.getHash();
50+
if (!expectHash || expectHash === actualHash) {
51+
resolve(actualHash);
52+
} else {
53+
reject(new SecurityError(
54+
`Bad hash. Expected ${expectHash} but got ${actualHash}`
55+
));
56+
}
57+
});
58+
// flow gets confused with the pipe/on types chain
59+
let cachedStream: Object = fs.createReadStream(localTarball);
60+
cachedStream
61+
.pipe(validateStream)
62+
.pipe(zlib.createUnzip())
63+
.on("error", reject)
64+
.pipe(extractor);
65+
});
66+
}
3067

3168
return this.config.requestManager.request({
3269
url: ref,
@@ -50,6 +87,19 @@ export default class TarballFetcher extends BaseFetcher {
5087
}
5188
});
5289

90+
let mirrorPath = config.getOfflineMirrorPath(registry, ref);
91+
let mirrorTarballStream;
92+
if (mirrorPath && saveForOffline) {
93+
mirrorTarballStream = fs.createWriteStream(mirrorPath);
94+
mirrorTarballStream.on("error", reject);
95+
}
96+
let mirrorSaver = through(function (chunk, enc, callback) {
97+
if (mirrorTarballStream) {
98+
mirrorTarballStream.write(chunk, enc);
99+
}
100+
callback(null, chunk);
101+
});
102+
53103
req
54104
.on("redirect", function () {
55105
if (hash) return;
@@ -64,6 +114,7 @@ export default class TarballFetcher extends BaseFetcher {
64114
}
65115
})
66116
.pipe(validateStream)
117+
.pipe(mirrorSaver)
67118
.pipe(zlib.createUnzip())
68119
.on("error", reject)
69120
.pipe(extractor);

‎src/lockfile/index.js‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,13 +24,18 @@ export { default as parse } from "./parse";
2424
export { default as stringify } from "./stringify";
2525

2626
export default class Lockfile {
27-
constructor(cache: ?Object, strict?: boolean) {
27+
constructor(cache: ?Object, strict?: boolean, save?: boolean) {
2828
this.strict = !!strict;
29+
this.save = !!save;
2930
this.cache = cache;
3031
}
3132

33+
// true if operation is just rehydrating node_modules folder
3234
strict: boolean;
3335

36+
// true if lockfile will be persisted
37+
save: boolean;
38+
3439
cache: ?{
3540
[key: string]: string | {
3641
name: string,
@@ -49,6 +54,7 @@ export default class Lockfile {
4954
dir: string,
5055
reporter: Reporter,
5156
strictIfPresent: boolean,
57+
save: boolean,
5258
): Promise<Lockfile> {
5359
// read the package.json in this directory
5460
let lockfileLoc = path.join(dir, constants.LOCKFILE_FILENAME);
@@ -68,7 +74,7 @@ export default class Lockfile {
6874
reporter.info("No lockfile found.");
6975
}
7076

71-
return new Lockfile(lockfile, strict);
77+
return new Lockfile(lockfile, strict, save);
7278
}
7379

7480
isStrict(): boolean {

‎src/package-fetcher.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ export default class PackageFetcher {
5555
await fs.mkdirp(dest);
5656

5757
try {
58-
let fetcher = new Fetcher(remote, this.config);
58+
let fetcher = new Fetcher(remote, this.config, ref.saveForOffline);
5959
return await fetcher.fetch(dest);
6060
} catch (err) {
6161
try {

‎src/package-reference.js‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ export default class PackageReference {
2222
request: PackageRequest,
2323
info: Manifest,
2424
remote: PackageRemote,
25+
saveForOffline: boolean,
2526
) {
2627
this.lockfile = request.rootLockfile;
2728
this.requests = [request];
@@ -40,6 +41,7 @@ export default class PackageReference {
4041
this.patterns = [];
4142
this.optional = null;
4243
this.location = null;
44+
this.saveForOffline = !!saveForOffline;
4345
}
4446

4547
requests: Array<PackageRequest>;
@@ -50,6 +52,7 @@ export default class PackageReference {
5052
version: string;
5153
uid: string;
5254
optional: ?boolean;
55+
saveForOffline: boolean;
5356
dependencies: Array<string>;
5457
patterns: Array<string>;
5558
permissions: { [key: string]: boolean };

‎src/package-request.js‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -252,7 +252,7 @@ export default class PackageRequest {
252252
invariant(remote, "Missing remote");
253253

254254
// set package reference
255-
let ref = new PackageReference(this, info, remote);
255+
let ref = new PackageReference(this, info, remote, this.resolver.lockfile.save);
256256

257257
// in order to support lockfiles inside transitive dependencies we need to block
258258
// resolution to fetch the package so we can peek inside of it for a fbkpm.lock
@@ -270,9 +270,19 @@ export default class PackageRequest {
270270
info.name,
271271
() => this.resolver.fetcher.fetch(ref)
272272
);
273+
let offlineMirrorPath = this.config.getOfflineMirrorPath(ref.remote.registry,
274+
ref.remote.reference);
275+
// replace resolved remote URL with local path
276+
if (this.resolver.lockfile.save && offlineMirrorPath) {
277+
if (await fs.exists(offlineMirrorPath)) {
278+
remote.resolved = path.relative(
279+
this.config.getOfflineMirrorPath(ref.remote.registry),
280+
offlineMirrorPath) + `#${ref.remote.hash}`;
281+
}
282+
}
283+
remote.hash = hash;
273284
newInfo.reference = ref;
274285
newInfo.remote = remote;
275-
remote.hash = hash;
276286
info = newInfo;
277287

278288
// find and load in fbkpm.lock from this module if it exists

‎src/registries/npm.js‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import Registry from "./_base.js";
1515
let userHome = require("user-home");
1616
let path = require("path");
1717
let _ = require("lodash");
18+
let ini = require("ini");
1819

1920
function getGlobalPrefix(): string {
2021
if (process.env.PREFIX) {
@@ -52,9 +53,13 @@ export default class NpmRegistry extends Registry {
5253

5354
for (let loc of possibles) {
5455
if (await fs.exists(loc)) {
55-
// TODO: merge it in!
56+
_.defaults(this.config, ini.parse(await fs.readFile(loc)));
5657
}
5758
}
59+
if (this.config["kpm-offline-mirror"]) {
60+
this.config["kpm-offline-mirror"] = path.resolve(this.cwd, this.config["kpm-offline-mirror"]);
61+
await fs.mkdirp(this.config["kpm-offline-mirror"]);
62+
}
5863

5964
_.defaults(this.config, {
6065
registry: "http://registry.npmjs.org"

0 commit comments

Comments
 (0)