Skip to content

Commit b7b4336

Browse files
Jimmy Gaussenmergify[bot]
Jimmy Gaussen
authored and
mergify[bot]
committed
fix(assets): support exceptions to exclude patterns (#4473)
* chore: better test * fix: folder exclusion * additional test cases * refactor existing tests * chore: add new test files * fix: listFilesRecursively refactor (wip) * fix: finish refactoring listFilesRecursively * fix: implement mkdirpSync * fix: symlink discovery * fix: don't follow symlinks early * fix: create empty directories * chore: remove useless let * fix: symlink fingerprinting * fix: don't throw when fingerprinting directories * chore: remove unneeded 'exclude' cloning * chore: refactor mkdirp calls * chore: refactor AssetFile * chore: refactor recursion * chore: prevent unneeded stats call * chore: createFsStructureFromTree, remove empty files * chore: cleanup * fix: empty-directory test * feat: shouldExcludeDeep * chore: fromTree in @/assert, cleanup fn, test * feat: shouldExcludeDirectory * chore: refactor listFiles with new methods (missing symlinks) * feat: add symlink support to fromTree * fix: fromTree external directory symlink * fix: listFiles symlink support * chore: additional contridactory test * chore: fix docs * chore: ExcludeRules class refactor * chore: evaluateFile refactor * chore: further evaluateFile refactor * chore: evaluateDirectory refactor * chore: move FsUtils to assets * chore: move FsUtils to assets (unstaged files)
1 parent 004dfd4 commit b7b4336

14 files changed

Lines changed: 1109 additions & 130 deletions

File tree

‎packages/@aws-cdk/assets/lib/fs/copy.ts‎

Lines changed: 17 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -2,56 +2,38 @@ import fs = require('fs');
22
import path = require('path');
33
import { CopyOptions } from './copy-options';
44
import { FollowMode } from './follow-mode';
5-
import { shouldExclude, shouldFollow } from './utils';
5+
import { mkdirpSync } from './mkdirpSync';
6+
import { listFilesRecursively, shouldFollow } from './utils';
67

78
export function copyDirectory(srcDir: string, destDir: string, options: CopyOptions = { }, rootDir?: string) {
89
const follow = options.follow !== undefined ? options.follow : FollowMode.EXTERNAL;
9-
const exclude = options.exclude || [];
1010

1111
rootDir = rootDir || srcDir;
1212

1313
if (!fs.statSync(srcDir).isDirectory()) {
1414
throw new Error(`${srcDir} is not a directory`);
1515
}
1616

17-
const files = fs.readdirSync(srcDir);
18-
for (const file of files) {
19-
const sourceFilePath = path.join(srcDir, file);
17+
for (const assetFile of listFilesRecursively(srcDir, {...options, follow}, rootDir)) {
18+
const filePath = assetFile.relativePath;
19+
const destFilePath = path.join(destDir, filePath);
2020

21-
if (shouldExclude(exclude, path.relative(rootDir, sourceFilePath))) {
22-
continue;
23-
}
24-
25-
const destFilePath = path.join(destDir, file);
26-
27-
let stat: fs.Stats | undefined = follow === FollowMode.ALWAYS
28-
? fs.statSync(sourceFilePath)
29-
: fs.lstatSync(sourceFilePath);
30-
31-
if (stat && stat.isSymbolicLink()) {
32-
const target = fs.readlinkSync(sourceFilePath);
21+
if (follow !== FollowMode.ALWAYS) {
22+
if (assetFile.isSymbolicLink) {
23+
const targetPath = path.normalize(path.resolve(srcDir, assetFile.symlinkTarget));
24+
if (!shouldFollow(follow, rootDir, targetPath)) {
25+
fs.symlinkSync(assetFile.symlinkTarget, destFilePath);
3326

34-
// determine if this is an external link (i.e. the target's absolute path
35-
// is outside of the root directory).
36-
const targetPath = path.normalize(path.resolve(srcDir, target));
37-
38-
if (shouldFollow(follow, rootDir, targetPath)) {
39-
stat = fs.statSync(sourceFilePath);
40-
} else {
41-
fs.symlinkSync(target, destFilePath);
42-
stat = undefined;
27+
continue;
28+
}
4329
}
4430
}
4531

46-
if (stat && stat.isDirectory()) {
47-
fs.mkdirSync(destFilePath);
48-
copyDirectory(sourceFilePath, destFilePath, options, rootDir);
49-
stat = undefined;
50-
}
51-
52-
if (stat && stat.isFile()) {
53-
fs.copyFileSync(sourceFilePath, destFilePath);
54-
stat = undefined;
32+
if (!assetFile.isDirectory) {
33+
mkdirpSync(path.dirname(destFilePath));
34+
fs.copyFileSync(assetFile.absolutePath, destFilePath);
35+
} else {
36+
mkdirpSync(destFilePath);
5537
}
5638
}
5739
}

‎packages/@aws-cdk/assets/lib/fs/fingerprint.ts‎

Lines changed: 16 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import fs = require('fs');
33
import path = require('path');
44
import { CopyOptions } from './copy-options';
55
import { FollowMode } from './follow-mode';
6-
import { shouldExclude, shouldFollow } from './utils';
6+
import { listFilesRecursively, shouldFollow } from './utils';
77

88
const BUFFER_SIZE = 8 * 1024;
99
const CTRL_SOH = '\x01';
@@ -38,40 +38,28 @@ export function fingerprint(fileOrDirectory: string, options: FingerprintOptions
3838
const rootDirectory = fs.statSync(fileOrDirectory).isDirectory()
3939
? fileOrDirectory
4040
: path.dirname(fileOrDirectory);
41-
const exclude = options.exclude || [];
42-
_processFileOrDirectory(fileOrDirectory);
4341

44-
return hash.digest('hex');
45-
46-
function _processFileOrDirectory(symbolicPath: string, realPath = symbolicPath) {
47-
if (shouldExclude(exclude, symbolicPath)) {
48-
return;
49-
}
50-
51-
const stat = fs.lstatSync(realPath);
52-
const relativePath = path.relative(fileOrDirectory, symbolicPath);
42+
for (const assetFile of listFilesRecursively(fileOrDirectory, {...options, follow}, rootDirectory)) {
43+
const relativePath = path.relative(fileOrDirectory, assetFile.absolutePath);
5344

54-
if (stat.isSymbolicLink()) {
55-
const linkTarget = fs.readlinkSync(realPath);
56-
const resolvedLinkTarget = path.resolve(path.dirname(realPath), linkTarget);
57-
if (shouldFollow(follow, rootDirectory, resolvedLinkTarget)) {
58-
_processFileOrDirectory(symbolicPath, resolvedLinkTarget);
45+
if (assetFile.isSymbolicLink) {
46+
const resolvedLinkTarget = path.resolve(path.dirname(assetFile.absolutePath), assetFile.symlinkTarget);
47+
if (!shouldFollow(follow, rootDirectory, resolvedLinkTarget)) {
48+
_hashField(hash, `link:${relativePath}`, assetFile.symlinkTarget);
5949
} else {
60-
_hashField(hash, `link:${relativePath}`, linkTarget);
50+
_hashField(hash, `file:${relativePath}`, _contentFingerprint(assetFile.absolutePath, assetFile.size));
6151
}
62-
} else if (stat.isFile()) {
63-
_hashField(hash, `file:${relativePath}`, _contentFingerprint(realPath, stat));
64-
} else if (stat.isDirectory()) {
65-
for (const item of fs.readdirSync(realPath).sort()) {
66-
_processFileOrDirectory(path.join(symbolicPath, item), path.join(realPath, item));
67-
}
68-
} else {
69-
throw new Error(`Unable to hash ${symbolicPath}: it is neither a file nor a directory`);
52+
} else if (assetFile.isFile) {
53+
_hashField(hash, `file:${relativePath}`, _contentFingerprint(assetFile.absolutePath, assetFile.size));
54+
} else if (!assetFile.isDirectory) {
55+
throw new Error(`Unable to hash ${assetFile.absolutePath}: it is neither a file nor a directory`);
7056
}
7157
}
58+
59+
return hash.digest('hex');
7260
}
7361

74-
function _contentFingerprint(file: string, stat: fs.Stats): string {
62+
function _contentFingerprint(file: string, size: number): string {
7563
const hash = crypto.createHash('sha256');
7664
const buffer = Buffer.alloc(BUFFER_SIZE);
7765
// tslint:disable-next-line: no-bitwise
@@ -85,7 +73,7 @@ function _contentFingerprint(file: string, stat: fs.Stats): string {
8573
} finally {
8674
fs.closeSync(fd);
8775
}
88-
return `${stat.size}:${hash.digest('hex')}`;
76+
return `${size}:${hash.digest('hex')}`;
8977
}
9078

9179
function _hashField(hash: crypto.Hash, header: string, value: string | Buffer | DataView) {

‎packages/@aws-cdk/assets/lib/fs/mkdirpSync.ts‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
// Slightly refactored version of fs-extra mkdirpSync
2+
// https://github.com/jprichardson/node-fs-extra/blob/d1a01e735e81688e08688557d7a254fa8297d98e/lib/mkdirs/mkdirs.js
3+
4+
import fs = require('fs');
5+
import path = require('path');
6+
7+
const INVALID_PATH_CHARS = /[<>:"|?*]/;
8+
const o777 = parseInt('0777', 8);
9+
10+
function getRootPath(p: string) {
11+
const paths = path.normalize(path.resolve(p)).split(path.sep);
12+
if (paths.length > 0) { return paths[0]; }
13+
return null;
14+
}
15+
16+
function invalidWin32Path(p: string) {
17+
const rp = getRootPath(p);
18+
p = p.replace(rp || '', '');
19+
return INVALID_PATH_CHARS.test(p);
20+
}
21+
22+
export function mkdirpSync(p: string, opts?: any, made?: any) {
23+
if (!opts || typeof opts !== 'object') {
24+
opts = { mode: opts };
25+
}
26+
27+
let mode = opts.mode;
28+
const xfs = opts.fs || fs;
29+
30+
if (process.platform === 'win32' && invalidWin32Path(p)) {
31+
const errInval = new Error(p + ' contains invalid WIN32 path characters.');
32+
// @ts-ignore
33+
errInval.code = 'EINVAL';
34+
throw errInval;
35+
}
36+
37+
if (mode === undefined) {
38+
// tslint:disable-next-line: no-bitwise
39+
mode = o777 & (~process.umask());
40+
}
41+
if (!made) { made = null; }
42+
43+
p = path.resolve(p);
44+
45+
try {
46+
xfs.mkdirSync(p, mode);
47+
made = made || p;
48+
} catch (err0) {
49+
if (err0.code === 'ENOENT') {
50+
if (path.dirname(p) === p) { throw err0; }
51+
made = mkdirpSync(path.dirname(p), opts, made);
52+
mkdirpSync(p, opts, made);
53+
} else {
54+
// In the case of any other error, just see if there's a dir there
55+
// already. If so, then hooray! If not, then something is borked.
56+
let stat;
57+
try {
58+
stat = xfs.statSync(p);
59+
} catch (err1) {
60+
throw err0;
61+
}
62+
if (!stat.isDirectory()) { throw err0; }
63+
}
64+
}
65+
66+
return made;
67+
}

0 commit comments

Comments
 (0)