Skip to content

Commit 964bbfa

Browse files
fix: delete no longer referenced files from the filesystem cache directory (#21528)
* fix: delete no longer referenced files from the filesystem cache directory * test: cover pack orphaning through entry churn and item expiry * test: make the corrupt-header rejection deterministic across environments * test: cover the full pack lifecycle including expiry of a formerly used pack * test: fix TS2774 in the zstd availability check * test: normalize spy keys so the memoization test passes on Windows * test: cover the cleanup abort on unreadable retained files * Update lib/serialization/FileMiddleware.js * Update lib/serialization/FileMiddleware.js * Update lib/serialization/FileMiddleware.js * Update test/FileMiddleware.unittest.js * Update test/FileMiddleware.unittest.js * Update test/FileMiddleware.unittest.js * Update test/FileMiddleware.unittest.js * test: fix stat-error case in FileMiddleware cleanup tests Restore the missing content binding and drop the duplicated fs declaration that a mis-anchored suggestion left in the stat-error case. * fix: version the referenced-files memo by mtime to survive in-place pack rewrites --------- Co-authored-by: Alexander Akait <[email protected]>
1 parent 075d0cd commit 964bbfa

6 files changed

Lines changed: 1046 additions & 60 deletions

File tree

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"webpack": patch
3+
---
4+
5+
Delete no longer referenced files from the filesystem cache directory after storing the cache.

‎lib/cache/PackFileCacheStrategy.js‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88
const FileSystemInfo = require("../FileSystemInfo");
99
const ProgressPlugin = require("../ProgressPlugin");
10+
const { getReferencedFilenames } = require("../serialization/FileMiddleware");
1011
const SerializerMiddleware = require("../serialization/SerializerMiddleware");
1112
const LazySet = require("../util/LazySet");
1213
const formatSize = require("../util/formatSize");
@@ -35,6 +36,10 @@ const {
3536
/** @typedef {Set<string>} BuildDependencies */
3637
/** @typedef {Map<string, PackItemInfo>} ItemInfo */
3738

39+
// Unreferenced files younger than this are kept to not race concurrent builds sharing the cache directory.
40+
// Kept below typical CI build durations so restored caches with refreshed modification times still get cleaned.
41+
const CLEANUP_GRACE_PERIOD = 30 * 60 * 1000;
42+
3843
class PackContainer {
3944
/**
4045
* Creates an instance of PackContainer.
@@ -1166,6 +1171,8 @@ class PackFileCacheStrategy {
11661171
});
11671172
/** @type {Compiler} */
11681173
this.compiler = compiler;
1174+
/** @type {IntermediateFileSystem} */
1175+
this.fs = fs;
11691176
/** @type {string} */
11701177
this.context = context;
11711178
/** @type {string} */
@@ -1182,6 +1189,10 @@ class PackFileCacheStrategy {
11821189
this.readonly = readonly;
11831190
/** @type {boolean | undefined} */
11841191
this.allowCollectingMemory = allowCollectingMemory;
1192+
// referenced names per on-disk file, versioned by mtime since concurrent
1193+
// processes may rewrite packs in place under the same name
1194+
/** @type {Map<string, { mtimeMs: number, referenced: string[] }>} */
1195+
this._referencedFilesCache = new Map();
11851196
/** @type {false | "gzip" | "brotli" | "zstd" | undefined} */
11861197
this.compression = compression;
11871198
// eslint-disable-next-line n/no-unsupported-features/node-builtins
@@ -1588,10 +1599,17 @@ class PackFileCacheStrategy {
15881599
/** @type {Snapshot} */
15891600
(this.resolveBuildDependenciesSnapshot)
15901601
);
1602+
const cleanup = this.fs.unlink !== undefined;
1603+
/** @type {Set<string> | undefined} */
1604+
const writtenFiles = cleanup ? new Set() : undefined;
1605+
/** @type {Set<string> | undefined} */
1606+
const retainedFiles = cleanup ? new Set() : undefined;
15911607
return this.fileSerializer
15921608
.serialize(content, {
15931609
filename: `${this.cacheLocation}/index${this._extension}`,
15941610
extension: `${this._extension}`,
1611+
writtenFiles,
1612+
retainedFiles,
15951613
logger: this.logger,
15961614
profile: this.profile
15971615
})
@@ -1608,9 +1626,17 @@ class PackFileCacheStrategy {
16081626
stats.count,
16091627
Math.round(stats.size / 1024 / 1024)
16101628
);
1629+
if (writtenFiles !== undefined) {
1630+
return this._cleanupUnusedFiles(
1631+
writtenFiles,
1632+
/** @type {Set<string>} */ (retainedFiles)
1633+
);
1634+
}
16111635
})
16121636
.catch((err) => {
16131637
this.logger.timeEnd("store pack");
1638+
// files may be in an unknown state after a failed store
1639+
this._referencedFilesCache.clear();
16141640
this.logger.warn(`Caching failed for pack: ${err}`);
16151641
this.logger.debug(err.stack);
16161642
});
@@ -1622,6 +1648,109 @@ class PackFileCacheStrategy {
16221648
}));
16231649
}
16241650

1651+
/**
1652+
* Deletes files from the cache directory that are no longer referenced by the
1653+
* stored pack. Retained files are walked on disk since nested lazy segments
1654+
* reference files not visible during serialization. Errors only log a warning.
1655+
* @param {Set<string>} writtenNames names (without extension) written by this store
1656+
* @param {Set<string>} retainedNames names (without extension) referenced but not rewritten
1657+
* @returns {Promise<void>} promise
1658+
*/
1659+
async _cleanupUnusedFiles(writtenNames, retainedNames) {
1660+
this.logger.time("cleanup unused cache files");
1661+
const fs = this.fs;
1662+
const extension = this._extension;
1663+
const cacheLocation = this.cacheLocation;
1664+
const referencedFilesCache = this._referencedFilesCache;
1665+
try {
1666+
// rewritten files may reference different names now
1667+
for (const name of writtenNames) referencedFilesCache.delete(name);
1668+
/** @type {Set<string>} */
1669+
const liveFiles = new Set([`index${extension}`]);
1670+
for (const name of writtenNames) liveFiles.add(`${name}${extension}`);
1671+
/** @type {string[]} */
1672+
const queue = [];
1673+
/**
1674+
* Marks a file live and queues it for walking its references.
1675+
* @param {string} name file name without extension
1676+
*/
1677+
const enqueue = (name) => {
1678+
const file = `${name}${extension}`;
1679+
if (liveFiles.has(file)) return;
1680+
liveFiles.add(file);
1681+
queue.push(name);
1682+
};
1683+
for (const name of retainedNames) enqueue(name);
1684+
while (queue.length > 0) {
1685+
const name = /** @type {string} */ (queue.pop());
1686+
const file = `${cacheLocation}/${name}${extension}`;
1687+
// an unchanged mtime proves the memo entry still matches the disk
1688+
const mtimeMs = await new Promise((resolve, reject) => {
1689+
fs.stat(file, (err, stats) => {
1690+
if (err) return reject(err);
1691+
resolve(
1692+
/** @type {number} */ (
1693+
/** @type {import("../util/fs").IStats} */ (stats).mtimeMs
1694+
)
1695+
);
1696+
});
1697+
});
1698+
const entry = referencedFilesCache.get(name);
1699+
let referenced;
1700+
if (entry !== undefined && entry.mtimeMs === mtimeMs) {
1701+
referenced = entry.referenced;
1702+
} else {
1703+
referenced = await getReferencedFilenames(fs, file);
1704+
referencedFilesCache.set(name, { mtimeMs, referenced });
1705+
}
1706+
for (const referencedName of referenced) enqueue(referencedName);
1707+
}
1708+
const files = await new Promise((resolve, reject) => {
1709+
fs.readdir(cacheLocation, (err, files) => {
1710+
if (err) return reject(err);
1711+
resolve(/** @type {string[]} */ (files));
1712+
});
1713+
});
1714+
const expireTime = Date.now() - CLEANUP_GRACE_PERIOD;
1715+
let deletedCount = 0;
1716+
for (const file of files) {
1717+
if (typeof file !== "string" || liveFiles.has(file)) continue;
1718+
const path = `${cacheLocation}/${file}`;
1719+
const deleted = await new Promise((resolve) => {
1720+
fs.stat(path, (err, stats) => {
1721+
if (
1722+
err ||
1723+
!(/** @type {import("../util/fs").IStats} */ (stats).isFile()) ||
1724+
/** @type {number} */ (
1725+
/** @type {import("../util/fs").IStats} */ (stats).mtimeMs
1726+
) > expireTime
1727+
) {
1728+
return resolve(false);
1729+
}
1730+
/** @type {NonNullable<IntermediateFileSystem["unlink"]>} */
1731+
(fs.unlink)(path, (err) => resolve(!err));
1732+
});
1733+
});
1734+
if (deleted) deletedCount++;
1735+
}
1736+
// drop entries of files that are no longer alive
1737+
for (const name of referencedFilesCache.keys()) {
1738+
if (!liveFiles.has(`${name}${extension}`)) {
1739+
referencedFilesCache.delete(name);
1740+
}
1741+
}
1742+
if (deletedCount > 0) {
1743+
this.logger.log("Deleted %d unused cache files", deletedCount);
1744+
}
1745+
} catch (err) {
1746+
this.logger.warn(
1747+
`Cleanup of unused cache files failed: ${/** @type {Error} */ (err)}`
1748+
);
1749+
this.logger.debug(/** @type {Error} */ (err).stack);
1750+
}
1751+
this.logger.timeEnd("cleanup unused cache files");
1752+
}
1753+
16251754
clear() {
16261755
this.fileSystemInfo.clear();
16271756
this.buildDependencies.clear();

0 commit comments

Comments
 (0)