77
88const FileSystemInfo = require ( "../FileSystemInfo" ) ;
99const ProgressPlugin = require ( "../ProgressPlugin" ) ;
10+ const { getReferencedFilenames } = require ( "../serialization/FileMiddleware" ) ;
1011const SerializerMiddleware = require ( "../serialization/SerializerMiddleware" ) ;
1112const LazySet = require ( "../util/LazySet" ) ;
1213const formatSize = require ( "../util/formatSize" ) ;
@@ -35,6 +36,10 @@ const {
3536/** @typedef {Set<string> } BuildDependencies */
3637/** @typedef {Map<string, PackItemInfo> } ItemInfo */
3738
39+ // Unreferenced files younger than this are kept to not race concurrent builds sharing the cache directory.
40+ // Kept below typical CI build durations so restored caches with refreshed modification times still get cleaned.
41+ const CLEANUP_GRACE_PERIOD = 30 * 60 * 1000 ;
42+
3843class PackContainer {
3944 /**
4045 * Creates an instance of PackContainer.
@@ -1166,6 +1171,8 @@ class PackFileCacheStrategy {
11661171 } ) ;
11671172 /** @type {Compiler } */
11681173 this . compiler = compiler ;
1174+ /** @type {IntermediateFileSystem } */
1175+ this . fs = fs ;
11691176 /** @type {string } */
11701177 this . context = context ;
11711178 /** @type {string } */
@@ -1182,6 +1189,10 @@ class PackFileCacheStrategy {
11821189 this . readonly = readonly ;
11831190 /** @type {boolean | undefined } */
11841191 this . allowCollectingMemory = allowCollectingMemory ;
1192+ // referenced names per on-disk file, versioned by mtime since concurrent
1193+ // processes may rewrite packs in place under the same name
1194+ /** @type {Map<string, { mtimeMs: number, referenced: string[] }> } */
1195+ this . _referencedFilesCache = new Map ( ) ;
11851196 /** @type {false | "gzip" | "brotli" | "zstd" | undefined } */
11861197 this . compression = compression ;
11871198 // eslint-disable-next-line n/no-unsupported-features/node-builtins
@@ -1588,10 +1599,17 @@ class PackFileCacheStrategy {
15881599 /** @type {Snapshot } */
15891600 ( this . resolveBuildDependenciesSnapshot )
15901601 ) ;
1602+ const cleanup = this . fs . unlink !== undefined ;
1603+ /** @type {Set<string> | undefined } */
1604+ const writtenFiles = cleanup ? new Set ( ) : undefined ;
1605+ /** @type {Set<string> | undefined } */
1606+ const retainedFiles = cleanup ? new Set ( ) : undefined ;
15911607 return this . fileSerializer
15921608 . serialize ( content , {
15931609 filename : `${ this . cacheLocation } /index${ this . _extension } ` ,
15941610 extension : `${ this . _extension } ` ,
1611+ writtenFiles,
1612+ retainedFiles,
15951613 logger : this . logger ,
15961614 profile : this . profile
15971615 } )
@@ -1608,9 +1626,17 @@ class PackFileCacheStrategy {
16081626 stats . count ,
16091627 Math . round ( stats . size / 1024 / 1024 )
16101628 ) ;
1629+ if ( writtenFiles !== undefined ) {
1630+ return this . _cleanupUnusedFiles (
1631+ writtenFiles ,
1632+ /** @type {Set<string> } */ ( retainedFiles )
1633+ ) ;
1634+ }
16111635 } )
16121636 . catch ( ( err ) => {
16131637 this . logger . timeEnd ( "store pack" ) ;
1638+ // files may be in an unknown state after a failed store
1639+ this . _referencedFilesCache . clear ( ) ;
16141640 this . logger . warn ( `Caching failed for pack: ${ err } ` ) ;
16151641 this . logger . debug ( err . stack ) ;
16161642 } ) ;
@@ -1622,6 +1648,109 @@ class PackFileCacheStrategy {
16221648 } ) ) ;
16231649 }
16241650
1651+ /**
1652+ * Deletes files from the cache directory that are no longer referenced by the
1653+ * stored pack. Retained files are walked on disk since nested lazy segments
1654+ * reference files not visible during serialization. Errors only log a warning.
1655+ * @param {Set<string> } writtenNames names (without extension) written by this store
1656+ * @param {Set<string> } retainedNames names (without extension) referenced but not rewritten
1657+ * @returns {Promise<void> } promise
1658+ */
1659+ async _cleanupUnusedFiles ( writtenNames , retainedNames ) {
1660+ this . logger . time ( "cleanup unused cache files" ) ;
1661+ const fs = this . fs ;
1662+ const extension = this . _extension ;
1663+ const cacheLocation = this . cacheLocation ;
1664+ const referencedFilesCache = this . _referencedFilesCache ;
1665+ try {
1666+ // rewritten files may reference different names now
1667+ for ( const name of writtenNames ) referencedFilesCache . delete ( name ) ;
1668+ /** @type {Set<string> } */
1669+ const liveFiles = new Set ( [ `index${ extension } ` ] ) ;
1670+ for ( const name of writtenNames ) liveFiles . add ( `${ name } ${ extension } ` ) ;
1671+ /** @type {string[] } */
1672+ const queue = [ ] ;
1673+ /**
1674+ * Marks a file live and queues it for walking its references.
1675+ * @param {string } name file name without extension
1676+ */
1677+ const enqueue = ( name ) => {
1678+ const file = `${ name } ${ extension } ` ;
1679+ if ( liveFiles . has ( file ) ) return ;
1680+ liveFiles . add ( file ) ;
1681+ queue . push ( name ) ;
1682+ } ;
1683+ for ( const name of retainedNames ) enqueue ( name ) ;
1684+ while ( queue . length > 0 ) {
1685+ const name = /** @type {string } */ ( queue . pop ( ) ) ;
1686+ const file = `${ cacheLocation } /${ name } ${ extension } ` ;
1687+ // an unchanged mtime proves the memo entry still matches the disk
1688+ const mtimeMs = await new Promise ( ( resolve , reject ) => {
1689+ fs . stat ( file , ( err , stats ) => {
1690+ if ( err ) return reject ( err ) ;
1691+ resolve (
1692+ /** @type {number } */ (
1693+ /** @type {import("../util/fs").IStats } */ ( stats ) . mtimeMs
1694+ )
1695+ ) ;
1696+ } ) ;
1697+ } ) ;
1698+ const entry = referencedFilesCache . get ( name ) ;
1699+ let referenced ;
1700+ if ( entry !== undefined && entry . mtimeMs === mtimeMs ) {
1701+ referenced = entry . referenced ;
1702+ } else {
1703+ referenced = await getReferencedFilenames ( fs , file ) ;
1704+ referencedFilesCache . set ( name , { mtimeMs, referenced } ) ;
1705+ }
1706+ for ( const referencedName of referenced ) enqueue ( referencedName ) ;
1707+ }
1708+ const files = await new Promise ( ( resolve , reject ) => {
1709+ fs . readdir ( cacheLocation , ( err , files ) => {
1710+ if ( err ) return reject ( err ) ;
1711+ resolve ( /** @type {string[] } */ ( files ) ) ;
1712+ } ) ;
1713+ } ) ;
1714+ const expireTime = Date . now ( ) - CLEANUP_GRACE_PERIOD ;
1715+ let deletedCount = 0 ;
1716+ for ( const file of files ) {
1717+ if ( typeof file !== "string" || liveFiles . has ( file ) ) continue ;
1718+ const path = `${ cacheLocation } /${ file } ` ;
1719+ const deleted = await new Promise ( ( resolve ) => {
1720+ fs . stat ( path , ( err , stats ) => {
1721+ if (
1722+ err ||
1723+ ! ( /** @type {import("../util/fs").IStats } */ ( stats ) . isFile ( ) ) ||
1724+ /** @type {number } */ (
1725+ /** @type {import("../util/fs").IStats } */ ( stats ) . mtimeMs
1726+ ) > expireTime
1727+ ) {
1728+ return resolve ( false ) ;
1729+ }
1730+ /** @type {NonNullable<IntermediateFileSystem["unlink"]> } */
1731+ ( fs . unlink ) ( path , ( err ) => resolve ( ! err ) ) ;
1732+ } ) ;
1733+ } ) ;
1734+ if ( deleted ) deletedCount ++ ;
1735+ }
1736+ // drop entries of files that are no longer alive
1737+ for ( const name of referencedFilesCache . keys ( ) ) {
1738+ if ( ! liveFiles . has ( `${ name } ${ extension } ` ) ) {
1739+ referencedFilesCache . delete ( name ) ;
1740+ }
1741+ }
1742+ if ( deletedCount > 0 ) {
1743+ this . logger . log ( "Deleted %d unused cache files" , deletedCount ) ;
1744+ }
1745+ } catch ( err ) {
1746+ this . logger . warn (
1747+ `Cleanup of unused cache files failed: ${ /** @type {Error } */ ( err ) } `
1748+ ) ;
1749+ this . logger . debug ( /** @type {Error } */ ( err ) . stack ) ;
1750+ }
1751+ this . logger . timeEnd ( "cleanup unused cache files" ) ;
1752+ }
1753+
16251754 clear ( ) {
16261755 this . fileSystemInfo . clear ( ) ;
16271756 this . buildDependencies . clear ( ) ;
0 commit comments