Skip to content

Commit ed37d24

Browse files
fix(arborist): record the linked .store layout in the hidden lockfile (backport #9630) (#9642)
Backport of #9630 to `release/v11`. Under `install-strategy=linked`, the hidden lockfile `node_modules/.package-lock.json` recorded the hoisted logical layout instead of the on-disk `.store`/symlink layout, so it was rejected on every reload and never served as the actual-tree cache it is meant to be. ## How `reify.js` serializes the hidden lockfile from the isolated tree, which mirrors the on-disk layout, while `package-lock.json` still comes from the logical tree. `assertNoNewer()` additionally validates the store package node_modules and undeclared-workspace subtrees that the plain `node_modules` walk cannot reach under the linked strategy, gated so the hoisted strategy keeps its existing, stricter validation. The original commit's change to `test/arborist/reify-npm-extension.js` was dropped because the `.npm-extension` feature does not exist on `release/v11`. ## References Backport of #9630 Part of #9608
1 parent e601d4a commit ed37d24

3 files changed

Lines changed: 201 additions & 7 deletions

File tree

‎workspaces/arborist/lib/arborist/reify.js‎

Lines changed: 59 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,8 @@ const optionalSet = require('../optional-set.js')
2626
const relpath = require('../relpath.js')
2727
const retirePath = require('../retire-path.js')
2828
const treeCheck = require('../tree-check.js')
29-
const { defaultLockfileVersion } = require('../shrinkwrap.js')
29+
const Shrinkwrap = require('../shrinkwrap.js')
30+
const { defaultLockfileVersion } = Shrinkwrap
3031
const { saveTypeMap, hasSubKey } = require('../add-rm-pkg-deps.js')
3132
const { IsolatedNode, IsolatedLink } = require('../isolated-classes.js')
3233

@@ -115,12 +116,15 @@ module.exports = cls => class Reifier extends cls {
115116
await this[_loadTrees](options)
116117

117118
const oldTree = this.idealTree
119+
// Kept to serialize the hidden lockfile from the on-disk .store/symlink layout.
120+
let isolatedTree = null
118121
if (linked) {
119122
// swap out the tree with the isolated tree
120123
// this is currently technical debt which will be resolved in a refactor
121124
// of Node/Link trees
122125
log.warn('reify', 'The "linked" install strategy is EXPERIMENTAL and may contain bugs.')
123126
this.idealTree = await this.createIsolatedTree()
127+
isolatedTree = this.idealTree
124128
if (this.actualTree) {
125129
this.#linkedActualForDiff = this.#buildLinkedActualForDiff(
126130
this.idealTree, this.actualTree
@@ -241,17 +245,24 @@ module.exports = cls => class Reifier extends cls {
241245
calcDepFlags(this.idealTree)
242246
}
243247

244-
// save the ideal's meta as a hidden lockfile after we actualize it
245-
this.idealTree.meta.filename =
246-
this.idealTree.realpath + '/node_modules/.package-lock.json'
247-
this.idealTree.meta.hiddenLockfile = true
248-
this.idealTree.meta.lockfileVersion = defaultLockfileVersion
248+
// save the ideal's meta as a hidden lockfile after we actualize it.
249+
// Under linked the logical tree is the hoisted layout, so the hidden lockfile is serialized from the isolated tree instead.
250+
if (!linked) {
251+
this.idealTree.meta.filename =
252+
this.idealTree.realpath + '/node_modules/.package-lock.json'
253+
this.idealTree.meta.hiddenLockfile = true
254+
this.idealTree.meta.lockfileVersion = defaultLockfileVersion
255+
}
249256

250257
this.actualTree = this.idealTree
251258
this.idealTree = null
252259

253260
if (!this.options.global && !this.options.dryRun) {
254-
await this.actualTree.meta.save()
261+
if (linked) {
262+
await this.#saveLinkedHiddenLockfile(isolatedTree)
263+
} else {
264+
await this.actualTree.meta.save()
265+
}
255266
const ignoreScripts = !!this.options.ignoreScripts
256267
// if we aren't doing a dry run or ignoring scripts and we actually made changes to the dep
257268
// tree, then run the dependencies scripts
@@ -834,6 +845,47 @@ module.exports = cls => class Reifier extends cls {
834845
return join(filePath)
835846
}
836847

848+
// Serialize the hidden lockfile from the isolated tree, which mirrors the on-disk .store/symlink layout.
849+
// Its children are every materialized node_modules entry: store package dirs and all symlinks.
850+
async #saveLinkedHiddenLockfile (isolatedTree) {
851+
const path = isolatedTree.realpath
852+
const meta = new Shrinkwrap({
853+
path,
854+
hiddenLockfile: true,
855+
lockfileVersion: defaultLockfileVersion,
856+
resolveOptions: this.options,
857+
})
858+
meta.reset()
859+
meta.filename = resolve(path, 'node_modules/.package-lock.json')
860+
const storeRe = /^(.*\/\.store\/.+?)\/node_modules\//
861+
const containers = new Set()
862+
const nodes = new Set()
863+
for (const node of isolatedTree.children.values()) {
864+
// Tree-only undeclared workspace self-links aren't on disk.
865+
if (node.isUndeclaredWorkspaceLink) {
866+
continue
867+
}
868+
nodes.add(node)
869+
// Record the enclosing .store/<key> dir so loadVirtual can resolve a store package's sibling deps.
870+
// node.location uses the platform separator; lockfile keys are posix.
871+
const m = node.location.replace(/\\/g, '/').match(storeRe)
872+
if (m) {
873+
containers.add(m[1])
874+
}
875+
}
876+
// Workspace dirs hold their own dep symlinks; record them so the cache can validate those subtrees.
877+
for (const ws of isolatedTree.fsChildren) {
878+
nodes.add(ws)
879+
}
880+
for (const node of nodes) {
881+
meta.add(node)
882+
}
883+
for (const loc of containers) {
884+
meta.data.packages[loc] = {}
885+
}
886+
await meta.save()
887+
}
888+
837889
// Build a flat actual tree wrapper for linked installs so the diff can correctly match store entries that already exist on disk.
838890
// The proxy tree from createIsolatedTree() is flat (all children on root), but loadActual() produces a nested tree where store entries are deep link targets.
839891
// This wrapper surfaces them at the root level for comparison.

‎workspaces/arborist/lib/shrinkwrap.js‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,35 @@ const assertNoNewer = async (path, data, lockTime, dir, seen) => {
172172
return
173173
}
174174

175+
// The walk above can't reach two linked-strategy layouts: a store package's sibling deps under .store/<key>/node_modules (.store is a skipped dot-dir), and an undeclared workspace not symlinked into root node_modules. Walk those dirs, derived from the lockfile entries.
176+
// A dir reachable through a link entry is skipped: it was already reached (or, if the symlink is stale, correctly stays unseen so the lockfile is rejected). This keeps the hoisted strategy's stale-symlink detection intact, since there every workspace is a link target.
177+
const linkTargets = new Set()
178+
for (const loc in data.packages) {
179+
const { link, resolved } = data.packages[loc]
180+
if (link && resolved) {
181+
linkTargets.add(resolved.replace(/\\/g, '/'))
182+
}
183+
}
184+
const extraDirs = new Set()
185+
for (const loc in data.packages) {
186+
const store = loc.match(/^(.*\/\.store\/.+?)\/node_modules\//)
187+
if (store) {
188+
// .store/<key> is never walked but has no entry, so mark it seen.
189+
seen.add(store[1])
190+
extraDirs.add(`${store[1]}/node_modules`)
191+
continue
192+
}
193+
// A workspace/fsChild dir outside node_modules, e.g. packages/a.
194+
const i = loc.indexOf('/node_modules/')
195+
const root = i === -1 ? loc : loc.slice(0, i)
196+
if (root && !/(^|\/)node_modules(\/|$)/.test(root) && !linkTargets.has(root)) {
197+
extraDirs.add(root)
198+
}
199+
}
200+
for (const rel of extraDirs) {
201+
await assertNoNewer(path, data, lockTime, resolve(path, rel), seen)
202+
}
203+
175204
// assert that all the entries in the lockfile were seen
176205
for (const loc in data.packages) {
177206
if (!seen.has(loc)) {

‎workspaces/arborist/test/arborist/reify.js‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4333,6 +4333,119 @@ t.test('install strategy linked', async (t) => {
43334333
t.ok(abbrev.isSymbolicLink(), 'abbrev got installed')
43344334
})
43354335

4336+
t.test('hidden lockfile records the linked .store layout and round-trips', async t => {
4337+
// Regression for #9612: the hidden lockfile must record the on-disk .store/symlink layout so it round-trips as a valid cache.
4338+
const Shrinkwrap = require('../../lib/shrinkwrap.js')
4339+
const path = t.testdir({
4340+
'package.json': JSON.stringify({
4341+
name: 'root',
4342+
version: '1.0.0',
4343+
// once depends on wrappy, so the store has a transitive symlink to validate
4344+
dependencies: { once: '1.4.0' },
4345+
}),
4346+
})
4347+
4348+
createRegistry(t, true)
4349+
await reify(path, { installStrategy: 'linked' })
4350+
4351+
const hidden = require(resolve(path, 'node_modules/.package-lock.json'))
4352+
const locs = Object.keys(hidden.packages)
4353+
// the layout is recorded at .store paths, not hoisted node_modules/<name>
4354+
t.ok(locs.some(l => /^node_modules\/\.store\/once@/.test(l)),
4355+
'once is recorded under .store')
4356+
t.ok(locs.some(l => /^node_modules\/\.store\/.*\/node_modules\/wrappy$/.test(l)),
4357+
'the transitive wrappy symlink is recorded inside the store')
4358+
t.notOk(locs.includes('node_modules/wrappy'),
4359+
'wrappy is not recorded at a hoisted path')
4360+
4361+
// the cache is accepted on reload: assertNoNewer matches it against the real disk layout
4362+
const meta = await Shrinkwrap.load({ path, hiddenLockfile: true })
4363+
t.equal(meta.loadedFromDisk, true, 'hidden lockfile is a valid cache of the disk layout')
4364+
4365+
// loadActual must reconstruct the tree from the cache with once->wrappy resolved through the store.
4366+
const actual = await newArb({ path, installStrategy: 'linked' }).loadActual()
4367+
const onceNode = [...actual.inventory.values()].find(n => n.name === 'once' && !n.isLink)
4368+
t.ok(onceNode, 'once is in the cached actual tree')
4369+
const wrappyEdge = onceNode.edgesOut.get('wrappy')
4370+
t.ok(wrappyEdge && !wrappyEdge.missing, 'once resolves its wrappy dep through the cached store layout')
4371+
})
4372+
4373+
t.test('hidden lockfile round-trips with an undeclared workspace', async t => {
4374+
// Regression for #9612: an undeclared workspace materializes deps in its own node_modules but isn't linked into root, and the cache must still validate that subtree.
4375+
const Shrinkwrap = require('../../lib/shrinkwrap.js')
4376+
const path = t.testdir({
4377+
'package.json': JSON.stringify({
4378+
name: 'host',
4379+
version: '1.0.0',
4380+
workspaces: ['packages/a'],
4381+
// root does not depend on the workspace, so it stays undeclared
4382+
}),
4383+
packages: {
4384+
a: {
4385+
'package.json': JSON.stringify({
4386+
name: 'a',
4387+
version: '1.0.0',
4388+
dependencies: { once: '1.4.0' },
4389+
}),
4390+
},
4391+
},
4392+
})
4393+
4394+
createRegistry(t, true)
4395+
await reify(path, { installStrategy: 'linked' })
4396+
4397+
// the workspace's dep is materialized under its own node_modules, not the root's
4398+
t.ok(fs.lstatSync(resolve(path, 'packages/a/node_modules/once')).isSymbolicLink(),
4399+
'once is symlinked into the workspace node_modules')
4400+
t.notOk(fs.existsSync(resolve(path, 'node_modules/a')),
4401+
'the undeclared workspace is not symlinked into the root node_modules')
4402+
4403+
const meta = await Shrinkwrap.load({ path, hiddenLockfile: true })
4404+
t.equal(meta.loadedFromDisk, true, 'hidden lockfile validates the undeclared workspace subtree')
4405+
})
4406+
4407+
t.test('hidden lockfile round-trips with an undeclared workspace and no store entries', async t => {
4408+
// Regression for #9612: with only local deps there is no .store, so the cache must still walk the undeclared workspace subtree to validate it.
4409+
const Shrinkwrap = require('../../lib/shrinkwrap.js')
4410+
const path = t.testdir({
4411+
'package.json': JSON.stringify({
4412+
name: 'host',
4413+
version: '1.0.0',
4414+
workspaces: ['packages/w', 'packages/a', 'packages/b'],
4415+
// only w is declared; a and b stay undeclared, and a depends on b locally
4416+
dependencies: { w: '1.0.0' },
4417+
}),
4418+
packages: {
4419+
w: { 'package.json': JSON.stringify({ name: 'w', version: '1.0.0' }) },
4420+
a: {
4421+
'package.json': JSON.stringify({
4422+
name: 'a',
4423+
version: '1.0.0',
4424+
dependencies: { b: '1.0.0' },
4425+
}),
4426+
},
4427+
b: { 'package.json': JSON.stringify({ name: 'b', version: '1.0.0' }) },
4428+
},
4429+
})
4430+
4431+
createRegistry(t, false)
4432+
await reify(path, { installStrategy: 'linked' })
4433+
4434+
t.notOk(fs.existsSync(resolve(path, 'node_modules/.store')),
4435+
'no store is created for an all-local graph')
4436+
t.ok(fs.lstatSync(resolve(path, 'packages/a/node_modules/b')).isSymbolicLink(),
4437+
'the undeclared workspace links its local dep')
4438+
4439+
const meta = await Shrinkwrap.load({ path, hiddenLockfile: true })
4440+
t.equal(meta.loadedFromDisk, true, 'hidden lockfile validates the subtree without any store entry')
4441+
4442+
// loadActual must reconstruct the undeclared workspace from the cache with its local dep resolved.
4443+
const actual = await newArb({ path, installStrategy: 'linked' }).loadActual()
4444+
const aNode = [...actual.inventory.values()].find(n => n.name === 'a' && !n.isLink)
4445+
const bEdge = aNode && aNode.edgesOut.get('b')
4446+
t.ok(bEdge && !bEdge.missing, 'the undeclared workspace resolves its local dep through the cache')
4447+
})
4448+
43364449
t.test('does not re-create a workspace dir removed from manifest', async t => {
43374450
// Regression test for https://github.com/npm/cli/issues/9331
43384451
const path = t.testdir({

0 commit comments

Comments
 (0)