perf(tests): cache dd-trace tarball for sandbox creation (#7351)

watson · web-flow · commit 62d65800534a · 2026-01-30T13:29:55.000Z
Parallel integration tests were bottlenecked by multiple processes
simultaneously packing dd-trace.tgz during sandbox creation. This
caused severe I/O contention.

Cache the tarball at the sandboxRoot level so it's only created
once per test run, then reused by all parallel workers.

Debugger integration tests performance improvement example:
- Sequential: 5:06
- Parallel (--jobs 2) before: 6:47 (slower due to contention)
- Parallel (--jobs 2) after: 3:02 (40% faster than sequential)
- System CPU overhead reduced from 424s to 164s.
diff --git a/integration-tests/helpers/index.js b/integration-tests/helpers/index.js
@@ -325,6 +325,68 @@ function execHelper (command, options) {
   }
 }
 
+/**
+ * Pack dd-trace into a tarball at the specified path.
+ *
+ * @param {string} tarballPath - The path where the tarball should be created
+ * @param {NodeJS.ProcessEnv} env - The environment to use for the pack command
+ */
+function packTarball (tarballPath, env) {
+  execHelper(`${BUN} pm pack --quiet --gzip-level 0 --filename ${tarballPath}`, { env })
+  log('Tarball packed successfully:', tarballPath)
+}
+
+/**
+ * Pack the tarball with file locking to coordinate between parallel workers.
+ * Only one worker will pack the tarball, others will wait for it to be ready.
+ *
+ * @param {string} tarballPath - The path where the tarball should be created
+ * @param {NodeJS.ProcessEnv} env - The environment to use for the pack command
+ * @returns {Promise<void>}
+ */
+async function packTarballWithLock (tarballPath, env) {
+  if (existsSync(tarballPath)) {
+    log('Tarball already exists:', tarballPath)
+    return
+  }
+
+  const lockFile = `${tarballPath}.lock`
+  let lockFd
+
+  try {
+    // Try to acquire the lock by creating the lock file exclusively
+    lockFd = await fs.open(lockFile, 'wx')
+    log('Lock acquired, packing tarball:', tarballPath)
+
+    // Double-check if tarball was created while we were acquiring the lock
+    if (existsSync(tarballPath)) {
+      log('Tarball already exists (created while waiting for lock):', tarballPath)
+      return
+    }
+
+    // We have the lock, pack the tarball
+    packTarball(tarballPath, env)
+  } catch (err) {
+    if (err.code === 'EEXIST') {
+      // Lock exists, another process is packing - wait for the tarball to appear
+      log('Lock file exists, waiting for tarball:', tarballPath)
+
+      while (!existsSync(tarballPath)) {
+        await new Promise(resolve => setTimeout(resolve, 100))
+      }
+
+      log('Tarball ready:', tarballPath)
+    } else {
+      throw err
+    }
+  } finally {
+    // Strictly no need to clean up the lock - it's in a temp directory
+    if (lockFd) {
+      lockFd.close().catch(() => {})
+    }
+  }
+}
+
 /**
  * @param {string[]} dependencies
  * @param {boolean} isGitRepo
@@ -365,15 +427,17 @@ async function createSandbox (
     return { folder: path.join(process.cwd(), 'integration-tests'), remove: async () => {} }
   }
   const folder = path.join(sandboxRoot, id().toString())
-  const out = path.join(sandboxRoot, 'dd-trace.tgz')
+  const tarballEnv = process.env.DD_TEST_SANDBOX_TARBALL_PATH
+  const out = tarballEnv && tarballEnv !== '0' && tarballEnv !== 'false'
+    ? tarballEnv
+    : path.join(sandboxRoot, 'dd-trace.tgz')
   const deps = cappedDependencies.concat(`file:${out}`)
 
   await fs.mkdir(folder, { recursive: true })
   const addOptions = { cwd: folder, env: restOfEnv }
   const addFlags = ['--trust']
-  if (!existsSync(out)) {
-    execHelper(`${BUN} pm pack --quiet --gzip-level 0 --filename ${out}`, { env: restOfEnv })
-  }
+
+  await packTarballWithLock(out, restOfEnv)
 
   if (process.env.OFFLINE === '1' || process.env.OFFLINE === 'true') {
     addFlags.push('--prefer-offline')
diff --git a/scripts/mocha-parallel-files.js b/scripts/mocha-parallel-files.js
@@ -362,8 +362,13 @@ async function main () {
           options.require = opts.require
         }
 
+        // Enable shared tarball caching for parallel runs to avoid redundant packing
+        const DD_TEST_SANDBOX_TARBALL_PATH = process.env.DD_TEST_SANDBOX_TARBALL_PATH ||
+          (jobs > 1 ? path.join(os.tmpdir(), 'dd-trace-integration-test.tgz') : undefined)
+
         const childEnv = {
           ...process.env,
+          DD_TEST_SANDBOX_TARBALL_PATH,
           MOCHA_RUN_FILE_CONFIG: JSON.stringify(options),
         }
 

Original file line number	Diff line number	Diff line change
`@@ -362,8 +362,13 @@ async function main () {`
`362`	`362`	`options.require = opts.require`
`363`	`363`	`}`
`364`	`364`
	`365`	`+ // Enable shared tarball caching for parallel runs to avoid redundant packing`
	`366`	`+ const DD_TEST_SANDBOX_TARBALL_PATH = process.env.DD_TEST_SANDBOX_TARBALL_PATH \|\|`
	`367`	`+ (jobs > 1 ? path.join(os.tmpdir(), 'dd-trace-integration-test.tgz') : undefined)`
	`368`	`+`
`365`	`369`	`const childEnv = {`
`366`	`370`	`...process.env,`
	`371`	`+ DD_TEST_SANDBOX_TARBALL_PATH,`
`367`	`372`	`MOCHA_RUN_FILE_CONFIG: JSON.stringify(options),`
`368`	`373`	`}`
`369`	`374`