[test optimization] Fix test.status reported in test sessions including quarantined tests (#7353)

juan-fernandez · web-flow · commit 5572149f187a · 2026-01-28T14:24:45.000+01:00
diff --git a/integration-tests/ci-visibility/test-management/test-partial-quarantine.js b/integration-tests/ci-visibility/test-management/test-partial-quarantine.js
@@ -0,0 +1,18 @@
+'use strict'
+
+const assert = require('assert')
+describe('partial quarantine tests', () => {
+  it('quarantined failing test', () => {
+    // This test fails but is quarantined, so it should not affect the session status
+    assert.strictEqual(1 + 2, 4)
+  })
+
+  it('non-quarantined failing test', () => {
+    // This test fails and is NOT quarantined, so it should cause the session to fail
+    assert.strictEqual(1 + 2, 5)
+  })
+
+  it('passing test', () => {
+    assert.strictEqual(1 + 2, 3)
+  })
+})
diff --git a/integration-tests/jest/jest.spec.js b/integration-tests/jest/jest.spec.js
@@ -2594,7 +2594,7 @@ describe(`jest@${JEST_VERSION} commonJS`, () => {
       })
     })
 
-    it('retries flaky tests and sets exit code to 0 as long as one attempt passes', (done) => {
+    it('retries flaky tests and sets exit code to 0 as long as one attempt passes', async () => {
       receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] })
       // Tests from ci-visibility/test/occasionally-failing-test will be considered new
       receiver.setKnownTests({ jest: {} })
@@ -2617,6 +2617,8 @@ describe(`jest@${JEST_VERSION} commonJS`, () => {
 
           const testSession = events.find(event => event.type === 'test_session_end').content
           assert.strictEqual(testSession.meta[TEST_EARLY_FLAKE_ENABLED], 'true')
+          // Session is passed because at least one retry of the new flaky test passes
+          assert.strictEqual(testSession.meta[TEST_STATUS], 'pass')
 
           const tests = events.filter(event => event.type === 'test').map(event => event.content)
 
@@ -2636,14 +2638,16 @@ describe(`jest@${JEST_VERSION} commonJS`, () => {
           })
         })
 
+      let testOutput = ''
       childProcess = exec(
         'node ./node_modules/jest/bin/jest --config config-jest.js',
         {
           cwd,
           env: {
             ...getCiVisEvpProxyConfig(receiver.port),
-            TESTS_TO_RUN: '**/ci-visibility/test-early-flake-detection/occasionally-failing-test*'
-          },
+            TESTS_TO_RUN: '**/ci-visibility/test-early-flake-detection/occasionally-failing-test*',
+            SHOULD_CHECK_RESULTS: '1'
+          }
         }
       )
 
@@ -2654,13 +2658,11 @@ describe(`jest@${JEST_VERSION} commonJS`, () => {
         testOutput += chunk.toString()
       })
 
-      childProcess.on('exit', (exitCode) => {
-        assert.match(testOutput, /2 failed, 2 passed/)
-        assert.strictEqual(exitCode, 0)
-        eventsPromise.then(() => {
-          done()
-        }).catch(done)
-      })
+      const [[exitCode]] = await Promise.all([once(childProcess, 'exit'), eventsPromise])
+
+      assert.match(testOutput, /2 failed, 2 passed/)
+      // Exit code is 0 because at least one retry of the new flaky test passes
+      assert.strictEqual(exitCode, 0)
     })
 
     // resetting snapshot state logic only works in latest versions
@@ -2694,6 +2696,8 @@ describe(`jest@${JEST_VERSION} commonJS`, () => {
 
           const testSession = events.find(event => event.type === 'test_session_end').content
           assert.strictEqual(testSession.meta[TEST_EARLY_FLAKE_ENABLED], 'true')
+          // Session is passed because at least one retry of each new flaky test passes
+          assert.strictEqual(testSession.meta[TEST_STATUS], 'pass')
 
           const tests = events.filter(event => event.type === 'test').map(event => event.content)
           // 6 tests, 4 of which are new: 4*(1 test + 3 retries) + 2*(1 test) = 18
@@ -2720,14 +2724,16 @@ describe(`jest@${JEST_VERSION} commonJS`, () => {
         env: {
           ...getCiVisEvpProxyConfig(receiver.port),
           TESTS_TO_RUN: 'ci-visibility/test-early-flake-detection/jest-snapshot',
-          CI: '1' // needs to be run as CI so snapshots are not written
-        },
+          CI: '1', // needs to be run as CI so snapshots are not written
+          SHOULD_CHECK_RESULTS: '1'
+        }
       })
 
       const [[exitCode]] = await Promise.all([
         once(childProcess, 'exit'),
         eventsPromise
       ])
+      // Exit code is 0 because at least one retry of each new flaky test passes
       assert.strictEqual(exitCode, 0)
     })
 
@@ -2757,6 +2763,8 @@ describe(`jest@${JEST_VERSION} commonJS`, () => {
 
           const testSession = events.find(event => event.type === 'test_session_end').content
           assert.strictEqual(testSession.meta[TEST_EARLY_FLAKE_ENABLED], 'true')
+          // Session is passed because at least one retry of the new flaky test passes
+          assert.strictEqual(testSession.meta[TEST_STATUS], 'pass')
 
           const tests = events.filter(event => event.type === 'test').map(event => event.content)
           // 1 new test
@@ -2779,14 +2787,16 @@ describe(`jest@${JEST_VERSION} commonJS`, () => {
         env: {
           ...getCiVisEvpProxyConfig(receiver.port),
           TESTS_TO_RUN: 'ci-visibility/test-early-flake-detection/jest-image-snapshot',
-          CI: '1'
-        },
+          CI: '1',
+          SHOULD_CHECK_RESULTS: '1'
+        }
       })
 
       const [[exitCode]] = await Promise.all([
         once(childProcess, 'exit'),
         eventsPromise
       ])
+      // Exit code is 0 because at least one retry of the new flaky test passes
       assert.strictEqual(exitCode, 0)
     })
 
@@ -4820,8 +4830,11 @@ describe(`jest@${JEST_VERSION} commonJS`, () => {
 
             if (isQuarantining) {
               assert.strictEqual(testSession.meta[TEST_MANAGEMENT_ENABLED], 'true')
+              // test session is passed even though a test fails because the test is quarantined
+              assert.strictEqual(testSession.meta[TEST_STATUS], 'pass')
             } else {
               assert.ok(!('TEST_MANAGEMENT_ENABLED' in testSession.meta))
+              assert.strictEqual(testSession.meta[TEST_STATUS], 'fail')
             }
 
             const resourceNames = tests.map(span => span.resource)
@@ -4855,7 +4868,7 @@ describe(`jest@${JEST_VERSION} commonJS`, () => {
             }
           })
 
-      const runQuarantineTest = (done, isQuarantining, extraEnvVars = {}, isParallel = false) => {
+      const runQuarantineTest = async (isQuarantining, extraEnvVars = {}, isParallel = false) => {
         let stdout = ''
         const testAssertionsPromise = getTestAssertions(isQuarantining, isParallel)
 
@@ -4868,7 +4881,7 @@ describe(`jest@${JEST_VERSION} commonJS`, () => {
               TESTS_TO_RUN: 'test-management/test-quarantine-1',
               SHOULD_CHECK_RESULTS: '1',
               ...extraEnvVars
-            },
+            }
           }
         )
 
@@ -4877,44 +4890,40 @@ describe(`jest@${JEST_VERSION} commonJS`, () => {
           stdout += chunk.toString()
         })
 
-        childProcess.on('exit', exitCode => {
-          testAssertionsPromise.then(() => {
-            // it runs regardless of quarantine status
-            assert.match(stdout, /I am running when quarantined/)
-            if (isQuarantining) {
-              // even though a test fails, the exit code is 0 because the test is quarantined
-              assert.strictEqual(exitCode, 0)
-            } else {
-              assert.strictEqual(exitCode, 1)
-            }
-            done()
-          }).catch(done)
-        })
+        const [[exitCode]] = await Promise.all([once(childProcess, 'exit'), testAssertionsPromise])
+
+        // it runs regardless of quarantine status
+        assert.match(stdout, /I am running when quarantined/)
+        if (isQuarantining) {
+          // even though a test fails, the exit code is 0 because the test is quarantined
+          assert.strictEqual(exitCode, 0)
+        } else {
+          assert.strictEqual(exitCode, 1)
+        }
       }
 
-      it('can quarantine tests', (done) => {
+      it('can quarantine tests', async () => {
         receiver.setSettings({ test_management: { enabled: true } })
 
-        runQuarantineTest(done, true)
+        await runQuarantineTest(true)
       })
 
-      it('fails if quarantine is not enabled', (done) => {
+      it('fails if quarantine is not enabled', async () => {
         receiver.setSettings({ test_management: { enabled: false } })
 
-        runQuarantineTest(done, false)
+        await runQuarantineTest(false)
       })
 
-      it('does not enable quarantine tests if DD_TEST_MANAGEMENT_ENABLED is set to false', (done) => {
+      it('does not enable quarantine tests if DD_TEST_MANAGEMENT_ENABLED is set to false', async () => {
         receiver.setSettings({ test_management: { enabled: true } })
 
-        runQuarantineTest(done, false, { DD_TEST_MANAGEMENT_ENABLED: '0' })
+        await runQuarantineTest(false, { DD_TEST_MANAGEMENT_ENABLED: '0' })
       })
 
-      it('can quarantine in parallel mode', (done) => {
+      it('can quarantine in parallel mode', async () => {
         receiver.setSettings({ test_management: { enabled: true } })
 
-        runQuarantineTest(
-          done,
+        await runQuarantineTest(
           true,
           {
             // we need to run more than 1 suite for parallel mode to kick in
@@ -4924,6 +4933,73 @@ describe(`jest@${JEST_VERSION} commonJS`, () => {
           true
         )
       })
+
+      it('fails if a non-quarantined test fails even when a quarantined test also fails', async () => {
+        receiver.setSettings({ test_management: { enabled: true } })
+
+        // Only quarantine one of the failing tests, leaving another failing test non-quarantined
+        receiver.setTestManagementTests({
+          jest: {
+            suites: {
+              'ci-visibility/test-management/test-partial-quarantine.js': {
+                tests: {
+                  'partial quarantine tests quarantined failing test': {
+                    properties: {
+                      quarantined: true
+                    }
+                  }
+                  // Note: 'partial quarantine tests non-quarantined failing test' is NOT quarantined
+                }
+              }
+            }
+          }
+        })
+
+        const testAssertionsPromise = receiver
+          .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => {
+            const events = payloads.flatMap(({ payload }) => payload.events)
+            const tests = events.filter(event => event.type === 'test').map(event => event.content)
+            const testSession = events.find(event => event.type === 'test_session_end').content
+
+            // Session should be marked as failed because a non-quarantined test failed
+            assert.strictEqual(testSession.meta[TEST_MANAGEMENT_ENABLED], 'true')
+            assert.strictEqual(testSession.meta[TEST_STATUS], 'fail')
+
+            // Verify the quarantined test has the quarantine tag
+            const quarantinedTest = tests.find(
+              test => test.meta[TEST_NAME] === 'partial quarantine tests quarantined failing test'
+            )
+            assert.strictEqual(quarantinedTest.meta[TEST_STATUS], 'fail')
+            assert.strictEqual(quarantinedTest.meta[TEST_MANAGEMENT_IS_QUARANTINED], 'true')
+
+            // Verify the non-quarantined test does NOT have the quarantine tag
+            const nonQuarantinedTest = tests.find(
+              test => test.meta[TEST_NAME] === 'partial quarantine tests non-quarantined failing test'
+            )
+            assert.strictEqual(nonQuarantinedTest.meta[TEST_STATUS], 'fail')
+            assert.ok(
+              !(TEST_MANAGEMENT_IS_QUARANTINED in nonQuarantinedTest.meta),
+              'Non-quarantined test should not have quarantine tag'
+            )
+          })
+
+        childProcess = exec(
+          runTestsCommand,
+          {
+            cwd,
+            env: {
+              ...getCiVisAgentlessConfig(receiver.port),
+              TESTS_TO_RUN: 'test-management/test-partial-quarantine',
+              SHOULD_CHECK_RESULTS: '1'
+            }
+          }
+        )
+
+        const [[exitCode]] = await Promise.all([once(childProcess, 'exit'), testAssertionsPromise])
+
+        // Exit code should be 1 because a non-quarantined test failed
+        assert.strictEqual(exitCode, 1)
+      })
     })
 
     it('does not crash if the request to get test management tests fails', async () => {
diff --git a/packages/datadog-instrumentations/src/jest.js b/packages/datadog-instrumentations/src/jest.js
