[test optimization] Fix quarantined tests being skipped rather than ignored in cypress (#7442)

juan-fernandez · web-flow · commit d5e899b9d1bc · 2026-02-05T13:02:41.000+01:00
diff --git a/integration-tests/cypress/cypress.spec.js b/integration-tests/cypress/cypress.spec.js
@@ -2700,15 +2700,20 @@ moduleTypes.forEach(({
 
               if (isQuarantining) {
                 assert.strictEqual(testSession.meta[TEST_MANAGEMENT_ENABLED], 'true')
+                // Session status should be 'pass' because Cypress sees the quarantined test as passed
+                assert.strictEqual(testSession.meta[TEST_STATUS], 'pass')
               } else {
                 assert.ok(!(TEST_MANAGEMENT_ENABLED in testSession.meta))
+                assert.strictEqual(testSession.meta[TEST_STATUS], 'fail')
               }
 
               assert.strictEqual(failedTest.resource, 'cypress/e2e/quarantine.js.quarantine is quarantined')
 
               if (isQuarantining) {
-                // TODO: run instead of skipping, but ignore its result
-                assert.strictEqual(failedTest.meta[TEST_STATUS], 'skip')
+                // Quarantined tests run normally but their failures are suppressed by Cypress.on('fail')
+                // in support.js. The test actually fails (reports 'fail' to Datadog) but Cypress sees
+                // it as passed, so the exit code is 0.
+                assert.strictEqual(failedTest.meta[TEST_STATUS], 'fail')
                 assert.strictEqual(failedTest.meta[TEST_MANAGEMENT_IS_QUARANTINED], 'true')
               } else {
                 assert.strictEqual(failedTest.meta[TEST_STATUS], 'fail')
diff --git a/packages/datadog-plugin-cypress/src/cypress-plugin.js b/packages/datadog-plugin-cypress/src/cypress-plugin.js
@@ -769,8 +769,10 @@ class CypressPlugin {
         if (cypressTest.displayError) {
           latestError = new Error(cypressTest.displayError)
         }
-        // Update test status
-        if (cypressTestStatus !== finishedTest.testStatus) {
+        // Update test status - but NOT for quarantined tests where we intentionally
+        // report 'fail' to Datadog even though Cypress sees it as 'pass'
+        const isQuarantinedTest = finishedTest.testSpan?.context()?._tags?.[TEST_MANAGEMENT_IS_QUARANTINED] === 'true'
+        if (cypressTestStatus !== finishedTest.testStatus && !isQuarantinedTest) {
           finishedTest.testSpan.setTag(TEST_STATUS, cypressTestStatus)
           finishedTest.testSpan.setTag('error', latestError)
         }
@@ -844,11 +846,12 @@ class CypressPlugin {
           return { shouldSkip: true }
         }
 
-        // TODO: I haven't found a way to trick cypress into ignoring a test
-        // The way we'll implement quarantine in cypress is by skipping the test altogether
-        if (!isAttemptToFix && (isDisabled || isQuarantined)) {
+        // For disabled tests (not attemptToFix), skip them
+        if (!isAttemptToFix && isDisabled) {
           return { shouldSkip: true }
         }
+        // Quarantined tests (not attemptToFix) run normally but their failures are caught
+        // by Cypress.on('fail') in support.js and suppressed, so Cypress sees them as passed
 
         if (!this.activeTestSpan) {
           this.activeTestSpan = this.getTestSpan({
@@ -880,6 +883,7 @@ class CypressPlugin {
           isEfdRetry,
           isAttemptToFix,
           isModified,
+          isQuarantined: isQuarantinedFromSupport,
         } = test
         if (coverage && this.isCodeCoverageEnabled && this.tracer._tracer._exporter?.exportCoverage) {
           const coverageFiles = getCoveredFilenamesFromCoverage(coverage)
@@ -952,6 +956,12 @@ class CypressPlugin {
           }
         }
 
+        // Ensure quarantined tests reported from support.js are tagged
+        // (This catches cases where the test ran but failed, but Cypress saw it as passed)
+        if (isQuarantinedFromSupport) {
+          this.activeTestSpan.setTag(TEST_MANAGEMENT_IS_QUARANTINED, 'true')
+        }
+
         const finishedTest = {
           testName,
           testStatus,
diff --git a/packages/datadog-plugin-cypress/src/support.js b/packages/datadog-plugin-cypress/src/support.js
@@ -12,6 +12,8 @@ let isModifiedTest = false
 let isTestIsolationEnabled = false
 // Array of test names that have been retried and the reason
 const retryReasonsByTestName = new Map()
+// Track quarantined test errors - we catch them in Cypress.on('fail') but need to report to Datadog
+const quarantinedTestErrors = new Map()
 
 // We need to grab the original window as soon as possible,
 // in case the test changes the origin. If the test does change the origin,
@@ -46,6 +48,30 @@ function getTestProperties (testName) {
   return { isAttemptToFix, isDisabled, isQuarantined }
 }
 
+// Catch test failures for quarantined tests and suppress them
+// By not re-throwing the error, Cypress marks the test as passed
+// This allows quarantined tests to run but not affect the exit code
+Cypress.on('fail', (err, runnable) => {
+  if (!isTestManagementEnabled) {
+    throw err
+  }
+
+  const testName = runnable.fullTitle()
+  const { isQuarantined, isAttemptToFix } = getTestProperties(testName)
+
+  // For pure quarantined tests (not attemptToFix), suppress the failure
+  // This makes the test "pass" from Cypress's perspective while we still track the error
+  if (isQuarantined && !isAttemptToFix) {
+    // Store the error so we can report it to Datadog in afterEach
+    quarantinedTestErrors.set(testName, err)
+    // Don't re-throw - this prevents Cypress from marking the test as failed
+    return
+  }
+
+  // For all other tests (including attemptToFix), let the error propagate normally
+  throw err
+})
+
 function getRetriedTests (test, numRetries, tags) {
   const retriedTests = []
   for (let retryIndex = 0; retryIndex < numRetries; retryIndex++) {
@@ -185,16 +211,31 @@ after(() => {
 
 afterEach(function () {
   const currentTest = Cypress.mocha.getRunner().suite.ctx.currentTest
+  const testName = currentTest.fullTitle()
+
+  // Check if this was a quarantined test that we suppressed the failure for
+  const quarantinedError = quarantinedTestErrors.get(testName)
+  const isQuarantinedTestThatFailed = !!quarantinedError
+
+  // For quarantined tests, convert Error to a serializable format for cy.task
+  const errorToReport = isQuarantinedTestThatFailed
+    ? { message: quarantinedError.message, stack: quarantinedError.stack }
+    : currentTest.err
+
   const testInfo = {
-    testName: currentTest.fullTitle(),
+    testName,
     testSuite: Cypress.mocha.getRootSuite().file,
     testSuiteAbsolutePath: Cypress.spec && Cypress.spec.absolute,
-    state: currentTest.state,
-    error: currentTest.err,
+    // For quarantined tests, report the actual state (failed) to Datadog, not what Cypress thinks (passed)
+    state: isQuarantinedTestThatFailed ? 'failed' : currentTest.state,
+    // For quarantined tests, include the actual error that was suppressed
+    error: errorToReport,
     isNew: currentTest._ddIsNew,
     isEfdRetry: currentTest._ddIsEfdRetry,
     isAttemptToFix: currentTest._ddIsAttemptToFix,
     isModified: currentTest._ddIsModified,
+    // Mark quarantined tests that failed so the plugin knows to tag them appropriately
+    isQuarantined: isQuarantinedTestThatFailed,
   }
   try {
     testInfo.testSourceLine = Cypress.mocha.getRunner().currentRunnable.invocationDetails.line
@@ -209,5 +250,11 @@ afterEach(function () {
   } catch {
     // ignore error and continue
   }
+
+  // Clean up the quarantined error tracking
+  if (isQuarantinedTestThatFailed) {
+    quarantinedTestErrors.delete(testName)
+  }
+
   cy.task('dd:afterEach', { test: testInfo, coverage })
 })
