Skip to content

Commit 4c2e857

Browse files
juan-fernandezjuan-fernandez
authored
[test optimization] Set test.has_failed_all_retries to EFD (#7549)
1 parent 323fd0d commit 4c2e857

File tree

11 files changed

+312
-0
lines changed

11 files changed

+312
-0
lines changed

‎integration-tests/cypress/cypress.spec.js‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1792,6 +1792,85 @@ moduleTypes.forEach(({
17921792
])
17931793
assert.match(testOutput, /Retrying "other context fails" to detect flakes because it is new/)
17941794
})
1795+
1796+
it('sets TEST_HAS_FAILED_ALL_RETRIES when all EFD attempts fail', async () => {
1797+
receiver.setSettings({
1798+
early_flake_detection: {
1799+
enabled: true,
1800+
slow_test_retries: {
1801+
'5s': NUM_RETRIES_EFD,
1802+
},
1803+
},
1804+
known_tests_enabled: true,
1805+
})
1806+
1807+
receiver.setKnownTests({
1808+
cypress: {
1809+
'cypress/e2e/spec.cy.js': [
1810+
'context passes', // known test that passes
1811+
// 'other context fails' is new and will fail all attempts
1812+
],
1813+
},
1814+
})
1815+
1816+
const receiverPromise = receiver
1817+
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => {
1818+
const events = payloads.flatMap(({ payload }) => payload.events)
1819+
const tests = events.filter(event => event.type === 'test').map(event => event.content)
1820+
1821+
// 1 known test + 1 new test with retries: 1 + (1 + 3) = 5 tests
1822+
assert.strictEqual(tests.length, 5)
1823+
1824+
const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true')
1825+
assert.strictEqual(newTests.length, NUM_RETRIES_EFD + 1)
1826+
1827+
// Check that TEST_HAS_FAILED_ALL_RETRIES is only set on the last attempt
1828+
const testsWithFailedAllRetries = newTests.filter(
1829+
test => test.meta[TEST_HAS_FAILED_ALL_RETRIES] === 'true'
1830+
)
1831+
assert.strictEqual(
1832+
testsWithFailedAllRetries.length,
1833+
1,
1834+
'Exactly one test should have TEST_HAS_FAILED_ALL_RETRIES set'
1835+
)
1836+
1837+
// Check that it's set on the last attempt
1838+
const lastAttempt = newTests[newTests.length - 1]
1839+
assert.strictEqual(lastAttempt.meta[TEST_HAS_FAILED_ALL_RETRIES], 'true')
1840+
1841+
// Check that earlier attempts don't have the flag
1842+
for (let i = 0; i < newTests.length - 1; i++) {
1843+
assert.ok(!(TEST_HAS_FAILED_ALL_RETRIES in newTests[i].meta))
1844+
}
1845+
1846+
const testSession = events.find(event => event.type === 'test_session_end').content
1847+
assert.strictEqual(testSession.meta[TEST_EARLY_FLAKE_ENABLED], 'true')
1848+
}, 25000)
1849+
1850+
const {
1851+
NODE_OPTIONS, // NODE_OPTIONS dd-trace config does not work with cypress
1852+
...restEnvVars
1853+
} = getCiVisEvpProxyConfig(receiver.port)
1854+
1855+
const specToRun = 'cypress/e2e/spec.cy.js'
1856+
1857+
childProcess = exec(
1858+
version === 'latest' ? testCommand : `${testCommand} --spec ${specToRun}`,
1859+
{
1860+
cwd,
1861+
env: {
1862+
...restEnvVars,
1863+
CYPRESS_BASE_URL: `http://localhost:${webAppPort}`,
1864+
SPEC_PATTERN: specToRun,
1865+
},
1866+
}
1867+
)
1868+
1869+
await Promise.all([
1870+
once(childProcess, 'exit'),
1871+
receiverPromise,
1872+
])
1873+
})
17951874
})
17961875

17971876
context('flaky test retries', () => {

‎integration-tests/jest/jest.spec.js‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2417,6 +2417,73 @@ describe(`jest@${JEST_VERSION} commonJS`, () => {
24172417
})
24182418
})
24192419

2420+
it('sets TEST_HAS_FAILED_ALL_RETRIES when all EFD attempts fail', (done) => {
2421+
receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] })
2422+
// fail-test.js will be considered new and will always fail
2423+
receiver.setKnownTests({
2424+
jest: {},
2425+
})
2426+
const NUM_RETRIES_EFD = 3
2427+
receiver.setSettings({
2428+
early_flake_detection: {
2429+
enabled: true,
2430+
slow_test_retries: {
2431+
'5s': NUM_RETRIES_EFD,
2432+
},
2433+
faulty_session_threshold: 100,
2434+
},
2435+
known_tests_enabled: true,
2436+
})
2437+
const eventsPromise = receiver
2438+
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => {
2439+
const events = payloads.flatMap(({ payload }) => payload.events)
2440+
2441+
const tests = events.filter(event => event.type === 'test').map(event => event.content)
2442+
const failTests = tests.filter(test =>
2443+
test.meta[TEST_SUITE] === 'ci-visibility/test/fail-test.js'
2444+
)
2445+
2446+
// Should have 1 initial attempt + NUM_RETRIES_EFD retries
2447+
assert.strictEqual(failTests.length, NUM_RETRIES_EFD + 1)
2448+
2449+
// All attempts should be marked as new
2450+
failTests.forEach(test => {
2451+
assert.strictEqual(test.meta[TEST_IS_NEW], 'true')
2452+
assert.strictEqual(test.meta[TEST_STATUS], 'fail')
2453+
})
2454+
2455+
// Check retries
2456+
const retriedTests = failTests.filter(test => test.meta[TEST_IS_RETRY] === 'true')
2457+
assert.strictEqual(retriedTests.length, NUM_RETRIES_EFD)
2458+
retriedTests.forEach(test => {
2459+
assert.strictEqual(test.meta[TEST_RETRY_REASON], TEST_RETRY_REASON_TYPES.efd)
2460+
})
2461+
2462+
// Only the last retry should have TEST_HAS_FAILED_ALL_RETRIES set
2463+
const lastRetry = failTests[failTests.length - 1]
2464+
assert.strictEqual(lastRetry.meta[TEST_HAS_FAILED_ALL_RETRIES], 'true')
2465+
2466+
// Earlier attempts should not have the flag
2467+
for (let i = 0; i < failTests.length - 1; i++) {
2468+
assert.ok(!(TEST_HAS_FAILED_ALL_RETRIES in failTests[i].meta))
2469+
}
2470+
})
2471+
2472+
childProcess = exec(
2473+
runTestsCommand,
2474+
{
2475+
cwd,
2476+
env: { ...getCiVisEvpProxyConfig(receiver.port), TESTS_TO_RUN: 'test/fail-test' },
2477+
}
2478+
)
2479+
2480+
childProcess.on('exit', () => {
2481+
eventsPromise.then(() => {
2482+
done()
2483+
}).catch(done)
2484+
})
2485+
})
2486+
24202487
it('resets mock state between early flake detection retries', async () => {
24212488
receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] })
24222489
// Test is considered new (not in known tests)

‎integration-tests/mocha/mocha.spec.js‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2105,6 +2105,77 @@ describe(`mocha@${MOCHA_VERSION}`, function () {
21052105
})
21062106
})
21072107

2108+
it('sets TEST_HAS_FAILED_ALL_RETRIES when all EFD attempts fail', (done) => {
2109+
// fail-test.js will be considered new and will always fail
2110+
receiver.setKnownTests({
2111+
mocha: {},
2112+
})
2113+
const NUM_RETRIES_EFD = 3
2114+
receiver.setSettings({
2115+
early_flake_detection: {
2116+
enabled: true,
2117+
slow_test_retries: {
2118+
'5s': NUM_RETRIES_EFD,
2119+
},
2120+
faulty_session_threshold: 100,
2121+
},
2122+
known_tests_enabled: true,
2123+
})
2124+
const eventsPromise = receiver
2125+
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => {
2126+
const events = payloads.flatMap(({ payload }) => payload.events)
2127+
2128+
const tests = events.filter(event => event.type === 'test').map(event => event.content)
2129+
const failTests = tests.filter(test =>
2130+
test.meta[TEST_SUITE] === 'ci-visibility/test/fail-test.js'
2131+
)
2132+
2133+
// Should have 1 initial attempt + NUM_RETRIES_EFD retries
2134+
assert.strictEqual(failTests.length, NUM_RETRIES_EFD + 1)
2135+
2136+
// All attempts should be marked as new
2137+
failTests.forEach(test => {
2138+
assert.strictEqual(test.meta[TEST_IS_NEW], 'true')
2139+
assert.strictEqual(test.meta[TEST_STATUS], 'fail')
2140+
})
2141+
2142+
// Check retries
2143+
const retriedTests = failTests.filter(test => test.meta[TEST_IS_RETRY] === 'true')
2144+
assert.strictEqual(retriedTests.length, NUM_RETRIES_EFD)
2145+
retriedTests.forEach(test => {
2146+
assert.strictEqual(test.meta[TEST_RETRY_REASON], TEST_RETRY_REASON_TYPES.efd)
2147+
})
2148+
2149+
// Only the last retry should have TEST_HAS_FAILED_ALL_RETRIES set
2150+
const lastRetry = failTests[failTests.length - 1]
2151+
assert.strictEqual(lastRetry.meta[TEST_HAS_FAILED_ALL_RETRIES], 'true')
2152+
2153+
// Earlier attempts should not have the flag
2154+
for (let i = 0; i < failTests.length - 1; i++) {
2155+
assert.ok(!(TEST_HAS_FAILED_ALL_RETRIES in failTests[i].meta))
2156+
}
2157+
})
2158+
2159+
childProcess = exec(
2160+
runTestsCommand,
2161+
{
2162+
cwd,
2163+
env: {
2164+
...getCiVisAgentlessConfig(receiver.port),
2165+
TESTS_TO_RUN: JSON.stringify([
2166+
'./test/fail-test.js',
2167+
]),
2168+
},
2169+
}
2170+
)
2171+
2172+
childProcess.on('exit', () => {
2173+
eventsPromise.then(() => {
2174+
done()
2175+
}).catch(done)
2176+
})
2177+
})
2178+
21082179
it('handles parameterized tests as a single unit', (done) => {
21092180
// Tests from ci-visibility/test-early-flake-detection/test-parameterized.js will be considered new
21102181
receiver.setKnownTests({

‎integration-tests/playwright/playwright.spec.js‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -879,6 +879,16 @@ versions.forEach((version) => {
879879
assert.strictEqual(test.meta[TEST_RETRY_REASON], TEST_RETRY_REASON_TYPES.efd)
880880
assert.strictEqual(test.meta[TEST_STATUS], 'fail')
881881
})
882+
883+
// Only the last retry should have TEST_HAS_FAILED_ALL_RETRIES set
884+
const lastRetry = newTests[newTests.length - 1]
885+
assert.strictEqual(lastRetry.meta[TEST_HAS_FAILED_ALL_RETRIES], 'true')
886+
887+
// Earlier attempts should not have the flag
888+
for (let i = 0; i < newTests.length - 1; i++) {
889+
assert.ok(!(TEST_HAS_FAILED_ALL_RETRIES in newTests[i].meta))
890+
}
891+
882892
// --retries works normally for old flaky tests
883893
const oldFlakyTests = tests.filter(
884894
test => test.meta[TEST_NAME] === 'playwright should retry old flaky tests'
@@ -951,6 +961,16 @@ versions.forEach((version) => {
951961
assert.strictEqual(test.meta[TEST_RETRY_REASON], TEST_RETRY_REASON_TYPES.efd)
952962
assert.strictEqual(test.meta[TEST_STATUS], 'fail')
953963
})
964+
965+
// Only the last retry should have TEST_HAS_FAILED_ALL_RETRIES set
966+
const lastRetry = newTests[newTests.length - 1]
967+
assert.strictEqual(lastRetry.meta[TEST_HAS_FAILED_ALL_RETRIES], 'true')
968+
969+
// Earlier attempts should not have the flag
970+
for (let i = 0; i < newTests.length - 1; i++) {
971+
assert.ok(!(TEST_HAS_FAILED_ALL_RETRIES in newTests[i].meta))
972+
}
973+
954974
// ATR works normally for old flaky tests
955975
const oldFlakyTests = tests.filter(
956976
test => test.meta[TEST_NAME] === 'playwright should retry old flaky tests'

‎integration-tests/vitest/vitest.spec.js‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -713,6 +713,28 @@ versions.forEach((version) => {
713713
const testSessionEvent = events.find(event => event.type === 'test_session_end').content
714714
assert.strictEqual(testSessionEvent.meta[TEST_STATUS], 'fail')
715715
assert.strictEqual(testSessionEvent.meta[TEST_EARLY_FLAKE_ENABLED], 'true')
716+
717+
// Check that TEST_HAS_FAILED_ALL_RETRIES is set for tests that fail all EFD attempts
718+
const alwaysFailTests = tests.filter(test =>
719+
test.meta[TEST_NAME] === 'early flake detection can retry tests that always pass'
720+
)
721+
assert.strictEqual(alwaysFailTests.length, 4) // 1 initial + 3 retries
722+
// The last execution should have TEST_HAS_FAILED_ALL_RETRIES set
723+
const testsWithFlag = alwaysFailTests.filter(test =>
724+
test.meta[TEST_HAS_FAILED_ALL_RETRIES] === 'true'
725+
)
726+
assert.strictEqual(
727+
testsWithFlag.length,
728+
1,
729+
'Exactly one test should have TEST_HAS_FAILED_ALL_RETRIES set'
730+
)
731+
// It should be the last one
732+
const lastAttempt = alwaysFailTests[alwaysFailTests.length - 1]
733+
assert.strictEqual(
734+
lastAttempt.meta[TEST_HAS_FAILED_ALL_RETRIES],
735+
'true',
736+
'Last attempt should have the flag'
737+
)
716738
})
717739

718740
childProcess = exec(

‎packages/datadog-instrumentations/src/cucumber.js‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -353,6 +353,20 @@ function wrapRun (pl, isLatestVersion, version) {
353353
isEfdRetry = numRetries > 0
354354
}
355355

356+
// Check if all EFD retries failed
357+
if (isEfdRetry && (isNew || isModified)) {
358+
const statuses = lastStatusByPickleId.get(this.pickle.id)
359+
if (statuses.length === earlyFlakeDetectionNumRetries + 1) {
360+
const { fail } = statuses.reduce((acc, status) => {
361+
acc[status]++
362+
return acc
363+
}, { pass: 0, fail: 0 })
364+
if (fail === earlyFlakeDetectionNumRetries + 1) {
365+
hasFailedAllRetries = true
366+
}
367+
}
368+
}
369+
356370
const attemptCtx = numAttemptToCtx.get(numAttempt)
357371

358372
const error = getErrorFromCucumberResult(result)

‎packages/datadog-instrumentations/src/jest.js‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -604,6 +604,13 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) {
604604
} else {
605605
newTestsTestStatuses.set(testName, [status])
606606
}
607+
const testStatuses = newTestsTestStatuses.get(testName)
608+
// Check if this is the last EFD retry.
609+
// If it is, we'll set the failedAllTests flag to true if all the tests failed
610+
if (testStatuses.length === earlyFlakeDetectionNumRetries + 1 &&
611+
testStatuses.every(status => status === 'fail')) {
612+
failedAllTests = true
613+
}
607614
}
608615
}
609616

‎packages/datadog-instrumentations/src/mocha/utils.js‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -271,6 +271,7 @@ function getOnTestEndHandler (config) {
271271
const testStatuses = testsStatuses.get(testName)
272272

273273
const isLastAttempt = testStatuses.length === config.testManagementAttemptToFixRetries + 1
274+
const isLastEfdRetry = testStatuses.length === config.earlyFlakeDetectionNumRetries + 1
274275

275276
if (test._ddIsAttemptToFix && isLastAttempt) {
276277
if (testStatuses.includes('fail')) {
@@ -283,6 +284,11 @@ function getOnTestEndHandler (config) {
283284
}
284285
}
285286

287+
if (test._ddIsEfdRetry && isLastEfdRetry &&
288+
testStatuses.every(status => status === 'fail')) {
289+
hasFailedAllRetries = true
290+
}
291+
286292
const isAttemptToFixRetry = test._ddIsAttemptToFix && testStatuses.length > 1
287293
const isAtrRetry = config.isFlakyTestRetriesEnabled &&
288294
!test._ddIsAttemptToFix &&

‎packages/datadog-instrumentations/src/playwright.js‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -396,6 +396,14 @@ function testEndHandler ({
396396
}
397397
}
398398

399+
// Check if all EFD retries failed
400+
if (testStatuses.length === earlyFlakeDetectionNumRetries + 1 &&
401+
(test._ddIsNew || test._ddIsModified) &&
402+
test._ddIsEfdRetry &&
403+
testStatuses.every(status => status === 'fail')) {
404+
test._ddHasFailedAllRetries = true
405+
}
406+
399407
// this handles tests that do not go through the worker process (because they're skipped)
400408
if (shouldCreateTestSpan) {
401409
const testResult = results.at(-1)

‎packages/datadog-instrumentations/src/vitest.js‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1036,6 +1036,17 @@ addHook({
10361036
}
10371037
}
10381038

1039+
// Check if all EFD retries failed
1040+
const providedContext = getProvidedContext()
1041+
if (providedContext.isEarlyFlakeDetectionEnabled && (newTasks.has(task) || modifiedTasks.has(task))) {
1042+
const statuses = taskToStatuses.get(task)
1043+
// statuses only includes repetitions (not the initial run), so we check against numRepeats (not +1)
1044+
if (statuses && statuses.length === providedContext.numRepeats &&
1045+
statuses.every(status => status === 'fail')) {
1046+
hasFailedAllRetries = true
1047+
}
1048+
}
1049+
10391050
if (testCtx) {
10401051
const isRetry = task.result?.retryCount > 0
10411052
// `duration` is the duration of all the retries, so it can't be used if there are retries

0 commit comments

Comments
 (0)