Skip to content

Commit 804902b

Browse files
juan-fernandezjuan-fernandez
authored
[test optimization] Add missing features to cucumber parallel mode (#7787)
1 parent be1f35c commit 804902b

File tree

10 files changed

+237
-22
lines changed

10 files changed

+237
-22
lines changed

‎integration-tests/ci-visibility/features-test-management-parallel/disabled.feature‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Feature: Disabled Parallel
2+
Scenario: Say disabled parallel
3+
When the greeter says disabled parallel
4+
Then I should have heard "disabled parallel"

‎integration-tests/ci-visibility/features-test-management-parallel/passing.feature‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Feature: Passing Parallel
2+
Scenario: Say passing parallel
3+
When the greeter says passing parallel
4+
Then I should have heard "passing parallel"

‎integration-tests/ci-visibility/features-test-management-parallel/quarantine.feature‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Feature: Quarantine Parallel
2+
Scenario: Say quarantine parallel
3+
When the greeter says quarantine parallel
4+
Then I should have heard "fail"

‎integration-tests/ci-visibility/features-test-management-parallel/support/steps.js‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
'use strict'
2+
3+
const assert = require('assert')
4+
const { When, Then } = require('@cucumber/cucumber')
5+
6+
Then('I should have heard {string}', function (expectedResponse) {
7+
assert.equal(this.whatIHeard, expectedResponse)
8+
})
9+
10+
When('the greeter says disabled parallel', function () {
11+
// eslint-disable-next-line no-console
12+
console.log('I am running disabled parallel')
13+
// expected to fail if not disabled
14+
this.whatIHeard = 'disabld parallel'
15+
})
16+
17+
When('the greeter says passing parallel', function () {
18+
this.whatIHeard = 'passing parallel'
19+
})
20+
21+
When('the greeter says quarantine parallel', function () {
22+
// eslint-disable-next-line no-console
23+
console.log('I am running quarantine parallel')
24+
// Will always fail the Then step — quarantined tests should not affect exit code
25+
this.whatIHeard = 'quarantine parallel'
26+
})

‎integration-tests/cucumber/cucumber.spec.js‎

Lines changed: 169 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1088,6 +1088,61 @@ describe(`cucumber@${version} commonJS`, () => {
10881088
}).catch(done)
10891089
})
10901090
})
1091+
1092+
onlyLatestIt('can skip suites in parallel mode', async () => {
1093+
receiver.setSuitesToSkip([{
1094+
type: 'suite',
1095+
attributes: {
1096+
suite: `${featuresPath}farewell.feature`,
1097+
},
1098+
}])
1099+
1100+
const eventsPromise = receiver
1101+
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => {
1102+
const events = payloads.flatMap(({ payload }) => payload.events)
1103+
const testSession = events.find(event => event.type === 'test_session_end').content
1104+
assert.strictEqual(testSession.meta[CUCUMBER_IS_PARALLEL], 'true')
1105+
assert.strictEqual(testSession.meta[TEST_ITR_SKIPPING_ENABLED], 'true')
1106+
assert.strictEqual(testSession.meta[TEST_ITR_TESTS_SKIPPED], 'true')
1107+
assert.strictEqual(testSession.meta[TEST_ITR_SKIPPING_TYPE], 'suite')
1108+
assert.strictEqual(testSession.metrics[TEST_ITR_SKIPPING_COUNT], 1)
1109+
1110+
const suites = events.filter(event => event.type === 'test_suite_end').map(event => event.content)
1111+
assert.strictEqual(suites.length, 2)
1112+
1113+
const skippedSuite = suites.find(s =>
1114+
s.resource === `test_suite.${featuresPath}farewell.feature`
1115+
)
1116+
assert.strictEqual(skippedSuite.meta[TEST_STATUS], 'skip')
1117+
assert.strictEqual(skippedSuite.meta[TEST_SKIPPED_BY_ITR], 'true')
1118+
1119+
// greetings.feature ran (not skipped)
1120+
const runningSuite = suites.find(s =>
1121+
s.resource === `test_suite.${featuresPath}greetings.feature`
1122+
)
1123+
assert.ok(runningSuite)
1124+
assert.ok(!(TEST_SKIPPED_BY_ITR in runningSuite.meta))
1125+
1126+
// Only tests from the non-skipped suite ran
1127+
const tests = events.filter(event => event.type === 'test').map(event => event.content)
1128+
assert.ok(tests.length > 0)
1129+
tests.forEach(test => {
1130+
assert.ok(!test.meta[TEST_SUITE].includes('farewell'))
1131+
})
1132+
})
1133+
1134+
childProcess = exec(
1135+
parallelModeCommand,
1136+
{
1137+
cwd,
1138+
env: getCiVisAgentlessConfig(receiver.port),
1139+
}
1140+
)
1141+
await Promise.all([
1142+
eventsPromise,
1143+
once(childProcess, 'exit'),
1144+
])
1145+
})
10911146
})
10921147

10931148
context('early flake detection', () => {
@@ -2833,6 +2888,119 @@ describe(`cucumber@${version} commonJS`, () => {
28332888
])
28342889
assert.match(testOutput, /Test management tests could not be fetched/)
28352890
})
2891+
2892+
onlyLatestIt('can disable tests in parallel mode', async () => {
2893+
receiver.setSettings({ test_management: { enabled: true } })
2894+
receiver.setTestManagementTests({
2895+
cucumber: {
2896+
suites: {
2897+
'ci-visibility/features-test-management-parallel/disabled.feature': {
2898+
tests: {
2899+
'Say disabled parallel': {
2900+
properties: { disabled: true },
2901+
},
2902+
},
2903+
},
2904+
},
2905+
},
2906+
})
2907+
2908+
const eventsPromise = receiver
2909+
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => {
2910+
const events = payloads.flatMap(({ payload }) => payload.events)
2911+
const testSession = events.find(event => event.type === 'test_session_end').content
2912+
assert.strictEqual(testSession.meta[CUCUMBER_IS_PARALLEL], 'true')
2913+
assert.strictEqual(testSession.meta[TEST_MANAGEMENT_ENABLED], 'true')
2914+
assert.strictEqual(testSession.meta[TEST_STATUS], 'pass')
2915+
2916+
const tests = events.filter(event => event.type === 'test').map(event => event.content)
2917+
assert.strictEqual(tests.length, 2)
2918+
2919+
const disabledTest = tests.find(t => t.meta[TEST_NAME] === 'Say disabled parallel')
2920+
assert.strictEqual(disabledTest.meta[TEST_STATUS], 'skip')
2921+
assert.strictEqual(disabledTest.meta[TEST_MANAGEMENT_IS_DISABLED], 'true')
2922+
2923+
const passingTest = tests.find(t => t.meta[TEST_NAME] === 'Say passing parallel')
2924+
assert.strictEqual(passingTest.meta[TEST_STATUS], 'pass')
2925+
})
2926+
2927+
let exitCode
2928+
childProcess = exec(
2929+
'./node_modules/.bin/cucumber-js' +
2930+
' ci-visibility/features-test-management-parallel/disabled.feature' +
2931+
' ci-visibility/features-test-management-parallel/passing.feature' +
2932+
' --parallel 2',
2933+
{
2934+
cwd,
2935+
env: getCiVisAgentlessConfig(receiver.port),
2936+
}
2937+
)
2938+
2939+
childProcess.on('exit', (code) => { exitCode = code })
2940+
2941+
await Promise.all([
2942+
eventsPromise,
2943+
once(childProcess, 'exit'),
2944+
])
2945+
2946+
assert.strictEqual(exitCode, 0)
2947+
})
2948+
2949+
onlyLatestIt('can quarantine tests in parallel mode', async () => {
2950+
receiver.setSettings({ test_management: { enabled: true } })
2951+
receiver.setTestManagementTests({
2952+
cucumber: {
2953+
suites: {
2954+
'ci-visibility/features-test-management-parallel/quarantine.feature': {
2955+
tests: {
2956+
'Say quarantine parallel': {
2957+
properties: { quarantined: true },
2958+
},
2959+
},
2960+
},
2961+
},
2962+
},
2963+
})
2964+
2965+
let exitCode
2966+
const eventsPromise = receiver
2967+
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => {
2968+
const events = payloads.flatMap(({ payload }) => payload.events)
2969+
const testSession = events.find(event => event.type === 'test_session_end').content
2970+
assert.strictEqual(testSession.meta[CUCUMBER_IS_PARALLEL], 'true')
2971+
assert.strictEqual(testSession.meta[TEST_MANAGEMENT_ENABLED], 'true')
2972+
assert.strictEqual(testSession.meta[TEST_STATUS], 'pass')
2973+
2974+
const tests = events.filter(event => event.type === 'test').map(event => event.content)
2975+
assert.strictEqual(tests.length, 2)
2976+
2977+
const quarantinedTest = tests.find(t => t.meta[TEST_NAME] === 'Say quarantine parallel')
2978+
assert.strictEqual(quarantinedTest.meta[TEST_STATUS], 'fail')
2979+
assert.strictEqual(quarantinedTest.meta[TEST_MANAGEMENT_IS_QUARANTINED], 'true')
2980+
2981+
const passingTest = tests.find(t => t.meta[TEST_NAME] === 'Say passing parallel')
2982+
assert.strictEqual(passingTest.meta[TEST_STATUS], 'pass')
2983+
})
2984+
2985+
childProcess = exec(
2986+
'./node_modules/.bin/cucumber-js ci-visibility/features-test-management-parallel/quarantine.feature' +
2987+
' ci-visibility/features-test-management-parallel/passing.feature --parallel 2',
2988+
{
2989+
cwd,
2990+
env: getCiVisAgentlessConfig(receiver.port),
2991+
}
2992+
)
2993+
2994+
childProcess.on('exit', (code) => { exitCode = code })
2995+
2996+
await Promise.all([
2997+
eventsPromise,
2998+
once(childProcess, 'exit'),
2999+
])
3000+
3001+
// Quarantined test fails but exit code should be 0
3002+
assert.strictEqual(exitCode, 0)
3003+
})
28363004
})
28373005

28383006
context('libraries capabilities', () => {
@@ -2852,11 +3020,7 @@ describe(`cucumber@${version} commonJS`, () => {
28523020

28533021
assert.ok(metadataDicts.length > 0)
28543022
metadataDicts.forEach(metadata => {
2855-
if (runMode === 'parallel') {
2856-
assert.strictEqual(metadata.test[DD_CAPABILITIES_TEST_IMPACT_ANALYSIS], undefined)
2857-
} else {
2858-
assert.strictEqual(metadata.test[DD_CAPABILITIES_TEST_IMPACT_ANALYSIS], '1')
2859-
}
3023+
assert.strictEqual(metadata.test[DD_CAPABILITIES_TEST_IMPACT_ANALYSIS], '1')
28603024
assert.strictEqual(metadata.test[DD_CAPABILITIES_EARLY_FLAKE_DETECTION], '1')
28613025
assert.strictEqual(metadata.test[DD_CAPABILITIES_AUTO_TEST_RETRIES], '1')
28623026
assert.strictEqual(metadata.test[DD_CAPABILITIES_IMPACTED_TESTS], '1')

‎packages/datadog-instrumentations/src/cucumber.js‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -166,9 +166,9 @@ function getErrorFromCucumberResult (cucumberResult) {
166166
return error
167167
}
168168

169-
function getChannelPromise (channelToPublishTo, isParallel = false, frameworkVersion = null) {
169+
function getChannelPromise (channelToPublishTo, frameworkVersion = null) {
170170
return new Promise(resolve => {
171-
channelToPublishTo.publish({ onDone: resolve, isParallel, frameworkVersion })
171+
channelToPublishTo.publish({ onDone: resolve, frameworkVersion })
172172
})
173173
}
174174

@@ -505,7 +505,7 @@ function getWrappedStart (start, frameworkVersion, isParallel = false, isCoordin
505505
}
506506
let errorSkippableRequest
507507

508-
const configurationResponse = await getChannelPromise(libraryConfigurationCh, isParallel, frameworkVersion)
508+
const configurationResponse = await getChannelPromise(libraryConfigurationCh, frameworkVersion)
509509

510510
isEarlyFlakeDetectionEnabled = configurationResponse.libraryConfig?.isEarlyFlakeDetectionEnabled
511511
earlyFlakeDetectionNumRetries = configurationResponse.libraryConfig?.earlyFlakeDetectionNumRetries
@@ -681,6 +681,7 @@ function getWrappedRunTestCase (runTestCaseFunction, isNewerCucumberVersion = fa
681681
let isQuarantined = false
682682
let isModified = false
683683

684+
const originalDryRun = this.options.dryRun
684685
if (isTestManagementTestsEnabled) {
685686
const testProperties = getTestProperties(testSuitePath, pickle.name)
686687
isAttemptToFix = testProperties.attemptToFix
@@ -719,6 +720,9 @@ function getWrappedRunTestCase (runTestCaseFunction, isNewerCucumberVersion = fa
719720
// TODO: for >=11 we could use `runTestCaseResult` instead of accumulating results in `lastStatusByPickleId`
720721
let runTestCaseResult = await runTestCaseFunction.apply(this, arguments)
721722

723+
// Restore dryRun so it doesn't affect subsequent tests in the same worker
724+
this.options.dryRun = originalDryRun
725+
722726
const testStatuses = lastStatusByPickleId.get(pickle.id)
723727
const lastTestStatus = testStatuses.at(-1)
724728

@@ -1053,6 +1057,12 @@ addHook({
10531057
this.options.worldParameters._ddIsFlakyTestRetriesEnabled = isFlakyTestRetriesEnabled
10541058
this.options.worldParameters._ddNumTestRetries = numTestRetries
10551059

1060+
if (isTestManagementTestsEnabled) {
1061+
this.options.worldParameters._ddIsTestManagementTestsEnabled = true
1062+
this.options.worldParameters._ddTestManagementTests = testManagementTests
1063+
this.options.worldParameters._ddTestManagementAttemptToFixRetries = testManagementAttemptToFixRetries
1064+
}
1065+
10561066
return startWorker.apply(this, arguments)
10571067
})
10581068
return adapterPackage
@@ -1090,6 +1100,11 @@ addHook({
10901100
}
10911101
isFlakyTestRetriesEnabled = !!this.options.worldParameters._ddIsFlakyTestRetriesEnabled
10921102
numTestRetries = this.options.worldParameters._ddNumTestRetries ?? 0
1103+
isTestManagementTestsEnabled = !!this.options.worldParameters._ddIsTestManagementTestsEnabled
1104+
if (isTestManagementTestsEnabled) {
1105+
testManagementTests = this.options.worldParameters._ddTestManagementTests
1106+
testManagementAttemptToFixRetries = this.options.worldParameters._ddTestManagementAttemptToFixRetries
1107+
}
10931108
}
10941109
)
10951110
return workerPackage

‎packages/datadog-instrumentations/src/mocha/main.js‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ function getFilteredSuites (originalSuites) {
101101
}, { suitesToRun: [], skippedSuites: new Set() })
102102
}
103103

104-
function getOnStartHandler (isParallel, frameworkVersion) {
104+
function getOnStartHandler (frameworkVersion) {
105105
return function () {
106106
const processArgv = process.argv.slice(2).join(' ')
107107
const command = `mocha ${processArgv}`
@@ -451,7 +451,7 @@ addHook({
451451

452452
const { suitesByTestFile, numSuitesByTestFile } = getSuitesByTestFile(this.suite)
453453

454-
this.once('start', getOnStartHandler(false, frameworkVersion))
454+
this.once('start', getOnStartHandler(frameworkVersion))
455455

456456
this.once('end', getOnEndHandler(false))
457457

@@ -622,7 +622,7 @@ addHook({
622622
return run.apply(this, arguments)
623623
}
624624

625-
this.once('start', getOnStartHandler(true, frameworkVersion))
625+
this.once('start', getOnStartHandler(frameworkVersion))
626626
this.once('end', getOnEndHandler(true))
627627

628628
// Populate unskippable suites before config is fetched (matches serial mode at Mocha.prototype.run)

‎packages/datadog-plugin-cypress/src/cypress-plugin.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -607,7 +607,7 @@ class CypressPlugin {
607607
[TEST_SESSION_NAME]: testSessionName,
608608
}
609609
}
610-
const libraryCapabilitiesTags = getLibraryCapabilitiesTags(this.constructor.id, false, this.frameworkVersion)
610+
const libraryCapabilitiesTags = getLibraryCapabilitiesTags(this.constructor.id, this.frameworkVersion)
611611
metadataTags.test = {
612612
...metadataTags.test,
613613
...libraryCapabilitiesTags,

‎packages/dd-trace/src/plugins/ci_plugin.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ module.exports = class CiPlugin extends Plugin {
125125
this._pendingRequestErrorTags = []
126126

127127
this.addSub(`ci:${this.constructor.id}:library-configuration`, (ctx) => {
128-
const { onDone, isParallel, frameworkVersion } = ctx
128+
const { onDone, frameworkVersion } = ctx
129129
ctx.currentStore = storage('legacy').getStore()
130130

131131
if (!this.tracer._exporter || !this.tracer._exporter.getLibraryConfiguration) {
@@ -143,7 +143,7 @@ module.exports = class CiPlugin extends Plugin {
143143
? getSessionRequestErrorTags(this.testSessionSpan)
144144
: Object.fromEntries(this._pendingRequestErrorTags.map(({ tag, value }) => [tag, value]))
145145

146-
const libraryCapabilitiesTags = getLibraryCapabilitiesTags(this.constructor.id, isParallel, frameworkVersion)
146+
const libraryCapabilitiesTags = getLibraryCapabilitiesTags(this.constructor.id, frameworkVersion)
147147
const metadataTags = {
148148
test: {
149149
...libraryCapabilitiesTags,

0 commit comments

Comments
 (0)