Skip to content

Commit f40634e

Browse files
juan-fernandezjuan-fernandez
authored
[test optimization] Add error tag for library settings request (#7590)
1 parent fa36eb1 commit f40634e

File tree

15 files changed

+333
-37
lines changed

15 files changed

+333
-37
lines changed

‎integration-tests/ci-visibility-intake.js‎

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ const DEFAULT_TEST_MANAGEMENT_TESTS = {}
4444
const DEFAULT_TEST_MANAGEMENT_TESTS_RESPONSE_STATUS = 200
4545

4646
let settings = DEFAULT_SETTINGS
47+
let settingsResponseStatusCode = 200
4748
let suitesToSkip = DEFAULT_SUITES_TO_SKIP
4849
let gitUploadStatus = DEFAULT_GIT_UPLOAD_STATUS
4950
let infoResponse = DEFAULT_INFO_RESPONSE
@@ -83,6 +84,10 @@ class FakeCiVisIntake extends FakeAgent {
8384
settings = newSettings
8485
}
8586

87+
setSettingsResponseCode (statusCode) {
88+
settingsResponseStatusCode = statusCode
89+
}
90+
8691
setWaitingTime (newWaitingTime) {
8792
waitingTime = newWaitingTime
8893
}
@@ -204,11 +209,16 @@ class FakeCiVisIntake extends FakeAgent {
204209
'/api/v2/libraries/tests/services/setting',
205210
'/evp_proxy/:version/api/v2/libraries/tests/services/setting',
206211
], (req, res) => {
207-
res.status(200).send(JSON.stringify({
208-
data: {
209-
attributes: settings,
210-
},
211-
}))
212+
res.status(settingsResponseStatusCode)
213+
if (settingsResponseStatusCode >= 200 && settingsResponseStatusCode < 300) {
214+
res.send(JSON.stringify({
215+
data: {
216+
attributes: settings,
217+
},
218+
}))
219+
} else {
220+
res.send(JSON.stringify({ errors: ['error'] }))
221+
}
212222
this.emit('message', {
213223
headers: req.headers,
214224
url: req.url,
@@ -312,6 +322,7 @@ class FakeCiVisIntake extends FakeAgent {
312322

313323
stop () {
314324
settings = DEFAULT_SETTINGS
325+
settingsResponseStatusCode = 200
315326
suitesToSkip = DEFAULT_SUITES_TO_SKIP
316327
gitUploadStatus = DEFAULT_GIT_UPLOAD_STATUS
317328
knownTestsStatusCode = DEFAULT_KNOWN_TESTS_RESPONSE_STATUS

‎integration-tests/cucumber/cucumber.spec.js‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ const {
7272
TEST_FRAMEWORK_VERSION,
7373
CI_APP_ORIGIN,
7474
TEST_SKIP_REASON,
75+
DD_CI_LIBRARY_CONFIGURATION_ERROR,
7576
} = require('../../packages/dd-trace/src/plugins/util/test')
7677
const { SAMPLING_PRIORITY } = require('../../ext/tags')
7778
const { AUTO_KEEP } = require('../../ext/priority')
@@ -326,6 +327,22 @@ describe(`cucumber@${version} commonJS`, () => {
326327
envVars = isAgentless ? getCiVisAgentlessConfig(receiver.port) : getCiVisEvpProxyConfig(receiver.port)
327328
logsEndpoint = isAgentless ? '/api/v2/logs' : '/debugger/v1/input'
328329
})
330+
331+
it('tags session and children with _dd.ci.library_configuration_error when settings fails 4xx', async () => {
332+
receiver.setSettingsResponseCode(404)
333+
const eventsPromise = receiver
334+
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => {
335+
const events = payloads.flatMap(({ payload }) => payload.events)
336+
const testSession = events.find(event => event.type === 'test_session_end').content
337+
assert.strictEqual(testSession.meta[DD_CI_LIBRARY_CONFIGURATION_ERROR], 'true')
338+
const testEvent = events.find(event => event.type === 'test')
339+
assert.ok(testEvent, 'should have test event')
340+
assert.strictEqual(testEvent.content.meta[DD_CI_LIBRARY_CONFIGURATION_ERROR], 'true')
341+
})
342+
childProcess = exec(runTestsCommand, { cwd, env: envVars })
343+
await Promise.all([eventsPromise, once(childProcess, 'exit')])
344+
})
345+
329346
const runModes = ['serial']
330347

331348
if (version !== '7.0.0') { // only on latest or 9 if node is old

‎integration-tests/cypress/cypress.spec.js‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ const {
6262
DD_CAPABILITIES_FAILED_TEST_REPLAY,
6363
TEST_RETRY_REASON_TYPES,
6464
TEST_IS_MODIFIED,
65+
DD_CI_LIBRARY_CONFIGURATION_ERROR,
6566
} = require('../../packages/dd-trace/src/plugins/util/test')
6667
const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env')
6768
const { ERROR_MESSAGE, ERROR_TYPE, COMPONENT } = require('../../packages/dd-trace/src/constants')
@@ -335,6 +336,40 @@ moduleTypes.forEach(({
335336
})
336337
}
337338

339+
it('tags session and children with _dd.ci.library_configuration_error when settings fails 4xx', async () => {
340+
const {
341+
NODE_OPTIONS, // NODE_OPTIONS dd-trace config does not work with cypress
342+
...restEnvVars
343+
} = getCiVisAgentlessConfig(receiver.port)
344+
345+
receiver.setSettingsResponseCode(404)
346+
const eventsPromise = receiver
347+
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => {
348+
const events = payloads.flatMap(({ payload }) => payload.events)
349+
const testSession = events.find(event => event.type === 'test_session_end').content
350+
assert.strictEqual(testSession.meta[DD_CI_LIBRARY_CONFIGURATION_ERROR], 'true',
351+
'test_session_end should have _dd.ci.library_configuration_error tag')
352+
const testEvent = events.find(event => event.type === 'test')
353+
assert.ok(testEvent, 'should have test event')
354+
assert.strictEqual(testEvent.content.meta[DD_CI_LIBRARY_CONFIGURATION_ERROR], 'true',
355+
'test event should have _dd.ci.library_configuration_error tag (from getSessionRequestErrorTags)')
356+
})
357+
358+
childProcess = exec(
359+
testCommand,
360+
{
361+
cwd,
362+
env: {
363+
...restEnvVars,
364+
CYPRESS_BASE_URL: `http://localhost:${webAppPort}`,
365+
SPEC_PATTERN: 'cypress/e2e/spec.cy.js',
366+
},
367+
}
368+
)
369+
370+
await Promise.all([eventsPromise, once(childProcess, 'exit')])
371+
})
372+
338373
it('does not crash if badly init', async () => {
339374
const {
340375
NODE_OPTIONS, // NODE_OPTIONS dd-trace config does not work with cypress

‎integration-tests/jest/jest.spec.js‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ const {
7676
TEST_FINAL_STATUS,
7777
GIT_COMMIT_SHA,
7878
GIT_REPOSITORY_URL,
79+
DD_CI_LIBRARY_CONFIGURATION_ERROR,
7980
} = require('../../packages/dd-trace/src/plugins/util/test')
8081
const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env')
8182
const { TELEMETRY_COVERAGE_UPLOAD } = require('../../packages/dd-trace/src/ci-visibility/telemetry')
@@ -2062,6 +2063,49 @@ describe(`jest@${JEST_VERSION} commonJS`, () => {
20622063
})
20632064
})
20642065

2066+
context('error tags', () => {
2067+
it('tags session and children with _dd.ci.library_configuration_error when settings fails 4xx', async () => {
2068+
receiver.setSettingsResponseCode(404)
2069+
const eventsPromise = receiver
2070+
.gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', (payloads) => {
2071+
const events = payloads.flatMap(({ payload }) => payload.events)
2072+
const testSession = events.find(event => event.type === 'test_session_end').content
2073+
assert.strictEqual(testSession.meta[DD_CI_LIBRARY_CONFIGURATION_ERROR], 'true')
2074+
const testEvent = events.find(event => event.type === 'test')
2075+
assert.ok(testEvent, 'should have test event')
2076+
assert.strictEqual(testEvent.content.meta[DD_CI_LIBRARY_CONFIGURATION_ERROR], 'true')
2077+
})
2078+
childProcess = exec(runTestsCommand, {
2079+
cwd,
2080+
env: getCiVisAgentlessConfig(receiver.port),
2081+
})
2082+
await Promise.all([eventsPromise, once(childProcess, 'exit')])
2083+
})
2084+
2085+
context('when jest is using workers to run tests in parallel', () => {
2086+
it('tags session and children with _dd.ci.library_configuration_error when settings fails 4xx', async () => {
2087+
receiver.setSettingsResponseCode(404)
2088+
const eventsPromise = receiver
2089+
.gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', (payloads) => {
2090+
const events = payloads.flatMap(({ payload }) => payload.events)
2091+
const testSession = events.find(event => event.type === 'test_session_end').content
2092+
assert.strictEqual(testSession.meta[DD_CI_LIBRARY_CONFIGURATION_ERROR], 'true')
2093+
const testEvent = events.find(event => event.type === 'test')
2094+
assert.ok(testEvent, 'should have test event')
2095+
assert.strictEqual(testEvent.content.meta[DD_CI_LIBRARY_CONFIGURATION_ERROR], 'true')
2096+
})
2097+
childProcess = exec(runTestsCommand, {
2098+
cwd,
2099+
env: {
2100+
...getCiVisAgentlessConfig(receiver.port),
2101+
RUN_IN_PARALLEL: 'true',
2102+
},
2103+
})
2104+
await Promise.all([eventsPromise, once(childProcess, 'exit')])
2105+
})
2106+
})
2107+
})
2108+
20652109
it('sets final_status tag to test status on regular tests without retry features', async () => {
20662110
receiver.setSettings({
20672111
itr_enabled: false,

‎integration-tests/mocha/mocha.spec.js‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ const {
6969
TEST_FRAMEWORK_VERSION,
7070
LIBRARY_VERSION,
7171
TEST_PARAMETERS,
72+
DD_CI_LIBRARY_CONFIGURATION_ERROR,
7273
} = require('../../packages/dd-trace/src/plugins/util/test')
7374
const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env')
7475
const {
@@ -2028,6 +2029,26 @@ describe(`mocha@${MOCHA_VERSION}`, function () {
20282029
})
20292030
})
20302031

2032+
context('error tags', () => {
2033+
it('tags session and children with _dd.ci.library_configuration_error when settings fails 4xx', async () => {
2034+
receiver.setSettingsResponseCode(404)
2035+
const eventsPromise = receiver
2036+
.gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', (payloads) => {
2037+
const events = payloads.flatMap(({ payload }) => payload.events)
2038+
const testSession = events.find(event => event.type === 'test_session_end').content
2039+
assert.strictEqual(testSession.meta[DD_CI_LIBRARY_CONFIGURATION_ERROR], 'true')
2040+
const testEvent = events.find(event => event.type === 'test')
2041+
assert.ok(testEvent, 'should have test event')
2042+
assert.strictEqual(testEvent.content.meta[DD_CI_LIBRARY_CONFIGURATION_ERROR], 'true')
2043+
})
2044+
childProcess = exec(runTestsCommand, {
2045+
cwd,
2046+
env: getCiVisAgentlessConfig(receiver.port),
2047+
})
2048+
await Promise.all([eventsPromise, once(childProcess, 'exit')])
2049+
})
2050+
})
2051+
20312052
context('early flake detection', () => {
20322053
it('retries new tests', (done) => {
20332054
// Tests from ci-visibility/test/ci-visibility-test-2.js will be considered new

‎integration-tests/playwright/playwright.spec.js‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ const {
5353
TEST_RETRY_REASON_TYPES,
5454
TEST_IS_MODIFIED,
5555
DD_CAPABILITIES_IMPACTED_TESTS,
56+
DD_CI_LIBRARY_CONFIGURATION_ERROR,
5657
} = require('../../packages/dd-trace/src/plugins/util/test')
5758
const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env')
5859
const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants')
@@ -133,6 +134,34 @@ versions.forEach((version) => {
133134

134135
reportMethods.forEach((reportMethod) => {
135136
context(`reporting via ${reportMethod}`, () => {
137+
it('tags session and children with _dd.ci.library_configuration_error when settings fail 4xx', async () => {
138+
const envVars = reportMethod === 'agentless'
139+
? getCiVisAgentlessConfig(receiver.port)
140+
: getCiVisEvpProxyConfig(receiver.port)
141+
receiver.setSettingsResponseCode(404)
142+
const eventsPromise = receiver
143+
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => {
144+
const events = payloads.flatMap(({ payload }) => payload.events)
145+
const testSession = events.find(event => event.type === 'test_session_end').content
146+
assert.strictEqual(testSession.meta[DD_CI_LIBRARY_CONFIGURATION_ERROR], 'true')
147+
const testEvent = events.find(event => event.type === 'test')
148+
assert.ok(testEvent, 'should have test event')
149+
assert.strictEqual(testEvent.content.meta[DD_CI_LIBRARY_CONFIGURATION_ERROR], 'true')
150+
})
151+
childProcess = exec(
152+
'./node_modules/.bin/playwright test -c playwright.config.js',
153+
{
154+
cwd,
155+
env: {
156+
...envVars,
157+
PW_BASE_URL: `http://localhost:${webAppPort}`,
158+
DD_TEST_SESSION_NAME: 'my-test-session',
159+
},
160+
}
161+
)
162+
await Promise.all([eventsPromise, once(childProcess, 'exit')])
163+
})
164+
136165
it('can run and report tests', (done) => {
137166
const envVars = reportMethod === 'agentless'
138167
? getCiVisAgentlessConfig(receiver.port)

‎integration-tests/vitest/vitest.spec.js‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ const {
5858
TEST_IS_TEST_FRAMEWORK_WORKER,
5959
GIT_COMMIT_SHA,
6060
GIT_REPOSITORY_URL,
61+
DD_CI_LIBRARY_CONFIGURATION_ERROR,
6162
} = require('../../packages/dd-trace/src/plugins/util/test')
6263
const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env')
6364
const { TELEMETRY_COVERAGE_UPLOAD } = require('../../packages/dd-trace/src/ci-visibility/telemetry')
@@ -242,6 +243,29 @@ versions.forEach((version) => {
242243
})
243244
})
244245

246+
context('error tags', () => {
247+
it('tags session and children with _dd.ci.library_configuration_error when settings fails', async () => {
248+
receiver.setSettingsResponseCode(404)
249+
const eventsPromise = receiver
250+
.gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', (payloads) => {
251+
const events = payloads.flatMap(({ payload }) => payload.events)
252+
const testSession = events.find(event => event.type === 'test_session_end').content
253+
assert.strictEqual(testSession.meta[DD_CI_LIBRARY_CONFIGURATION_ERROR], 'true')
254+
const testEvent = events.find(event => event.type === 'test')
255+
assert.ok(testEvent, 'should have test event')
256+
assert.strictEqual(testEvent.content.meta[DD_CI_LIBRARY_CONFIGURATION_ERROR], 'true')
257+
})
258+
childProcess = exec('./node_modules/.bin/vitest run', {
259+
cwd,
260+
env: {
261+
...getCiVisAgentlessConfig(receiver.port),
262+
NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init',
263+
},
264+
})
265+
await Promise.all([eventsPromise, once(childProcess, 'exit')])
266+
})
267+
})
268+
245269
it('sends telemetry with test_session metric when telemetry is enabled', async () => {
246270
const telemetryPromise = receiver
247271
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/apmtelemetry'), (payloads) => {

‎packages/datadog-plugin-cucumber/src/index.js‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -128,12 +128,15 @@ class CucumberPlugin extends CiPlugin {
128128
const testSuitePath = getTestSuitePath(testFileAbsolutePath, process.cwd())
129129
const testSourceFile = getTestSuitePath(testFileAbsolutePath, this.repositoryRoot)
130130

131-
const testSuiteMetadata = getTestSuiteCommonTags(
132-
this.command,
133-
this.frameworkVersion,
134-
testSuitePath,
135-
'cucumber'
136-
)
131+
const testSuiteMetadata = {
132+
...getTestSuiteCommonTags(
133+
this.command,
134+
this.frameworkVersion,
135+
testSuitePath,
136+
'cucumber'
137+
),
138+
...this.getSessionRequestErrorTags(),
139+
}
137140
if (isUnskippable) {
138141
this.telemetry.count(TELEMETRY_ITR_UNSKIPPABLE, { testLevel: 'suite' })
139142
testSuiteMetadata[TEST_ITR_UNSKIPPABLE] = 'true'

‎packages/datadog-plugin-cypress/src/cypress-plugin.js‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,8 @@ const {
4646
TEST_RETRY_REASON_TYPES,
4747
getPullRequestDiff,
4848
getModifiedFilesFromDiff,
49+
getSessionRequestErrorTags,
50+
DD_CI_LIBRARY_CONFIGURATION_ERROR,
4951
TEST_IS_MODIFIED,
5052
getPullRequestBaseBranch,
5153
} = require('../../dd-trace/src/plugins/util/test')
@@ -317,10 +319,15 @@ class CypressPlugin {
317319
this.testEnvironmentMetadata[DD_TEST_IS_USER_PROVIDED_SERVICE] =
318320
tracer._tracer._config.isServiceUserProvided ? 'true' : 'false'
319321

322+
this._pendingRequestErrorTags = []
320323
this.libraryConfigurationPromise = getLibraryConfiguration(this.tracer, this.testConfiguration)
321324
.then((libraryConfigurationResponse) => {
322325
if (libraryConfigurationResponse.err) {
323326
log.error('Cypress plugin library config response error', libraryConfigurationResponse.err)
327+
this._pendingRequestErrorTags.push({
328+
tag: DD_CI_LIBRARY_CONFIGURATION_ERROR,
329+
value: 'true',
330+
})
324331
} else {
325332
const {
326333
libraryConfig: {
@@ -415,6 +422,7 @@ class CypressPlugin {
415422
if (this.testSessionSpan && this.testModuleSpan) {
416423
testSuiteTags[TEST_SESSION_ID] = this.testSessionSpan.context().toTraceId()
417424
testSuiteTags[TEST_MODULE_ID] = this.testModuleSpan.context().toSpanId()
425+
Object.assign(testSuiteTags, this.getSessionRequestErrorTags())
418426
// If testSuiteSpan couldn't be created, we'll use the testModuleSpan as the parent
419427
if (!this.testSuiteSpan) {
420428
testSuiteTags[TEST_SUITE_ID] = this.testModuleSpan.context().toSpanId()
@@ -471,6 +479,14 @@ class CypressPlugin {
471479
})
472480
}
473481

482+
/**
483+
* Returns request error tags from the test session span for propagation to test spans.
484+
* @returns {Record<string, string>}
485+
*/
486+
getSessionRequestErrorTags () {
487+
return getSessionRequestErrorTags(this.testSessionSpan)
488+
}
489+
474490
ciVisEvent (name, testLevel, tags = {}) {
475491
incrementCountMetric(name, {
476492
testLevel,
@@ -599,14 +615,20 @@ class CypressPlugin {
599615
},
600616
integrationName: TEST_FRAMEWORK_NAME,
601617
})
618+
for (const { tag, value } of this._pendingRequestErrorTags) {
619+
this.testSessionSpan.setTag(tag, value)
620+
}
621+
this._pendingRequestErrorTags = []
602622
this.ciVisEvent(TELEMETRY_EVENT_CREATED, 'session')
603623

624+
const sessionRequestErrorTags = getSessionRequestErrorTags(this.testSessionSpan)
604625
this.testModuleSpan = this.tracer.startSpan(`${TEST_FRAMEWORK_NAME}.test_module`, {
605626
childOf: this.testSessionSpan,
606627
tags: {
607628
[COMPONENT]: TEST_FRAMEWORK_NAME,
608629
...this.testEnvironmentMetadata,
609630
...testModuleSpanMetadata,
631+
...sessionRequestErrorTags,
610632
},
611633
integrationName: TEST_FRAMEWORK_NAME,
612634
})

0 commit comments

Comments
 (0)