Skip to content

Commit 619c7c2

Browse files
juan-fernandezjuan-fernandez
authored
[test optimization] Add missing telemetry events and tags (#7463)
1 parent 510563d commit 619c7c2

File tree

12 files changed

+204
-45
lines changed

12 files changed

+204
-45
lines changed

‎integration-tests/jest/jest.spec.js‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ const {
7878
GIT_REPOSITORY_URL,
7979
} = require('../../packages/dd-trace/src/plugins/util/test')
8080
const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env')
81+
const { TELEMETRY_COVERAGE_UPLOAD } = require('../../packages/dd-trace/src/ci-visibility/telemetry')
8182
const { ERROR_MESSAGE, ERROR_TYPE, ORIGIN_KEY, COMPONENT } = require('../../packages/dd-trace/src/constants')
8283
const { NODE_MAJOR } = require('../../version')
8384
const { version: ddTraceVersion } = require('../../package.json')
@@ -6093,6 +6094,45 @@ describe(`jest@${JEST_VERSION} commonJS`, () => {
60936094
])
60946095
})
60956096

6097+
it('sends coverage_upload.request telemetry metric when coverage is uploaded', async () => {
6098+
receiver.setSettings({
6099+
coverage_report_upload_enabled: true,
6100+
})
6101+
receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] })
6102+
6103+
const telemetryPromise = receiver
6104+
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/apmtelemetry'), (payloads) => {
6105+
const telemetryMetrics = payloads.flatMap(({ payload }) => payload.payload.series)
6106+
6107+
const coverageUploadMetric = telemetryMetrics.find(
6108+
({ metric }) => metric === TELEMETRY_COVERAGE_UPLOAD
6109+
)
6110+
6111+
assert.ok(coverageUploadMetric, 'coverage_upload.request telemetry metric should be sent')
6112+
})
6113+
6114+
childProcess = exec(
6115+
runTestsCommand,
6116+
{
6117+
cwd,
6118+
env: {
6119+
...getCiVisEvpProxyConfig(receiver.port),
6120+
DD_INSTRUMENTATION_TELEMETRY_ENABLED: 'true',
6121+
ENABLE_CODE_COVERAGE: 'true',
6122+
COVERAGE_REPORTERS: 'lcov',
6123+
COLLECT_COVERAGE_FROM: 'ci-visibility/test/*.js',
6124+
DD_GIT_COMMIT_SHA: gitCommitSha,
6125+
DD_GIT_REPOSITORY_URL: gitRepositoryUrl,
6126+
},
6127+
}
6128+
)
6129+
6130+
await Promise.all([
6131+
once(childProcess, 'exit'),
6132+
telemetryPromise,
6133+
])
6134+
})
6135+
60966136
it('does not upload coverage report when coverage_report_upload_enabled is false', async () => {
60976137
receiver.setSettings({
60986138
coverage_report_upload_enabled: false,

‎integration-tests/vitest/vitest.spec.js‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ const {
6060
GIT_REPOSITORY_URL,
6161
} = require('../../packages/dd-trace/src/plugins/util/test')
6262
const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env')
63+
const { TELEMETRY_COVERAGE_UPLOAD } = require('../../packages/dd-trace/src/ci-visibility/telemetry')
6364
const { NODE_MAJOR } = require('../../version')
6465

6566
const NUM_RETRIES_EFD = 3
@@ -2332,6 +2333,45 @@ versions.forEach((version) => {
23322333
])
23332334
})
23342335

2336+
it('sends coverage_upload.request telemetry metric when coverage is uploaded', async () => {
2337+
receiver.setSettings({
2338+
coverage_report_upload_enabled: true,
2339+
})
2340+
receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] })
2341+
2342+
const telemetryPromise = receiver
2343+
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/apmtelemetry'), (payloads) => {
2344+
const telemetryMetrics = payloads.flatMap(({ payload }) => payload.payload.series)
2345+
2346+
const coverageUploadMetric = telemetryMetrics.find(
2347+
({ metric }) => metric === TELEMETRY_COVERAGE_UPLOAD
2348+
)
2349+
2350+
assert.ok(coverageUploadMetric, 'coverage_upload.request telemetry metric should be sent')
2351+
})
2352+
2353+
childProcess = exec(
2354+
'./node_modules/.bin/vitest run --coverage',
2355+
{
2356+
cwd,
2357+
env: {
2358+
...getCiVisEvpProxyConfig(receiver.port),
2359+
NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init',
2360+
DD_INSTRUMENTATION_TELEMETRY_ENABLED: 'true',
2361+
COVERAGE_PROVIDER: 'v8',
2362+
TEST_DIR: 'ci-visibility/vitest-tests/coverage-test.mjs',
2363+
DD_GIT_COMMIT_SHA: gitCommitSha,
2364+
DD_GIT_REPOSITORY_URL: gitRepositoryUrl,
2365+
},
2366+
}
2367+
)
2368+
2369+
await Promise.all([
2370+
once(childProcess, 'exit'),
2371+
telemetryPromise,
2372+
])
2373+
})
2374+
23352375
it('does not upload coverage report when coverage_report_upload_enabled is false', async () => {
23362376
receiver.setSettings({
23372377
coverage_report_upload_enabled: false,

‎packages/datadog-plugin-cucumber/src/index.js‎

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -13,15 +13,13 @@ const {
1313
isModifiedTest,
1414
CUCUMBER_IS_PARALLEL,
1515
ITR_CORRELATION_ID,
16-
TEST_BROWSER_DRIVER,
1716
TEST_CODE_OWNERS,
1817
TEST_EARLY_FLAKE_ABORT_REASON,
1918
TEST_EARLY_FLAKE_ENABLED,
2019
TEST_HAS_FAILED_ALL_RETRIES,
2120
TEST_IS_MODIFIED,
2221
TEST_IS_NEW,
2322
TEST_IS_RETRY,
24-
TEST_IS_RUM_ACTIVE,
2523
TEST_ITR_FORCED_RUN,
2624
TEST_ITR_UNSKIPPABLE,
2725
TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED,
@@ -362,13 +360,7 @@ class CucumberPlugin extends CiPlugin {
362360
}
363361
}
364362

365-
const spanTags = span.context()._tags
366-
const telemetryTags = {
367-
hasCodeOwners: !!spanTags[TEST_CODE_OWNERS],
368-
isNew,
369-
isRum: spanTags[TEST_IS_RUM_ACTIVE] === 'true',
370-
browserDriver: spanTags[TEST_BROWSER_DRIVER],
371-
}
363+
const telemetryTags = this.getTestTelemetryTags(span)
372364
span.finish()
373365
if (!isStep) {
374366
this.telemetry.ciVisEvent(

‎packages/datadog-plugin-cypress/src/cypress-plugin.js‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -976,11 +976,15 @@ class CypressPlugin {
976976
this.finishedTestsByFile[testSuite] = [finishedTest]
977977
}
978978
// test spans are finished at after:spec
979+
const activeSpanTags = this.activeTestSpan.context()._tags
979980
this.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'test', {
980-
hasCodeOwners: !!this.activeTestSpan.context()._tags[TEST_CODE_OWNERS],
981+
hasCodeOwners: !!activeSpanTags[TEST_CODE_OWNERS],
981982
isNew,
982983
isRum: isRUMActive,
983984
browserDriver: 'cypress',
985+
isQuarantined: isQuarantinedFromSupport,
986+
isModified,
987+
isDisabled: activeSpanTags[TEST_MANAGEMENT_IS_DISABLED] === 'true',
984988
})
985989
this.activeTestSpan = null
986990

‎packages/datadog-plugin-jest/src/index.js‎

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,6 @@ const {
2525
TEST_EARLY_FLAKE_ENABLED,
2626
TEST_EARLY_FLAKE_ABORT_REASON,
2727
JEST_DISPLAY_NAME,
28-
TEST_IS_RUM_ACTIVE,
29-
TEST_BROWSER_DRIVER,
3028
getFormattedError,
3129
TEST_RETRY_REASON,
3230
TEST_MANAGEMENT_ENABLED,
@@ -391,16 +389,10 @@ class JestPlugin extends CiPlugin {
391389
span.setTag(TEST_RETRY_REASON, TEST_RETRY_REASON_TYPES.atr)
392390
}
393391

394-
const spanTags = span.context()._tags
395392
this.telemetry.ciVisEvent(
396393
TELEMETRY_EVENT_FINISHED,
397394
'test',
398-
{
399-
hasCodeOwners: !!spanTags[TEST_CODE_OWNERS],
400-
isNew: spanTags[TEST_IS_NEW] === 'true',
401-
isRum: spanTags[TEST_IS_RUM_ACTIVE] === 'true',
402-
browserDriver: spanTags[TEST_BROWSER_DRIVER],
403-
}
395+
this.getTestTelemetryTags(span)
404396
)
405397

406398
span.finish()

‎packages/datadog-plugin-mocha/src/index.js‎

Lines changed: 2 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,6 @@ const {
2323
TEST_EARLY_FLAKE_ENABLED,
2424
TEST_EARLY_FLAKE_ABORT_REASON,
2525
MOCHA_IS_PARALLEL,
26-
TEST_IS_RUM_ACTIVE,
27-
TEST_BROWSER_DRIVER,
2826
TEST_RETRY_REASON,
2927
TEST_MANAGEMENT_ENABLED,
3028
TEST_MANAGEMENT_IS_QUARANTINED,
@@ -236,16 +234,10 @@ class MochaPlugin extends CiPlugin {
236234
span.setTag(TEST_RETRY_REASON, TEST_RETRY_REASON_TYPES.atf)
237235
}
238236

239-
const spanTags = span.context()._tags
240237
this.telemetry.ciVisEvent(
241238
TELEMETRY_EVENT_FINISHED,
242239
'test',
243-
{
244-
hasCodeOwners: !!spanTags[TEST_CODE_OWNERS],
245-
isNew: spanTags[TEST_IS_NEW] === 'true',
246-
isRum: spanTags[TEST_IS_RUM_ACTIVE] === 'true',
247-
browserDriver: spanTags[TEST_BROWSER_DRIVER],
248-
}
240+
this.getTestTelemetryTags(span)
249241
)
250242

251243
span.finish()
@@ -310,16 +302,10 @@ class MochaPlugin extends CiPlugin {
310302
span.setTag('error', err)
311303
}
312304

313-
const spanTags = span.context()._tags
314305
this.telemetry.ciVisEvent(
315306
TELEMETRY_EVENT_FINISHED,
316307
'test',
317-
{
318-
hasCodeOwners: !!spanTags[TEST_CODE_OWNERS],
319-
isNew: spanTags[TEST_IS_NEW] === 'true',
320-
isRum: spanTags[TEST_IS_RUM_ACTIVE] === 'true',
321-
browserDriver: spanTags[TEST_BROWSER_DRIVER],
322-
}
308+
this.getTestTelemetryTags(span)
323309
)
324310
if (isFirstAttempt && willBeRetried && this.di && this.libraryConfig?.isDiEnabled) {
325311
const probeInformation = this.addDiProbe(err)

‎packages/datadog-plugin-playwright/src/index.js‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -388,6 +388,9 @@ class PlaywrightPlugin extends CiPlugin {
388388
isNew,
389389
isRum: isRUMActive,
390390
browserDriver: 'playwright',
391+
isQuarantined,
392+
isDisabled,
393+
isModified,
391394
}
392395
)
393396
span.finish()

‎packages/datadog-plugin-vitest/src/index.js‎

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -206,9 +206,7 @@ class VitestPlugin extends CiPlugin {
206206

207207
this.addSub('ci:vitest:test:pass', ({ span, task }) => {
208208
if (span) {
209-
this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'test', {
210-
hasCodeowners: !!span.context()._tags[TEST_CODE_OWNERS],
211-
})
209+
this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'test', this.getTestTelemetryTags(span))
212210
span.setTag(TEST_STATUS, 'pass')
213211
span.finish(this.taskToFinishTime.get(task))
214212
finishAllTraceSpans(span)
@@ -236,9 +234,7 @@ class VitestPlugin extends CiPlugin {
236234
promises.setProbePromise = setProbePromise
237235
}
238236
}
239-
this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'test', {
240-
hasCodeowners: !!span.context()._tags[TEST_CODE_OWNERS],
241-
})
237+
this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'test', this.getTestTelemetryTags(span))
242238
span.setTag(TEST_STATUS, 'fail')
243239

244240
if (error) {
@@ -272,9 +268,7 @@ class VitestPlugin extends CiPlugin {
272268
...(isNew ? { [TEST_IS_NEW]: 'true' } : {}),
273269
}
274270
)
275-
this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'test', {
276-
hasCodeowners: !!testSpan.context()._tags[TEST_CODE_OWNERS],
277-
})
271+
this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'test', this.getTestTelemetryTags(testSpan))
278272
testSpan.finish()
279273
})
280274

‎packages/dd-trace/src/ci-visibility/requests/upload-coverage-report.js‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,14 @@ const FormData = require('../../exporters/common/form-data')
77
const request = require('../../exporters/common/request')
88
const log = require('../../log')
99
const { getValueFromEnvSources } = require('../../config/helper')
10+
const {
11+
incrementCountMetric,
12+
distributionMetric,
13+
TELEMETRY_COVERAGE_UPLOAD,
14+
TELEMETRY_COVERAGE_UPLOAD_MS,
15+
TELEMETRY_COVERAGE_UPLOAD_ERRORS,
16+
TELEMETRY_COVERAGE_UPLOAD_BYTES,
17+
} = require('../telemetry')
1018

1119
const UPLOAD_TIMEOUT_MS = 30_000
1220

@@ -79,8 +87,15 @@ function uploadCoverageReport (
7987

8088
log.debug('Uploading coverage report %s to %s%s', filePath, url, options.path)
8189

90+
incrementCountMetric(TELEMETRY_COVERAGE_UPLOAD)
91+
distributionMetric(TELEMETRY_COVERAGE_UPLOAD_BYTES, {}, compressedCoverage.length)
92+
93+
const startTime = Date.now()
94+
8295
request(form, options, (err, res, statusCode) => {
96+
distributionMetric(TELEMETRY_COVERAGE_UPLOAD_MS, {}, Date.now() - startTime)
8397
if (err) {
98+
incrementCountMetric(TELEMETRY_COVERAGE_UPLOAD_ERRORS, { statusCode })
8499
log.error('Error uploading coverage report: %s', err.message)
85100
return callback(err)
86101
}

‎packages/dd-trace/src/ci-visibility/telemetry.js‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,17 @@ const formattedTags = {
1717
isRum: 'is_rum',
1818
browserDriver: 'browser_driver',
1919
autoInjected: 'auto_injected',
20+
isQuarantined: 'is_quarantined',
21+
isDisabled: 'is_disabled',
22+
isTestManagementEnabled: 'test_management_enabled',
23+
isItrEnabled: 'itr_enabled',
24+
isEarlyFlakeDetectionEnabled: 'early_flake_detection_enabled',
25+
isFlakyTestRetriesEnabled: 'flaky_test_retries_enabled',
26+
isKnownTestsEnabled: 'known_tests_enabled',
27+
isImpactedTestsEnabled: 'impacted_tests_enabled',
28+
isDiEnabled: 'failed_test_replay_enabled',
29+
requireGit: 'require_git',
30+
isModified: 'is_modified',
2031
}
2132

2233
// Transform tags dictionary to array of strings.
@@ -101,6 +112,17 @@ const TELEMETRY_KNOWN_TESTS_MS = 'early_flake_detection.request_ms'
101112
const TELEMETRY_KNOWN_TESTS_ERRORS = 'early_flake_detection.request_errors'
102113
const TELEMETRY_KNOWN_TESTS_RESPONSE_TESTS = 'early_flake_detection.response_tests'
103114
const TELEMETRY_KNOWN_TESTS_RESPONSE_BYTES = 'early_flake_detection.response_bytes'
115+
// coverage upload
116+
const TELEMETRY_COVERAGE_UPLOAD = 'coverage_upload.request'
117+
const TELEMETRY_COVERAGE_UPLOAD_MS = 'coverage_upload.request_ms'
118+
const TELEMETRY_COVERAGE_UPLOAD_ERRORS = 'coverage_upload.request_errors'
119+
const TELEMETRY_COVERAGE_UPLOAD_BYTES = 'coverage_upload.request_bytes'
120+
// test management
121+
const TELEMETRY_TEST_MANAGEMENT_TESTS = 'test_management_tests.request'
122+
const TELEMETRY_TEST_MANAGEMENT_TESTS_MS = 'test_management_tests.request_ms'
123+
const TELEMETRY_TEST_MANAGEMENT_TESTS_ERRORS = 'test_management_tests.request_errors'
124+
const TELEMETRY_TEST_MANAGEMENT_TESTS_RESPONSE_TESTS = 'test_management_tests.response_tests'
125+
const TELEMETRY_TEST_MANAGEMENT_TESTS_RESPONSE_BYTES = 'test_management_tests.response_bytes'
104126

105127
function isStatusCode400 (statusCode) {
106128
return statusCode >= 400 && statusCode < 500
@@ -165,4 +187,13 @@ module.exports = {
165187
TELEMETRY_KNOWN_TESTS_ERRORS,
166188
TELEMETRY_KNOWN_TESTS_RESPONSE_TESTS,
167189
TELEMETRY_KNOWN_TESTS_RESPONSE_BYTES,
190+
TELEMETRY_COVERAGE_UPLOAD,
191+
TELEMETRY_COVERAGE_UPLOAD_MS,
192+
TELEMETRY_COVERAGE_UPLOAD_ERRORS,
193+
TELEMETRY_COVERAGE_UPLOAD_BYTES,
194+
TELEMETRY_TEST_MANAGEMENT_TESTS,
195+
TELEMETRY_TEST_MANAGEMENT_TESTS_MS,
196+
TELEMETRY_TEST_MANAGEMENT_TESTS_ERRORS,
197+
TELEMETRY_TEST_MANAGEMENT_TESTS_RESPONSE_TESTS,
198+
TELEMETRY_TEST_MANAGEMENT_TESTS_RESPONSE_BYTES,
168199
}

0 commit comments

Comments
 (0)