Skip to content

Commit 71c4307

Browse files
[test optimization] [SDTEST-1355] Fix DI issues with auto test retries (#5072)
Co-authored-by: Thomas Watson <[email protected]>
1 parent af641d6 commit 71c4307

18 files changed

Lines changed: 583 additions & 428 deletions

File tree

integration-tests/cucumber/cucumber.spec.js

Lines changed: 30 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -39,9 +39,10 @@ const {
3939
TEST_SESSION_NAME,
4040
TEST_LEVEL_EVENT_TYPES,
4141
DI_ERROR_DEBUG_INFO_CAPTURED,
42-
DI_DEBUG_ERROR_FILE,
43-
DI_DEBUG_ERROR_SNAPSHOT_ID,
44-
DI_DEBUG_ERROR_LINE
42+
DI_DEBUG_ERROR_PREFIX,
43+
DI_DEBUG_ERROR_FILE_SUFFIX,
44+
DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX,
45+
DI_DEBUG_ERROR_LINE_SUFFIX
4546
} = require('../../packages/dd-trace/src/plugins/util/test')
4647
const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env')
4748

@@ -1559,10 +1560,12 @@ versions.forEach(version => {
15591560
assert.equal(retriedTests.length, 1)
15601561
const [retriedTest] = retriedTests
15611562

1562-
assert.notProperty(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED)
1563-
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_FILE)
1564-
assert.notProperty(retriedTest.metrics, DI_DEBUG_ERROR_LINE)
1565-
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_SNAPSHOT_ID)
1563+
const hasDebugTags = Object.keys(retriedTest.meta)
1564+
.some(property =>
1565+
property.startsWith(DI_DEBUG_ERROR_PREFIX) || property === DI_ERROR_DEBUG_INFO_CAPTURED
1566+
)
1567+
1568+
assert.isFalse(hasDebugTags)
15661569
})
15671570
const logsPromise = receiver
15681571
.gatherPayloadsMaxTimeout(({ url }) => url === logsEndpoint, (payloads) => {
@@ -1602,11 +1605,12 @@ versions.forEach(version => {
16021605

16031606
assert.equal(retriedTests.length, 1)
16041607
const [retriedTest] = retriedTests
1608+
const hasDebugTags = Object.keys(retriedTest.meta)
1609+
.some(property =>
1610+
property.startsWith(DI_DEBUG_ERROR_PREFIX) || property === DI_ERROR_DEBUG_INFO_CAPTURED
1611+
)
16051612

1606-
assert.notProperty(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED)
1607-
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_FILE)
1608-
assert.notProperty(retriedTest.metrics, DI_DEBUG_ERROR_LINE)
1609-
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_SNAPSHOT_ID)
1613+
assert.isFalse(hasDebugTags)
16101614
})
16111615
const logsPromise = receiver
16121616
.gatherPayloadsMaxTimeout(({ url }) => url === logsEndpoint, (payloads) => {
@@ -1655,15 +1659,17 @@ versions.forEach(version => {
16551659
const [retriedTest] = retriedTests
16561660

16571661
assert.propertyVal(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED, 'true')
1658-
assert.propertyVal(
1659-
retriedTest.meta,
1660-
DI_DEBUG_ERROR_FILE,
1661-
'ci-visibility/features-di/support/sum.js'
1662+
1663+
assert.isTrue(
1664+
retriedTest.meta[`${DI_DEBUG_ERROR_PREFIX}.0.${DI_DEBUG_ERROR_FILE_SUFFIX}`]
1665+
.endsWith('ci-visibility/features-di/support/sum.js')
16621666
)
1663-
assert.equal(retriedTest.metrics[DI_DEBUG_ERROR_LINE], 4)
1664-
assert.exists(retriedTest.meta[DI_DEBUG_ERROR_SNAPSHOT_ID])
1667+
assert.equal(retriedTest.metrics[`${DI_DEBUG_ERROR_PREFIX}.0.${DI_DEBUG_ERROR_LINE_SUFFIX}`], 4)
1668+
1669+
const snapshotIdKey = `${DI_DEBUG_ERROR_PREFIX}.0.${DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX}`
1670+
assert.exists(retriedTest.meta[snapshotIdKey])
16651671

1666-
snapshotIdByTest = retriedTest.meta[DI_DEBUG_ERROR_SNAPSHOT_ID]
1672+
snapshotIdByTest = retriedTest.meta[snapshotIdKey]
16671673
spanIdByTest = retriedTest.span_id.toString()
16681674
traceIdByTest = retriedTest.trace_id.toString()
16691675
})
@@ -1733,14 +1739,12 @@ versions.forEach(version => {
17331739
assert.equal(retriedTests.length, 1)
17341740
const [retriedTest] = retriedTests
17351741

1736-
assert.propertyVal(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED, 'true')
1737-
assert.propertyVal(
1738-
retriedTest.meta,
1739-
DI_DEBUG_ERROR_FILE,
1740-
'ci-visibility/features-di/support/sum.js'
1741-
)
1742-
assert.equal(retriedTest.metrics[DI_DEBUG_ERROR_LINE], 4)
1743-
assert.exists(retriedTest.meta[DI_DEBUG_ERROR_SNAPSHOT_ID])
1742+
const hasDebugTags = Object.keys(retriedTest.meta)
1743+
.some(property =>
1744+
property.startsWith(DI_DEBUG_ERROR_PREFIX) || property === DI_ERROR_DEBUG_INFO_CAPTURED
1745+
)
1746+
1747+
assert.isFalse(hasDebugTags)
17441748
})
17451749
const logsPromise = receiver
17461750
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/logs'), (payloads) => {

integration-tests/jest/jest.spec.js

Lines changed: 26 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,10 @@ const {
3535
TEST_SESSION_NAME,
3636
TEST_LEVEL_EVENT_TYPES,
3737
DI_ERROR_DEBUG_INFO_CAPTURED,
38-
DI_DEBUG_ERROR_FILE,
39-
DI_DEBUG_ERROR_SNAPSHOT_ID,
40-
DI_DEBUG_ERROR_LINE
38+
DI_DEBUG_ERROR_PREFIX,
39+
DI_DEBUG_ERROR_FILE_SUFFIX,
40+
DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX,
41+
DI_DEBUG_ERROR_LINE_SUFFIX
4142
} = require('../../packages/dd-trace/src/plugins/util/test')
4243
const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env')
4344
const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants')
@@ -2426,11 +2427,12 @@ describe('jest CommonJS', () => {
24262427
assert.equal(retriedTests.length, 1)
24272428
const [retriedTest] = retriedTests
24282429

2429-
assert.notProperty(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED)
2430-
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_FILE)
2431-
assert.notProperty(retriedTest.metrics, DI_DEBUG_ERROR_LINE)
2432-
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_SNAPSHOT_ID)
2430+
const hasDebugTags = Object.keys(retriedTest.meta)
2431+
.some(property => property.startsWith(DI_DEBUG_ERROR_PREFIX) || property === DI_ERROR_DEBUG_INFO_CAPTURED)
2432+
2433+
assert.isFalse(hasDebugTags)
24332434
})
2435+
24342436
const logsPromise = receiver
24352437
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/logs'), (payloads) => {
24362438
if (payloads.length > 0) {
@@ -2472,10 +2474,10 @@ describe('jest CommonJS', () => {
24722474
assert.equal(retriedTests.length, 1)
24732475
const [retriedTest] = retriedTests
24742476

2475-
assert.notProperty(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED)
2476-
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_FILE)
2477-
assert.notProperty(retriedTest.metrics, DI_DEBUG_ERROR_LINE)
2478-
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_SNAPSHOT_ID)
2477+
const hasDebugTags = Object.keys(retriedTest.meta)
2478+
.some(property => property.startsWith(DI_DEBUG_ERROR_PREFIX) || property === DI_ERROR_DEBUG_INFO_CAPTURED)
2479+
2480+
assert.isFalse(hasDebugTags)
24792481
})
24802482
const logsPromise = receiver
24812483
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/logs'), (payloads) => {
@@ -2522,15 +2524,17 @@ describe('jest CommonJS', () => {
25222524
const [retriedTest] = retriedTests
25232525

25242526
assert.propertyVal(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED, 'true')
2525-
assert.propertyVal(
2526-
retriedTest.meta,
2527-
DI_DEBUG_ERROR_FILE,
2528-
'ci-visibility/dynamic-instrumentation/dependency.js'
2527+
2528+
assert.isTrue(
2529+
retriedTest.meta[`${DI_DEBUG_ERROR_PREFIX}.0.${DI_DEBUG_ERROR_FILE_SUFFIX}`]
2530+
.endsWith('ci-visibility/dynamic-instrumentation/dependency.js')
25292531
)
2530-
assert.equal(retriedTest.metrics[DI_DEBUG_ERROR_LINE], 4)
2531-
assert.exists(retriedTest.meta[DI_DEBUG_ERROR_SNAPSHOT_ID])
2532+
assert.equal(retriedTest.metrics[`${DI_DEBUG_ERROR_PREFIX}.0.${DI_DEBUG_ERROR_LINE_SUFFIX}`], 4)
25322533

2533-
snapshotIdByTest = retriedTest.meta[DI_DEBUG_ERROR_SNAPSHOT_ID]
2534+
const snapshotIdKey = `${DI_DEBUG_ERROR_PREFIX}.0.${DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX}`
2535+
assert.exists(retriedTest.meta[snapshotIdKey])
2536+
2537+
snapshotIdByTest = retriedTest.meta[snapshotIdKey]
25342538
spanIdByTest = retriedTest.span_id.toString()
25352539
traceIdByTest = retriedTest.trace_id.toString()
25362540

@@ -2603,14 +2607,10 @@ describe('jest CommonJS', () => {
26032607
assert.equal(retriedTests.length, 1)
26042608
const [retriedTest] = retriedTests
26052609

2606-
assert.propertyVal(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED, 'true')
2607-
assert.propertyVal(
2608-
retriedTest.meta,
2609-
DI_DEBUG_ERROR_FILE,
2610-
'ci-visibility/dynamic-instrumentation/dependency.js'
2611-
)
2612-
assert.equal(retriedTest.metrics[DI_DEBUG_ERROR_LINE], 4)
2613-
assert.exists(retriedTest.meta[DI_DEBUG_ERROR_SNAPSHOT_ID])
2610+
const hasDebugTags = Object.keys(retriedTest.meta)
2611+
.some(property => property.startsWith(DI_DEBUG_ERROR_PREFIX) || property === DI_ERROR_DEBUG_INFO_CAPTURED)
2612+
2613+
assert.isFalse(hasDebugTags)
26142614
})
26152615
const logsPromise = receiver
26162616
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/logs'), (payloads) => {

integration-tests/mocha/mocha.spec.js

Lines changed: 25 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -37,9 +37,10 @@ const {
3737
TEST_LEVEL_EVENT_TYPES,
3838
TEST_EARLY_FLAKE_ABORT_REASON,
3939
DI_ERROR_DEBUG_INFO_CAPTURED,
40-
DI_DEBUG_ERROR_FILE,
41-
DI_DEBUG_ERROR_SNAPSHOT_ID,
42-
DI_DEBUG_ERROR_LINE
40+
DI_DEBUG_ERROR_PREFIX,
41+
DI_DEBUG_ERROR_FILE_SUFFIX,
42+
DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX,
43+
DI_DEBUG_ERROR_LINE_SUFFIX
4344
} = require('../../packages/dd-trace/src/plugins/util/test')
4445
const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env')
4546
const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants')
@@ -2166,10 +2167,10 @@ describe('mocha CommonJS', function () {
21662167
assert.equal(retriedTests.length, 1)
21672168
const [retriedTest] = retriedTests
21682169

2169-
assert.notProperty(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED)
2170-
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_FILE)
2171-
assert.notProperty(retriedTest.metrics, DI_DEBUG_ERROR_LINE)
2172-
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_SNAPSHOT_ID)
2170+
const hasDebugTags = Object.keys(retriedTest.meta)
2171+
.some(property => property.startsWith(DI_DEBUG_ERROR_PREFIX) || property === DI_ERROR_DEBUG_INFO_CAPTURED)
2172+
2173+
assert.isFalse(hasDebugTags)
21732174
})
21742175

21752176
const logsPromise = receiver
@@ -2217,10 +2218,10 @@ describe('mocha CommonJS', function () {
22172218
assert.equal(retriedTests.length, 1)
22182219
const [retriedTest] = retriedTests
22192220

2220-
assert.notProperty(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED)
2221-
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_FILE)
2222-
assert.notProperty(retriedTest.metrics, DI_DEBUG_ERROR_LINE)
2223-
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_SNAPSHOT_ID)
2221+
const hasDebugTags = Object.keys(retriedTest.meta)
2222+
.some(property => property.startsWith(DI_DEBUG_ERROR_PREFIX) || property === DI_ERROR_DEBUG_INFO_CAPTURED)
2223+
2224+
assert.isFalse(hasDebugTags)
22242225
})
22252226

22262227
const logsPromise = receiver
@@ -2273,15 +2274,17 @@ describe('mocha CommonJS', function () {
22732274
const [retriedTest] = retriedTests
22742275

22752276
assert.propertyVal(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED, 'true')
2276-
assert.propertyVal(
2277-
retriedTest.meta,
2278-
DI_DEBUG_ERROR_FILE,
2279-
'ci-visibility/dynamic-instrumentation/dependency.js'
2277+
assert.isTrue(
2278+
retriedTest.meta[`${DI_DEBUG_ERROR_PREFIX}.0.${DI_DEBUG_ERROR_FILE_SUFFIX}`]
2279+
.endsWith('ci-visibility/dynamic-instrumentation/dependency.js')
22802280
)
2281-
assert.equal(retriedTest.metrics[DI_DEBUG_ERROR_LINE], 4)
2282-
assert.exists(retriedTest.meta[DI_DEBUG_ERROR_SNAPSHOT_ID])
2281+
assert.equal(retriedTest.metrics[`${DI_DEBUG_ERROR_PREFIX}.0.${DI_DEBUG_ERROR_LINE_SUFFIX}`], 4)
2282+
2283+
const snapshotIdKey = `${DI_DEBUG_ERROR_PREFIX}.0.${DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX}`
2284+
2285+
assert.exists(retriedTest.meta[snapshotIdKey])
22832286

2284-
snapshotIdByTest = retriedTest.meta[DI_DEBUG_ERROR_SNAPSHOT_ID]
2287+
snapshotIdByTest = retriedTest.meta[snapshotIdKey]
22852288
spanIdByTest = retriedTest.span_id.toString()
22862289
traceIdByTest = retriedTest.trace_id.toString()
22872290

@@ -2358,14 +2361,10 @@ describe('mocha CommonJS', function () {
23582361
assert.equal(retriedTests.length, 1)
23592362
const [retriedTest] = retriedTests
23602363

2361-
assert.propertyVal(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED, 'true')
2362-
assert.propertyVal(
2363-
retriedTest.meta,
2364-
DI_DEBUG_ERROR_FILE,
2365-
'ci-visibility/dynamic-instrumentation/dependency.js'
2366-
)
2367-
assert.equal(retriedTest.metrics[DI_DEBUG_ERROR_LINE], 4)
2368-
assert.exists(retriedTest.meta[DI_DEBUG_ERROR_SNAPSHOT_ID])
2364+
const hasDebugTags = Object.keys(retriedTest.meta)
2365+
.some(property => property.startsWith(DI_DEBUG_ERROR_PREFIX) || property === DI_ERROR_DEBUG_INFO_CAPTURED)
2366+
2367+
assert.isFalse(hasDebugTags)
23692368
})
23702369
const logsPromise = receiver
23712370
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/logs'), (payloads) => {

integration-tests/vitest/vitest.spec.js

Lines changed: 31 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -26,15 +26,16 @@ const {
2626
TEST_EARLY_FLAKE_ABORT_REASON,
2727
TEST_SUITE,
2828
DI_ERROR_DEBUG_INFO_CAPTURED,
29-
DI_DEBUG_ERROR_FILE,
30-
DI_DEBUG_ERROR_LINE,
31-
DI_DEBUG_ERROR_SNAPSHOT_ID
29+
DI_DEBUG_ERROR_PREFIX,
30+
DI_DEBUG_ERROR_FILE_SUFFIX,
31+
DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX,
32+
DI_DEBUG_ERROR_LINE_SUFFIX
3233
} = require('../../packages/dd-trace/src/plugins/util/test')
3334
const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env')
3435

3536
const NUM_RETRIES_EFD = 3
3637

37-
const versions = ['1.6.0', 'latest']
38+
const versions = ['latest']
3839

3940
const linePctMatchRegex = /Lines\s+:\s+([\d.]+)%/
4041

@@ -920,10 +921,12 @@ versions.forEach((version) => {
920921
assert.equal(retriedTests.length, 1)
921922
const [retriedTest] = retriedTests
922923

923-
assert.notProperty(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED)
924-
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_FILE)
925-
assert.notProperty(retriedTest.metrics, DI_DEBUG_ERROR_LINE)
926-
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_SNAPSHOT_ID)
924+
const hasDebugTags = Object.keys(retriedTest.meta)
925+
.some(property =>
926+
property.startsWith(DI_DEBUG_ERROR_PREFIX) || property === DI_ERROR_DEBUG_INFO_CAPTURED
927+
)
928+
929+
assert.isFalse(hasDebugTags)
927930
})
928931

929932
const logsPromise = receiver
@@ -968,11 +971,12 @@ versions.forEach((version) => {
968971

969972
assert.equal(retriedTests.length, 1)
970973
const [retriedTest] = retriedTests
974+
const hasDebugTags = Object.keys(retriedTest.meta)
975+
.some(property =>
976+
property.startsWith(DI_DEBUG_ERROR_PREFIX) || property === DI_ERROR_DEBUG_INFO_CAPTURED
977+
)
971978

972-
assert.notProperty(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED)
973-
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_FILE)
974-
assert.notProperty(retriedTest.metrics, DI_DEBUG_ERROR_LINE)
975-
assert.notProperty(retriedTest.meta, DI_DEBUG_ERROR_SNAPSHOT_ID)
979+
assert.isFalse(hasDebugTags)
976980
})
977981

978982
const logsPromise = receiver
@@ -1023,15 +1027,17 @@ versions.forEach((version) => {
10231027
const [retriedTest] = retriedTests
10241028

10251029
assert.propertyVal(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED, 'true')
1026-
assert.propertyVal(
1027-
retriedTest.meta,
1028-
DI_DEBUG_ERROR_FILE,
1029-
'ci-visibility/vitest-tests/bad-sum.mjs'
1030+
1031+
assert.isTrue(
1032+
retriedTest.meta[`${DI_DEBUG_ERROR_PREFIX}.0.${DI_DEBUG_ERROR_FILE_SUFFIX}`]
1033+
.endsWith('ci-visibility/vitest-tests/bad-sum.mjs')
10301034
)
1031-
assert.equal(retriedTest.metrics[DI_DEBUG_ERROR_LINE], 4)
1032-
assert.exists(retriedTest.meta[DI_DEBUG_ERROR_SNAPSHOT_ID])
1035+
assert.equal(retriedTest.metrics[`${DI_DEBUG_ERROR_PREFIX}.0.${DI_DEBUG_ERROR_LINE_SUFFIX}`], 4)
1036+
1037+
const snapshotIdKey = `${DI_DEBUG_ERROR_PREFIX}.0.${DI_DEBUG_ERROR_SNAPSHOT_ID_SUFFIX}`
1038+
assert.exists(retriedTest.meta[snapshotIdKey])
10331039

1034-
snapshotIdByTest = retriedTest.meta[DI_DEBUG_ERROR_SNAPSHOT_ID]
1040+
snapshotIdByTest = retriedTest.meta[snapshotIdKey]
10351041
spanIdByTest = retriedTest.span_id.toString()
10361042
traceIdByTest = retriedTest.trace_id.toString()
10371043

@@ -1107,14 +1113,12 @@ versions.forEach((version) => {
11071113
assert.equal(retriedTests.length, 1)
11081114
const [retriedTest] = retriedTests
11091115

1110-
assert.propertyVal(retriedTest.meta, DI_ERROR_DEBUG_INFO_CAPTURED, 'true')
1111-
assert.propertyVal(
1112-
retriedTest.meta,
1113-
DI_DEBUG_ERROR_FILE,
1114-
'ci-visibility/vitest-tests/bad-sum.mjs'
1115-
)
1116-
assert.equal(retriedTest.metrics[DI_DEBUG_ERROR_LINE], 4)
1117-
assert.exists(retriedTest.meta[DI_DEBUG_ERROR_SNAPSHOT_ID])
1116+
const hasDebugTags = Object.keys(retriedTest.meta)
1117+
.some(property =>
1118+
property.startsWith(DI_DEBUG_ERROR_PREFIX) || property === DI_ERROR_DEBUG_INFO_CAPTURED
1119+
)
1120+
1121+
assert.isFalse(hasDebugTags)
11181122
})
11191123

11201124
const logsPromise = receiver

0 commit comments

Comments
 (0)