Skip to content

Commit de79e08

Browse files
AlexeyKuznetsov-DDAlexeyKuznetsov-DD
authored
Unified timeout of 15 minutes. Improved logic of taking thread and heap dumps. (#9552)
1 parent 166f773 commit de79e08

9 files changed

Lines changed: 24 additions & 76 deletions

File tree

‎dd-java-agent/agent-ci-visibility/build.gradle‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,3 @@
1-
import java.time.Duration
2-
import java.time.temporal.ChronoUnit
3-
41
buildscript {
52
repositories {
63
mavenLocal()
@@ -67,7 +64,3 @@ shadowJar {
6764
jar {
6865
archiveClassifier = 'unbundled'
6966
}
70-
71-
tasks.withType(Test).configureEach {
72-
timeout = Duration.of(12, ChronoUnit.MINUTES)
73-
}

‎dd-java-agent/instrumentation/aws-java/aws-java-sqs-1.0/build.gradle‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,3 @@
1-
import java.time.Duration
2-
import java.time.temporal.ChronoUnit
3-
41
muzzle {
52
pass {
63
group = "com.amazonaws"
@@ -18,15 +15,6 @@ addTestSuiteExtendingForDir('latestDepForkedTest', 'latestDepTest', 'test')
1815

1916
apply from: "$rootDir/gradle/configure_tests.gradle"
2017

21-
// Increased time needed in CI due to migration from JUnit 4 SpockRunner to JUnit 5 SpockExtension
22-
forkedTest {
23-
timeout = Duration.of(15, ChronoUnit.MINUTES)
24-
}
25-
26-
latestDepForkedTest {
27-
timeout = Duration.of(15, ChronoUnit.MINUTES)
28-
}
29-
3018
dependencies {
3119
compileOnly group: 'com.amazonaws', name: 'aws-java-sdk-sqs', version: '1.11.0'
3220
compileOnly group: 'com.amazonaws', name: 'amazon-sqs-java-messaging-lib', version: '1.0.0'

‎dd-java-agent/instrumentation/aws-java/aws-java-sqs-2.0/build.gradle‎

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,3 @@
1-
import java.time.Duration
2-
import java.time.temporal.ChronoUnit
3-
41
muzzle {
52
pass {
63
group = "software.amazon.awssdk"
@@ -19,16 +16,6 @@ addTestSuiteExtendingForDir('latestDepForkedTest', 'latestDepTest', 'test')
1916

2017
apply from: "$rootDir/gradle/configure_tests.gradle"
2118

22-
// Increased time needed in CI due to migration from JUnit 4 SpockRunner to JUnit 5 SpockExtension
23-
forkedTest {
24-
timeout = Duration.of(15, ChronoUnit.MINUTES)
25-
}
26-
27-
latestDepForkedTest {
28-
timeout = Duration.of(15, ChronoUnit.MINUTES)
29-
}
30-
31-
3219
dependencies {
3320
compileOnly group: 'software.amazon.awssdk', name: 'sqs', version: '2.2.0'
3421
compileOnly group: 'com.amazonaws', name: 'amazon-sqs-java-messaging-lib', version: '2.0.0'

‎dd-java-agent/instrumentation/kafka/kafka-clients-0.11/build.gradle‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,3 @@
1-
import java.time.Duration
2-
import java.time.temporal.ChronoUnit
3-
41
muzzle {
52
pass {
63
name = "since-0.11"
@@ -25,11 +22,6 @@ addTestSuite('iastLatestDepTest3')
2522

2623
apply from: "$rootDir/gradle/configure_tests.gradle"
2724

28-
// Increased time needed in CI due to migration from JUnit 4 SpockRunner to JUnit 5 SpockExtension
29-
forkedTest {
30-
timeout = Duration.of(15, ChronoUnit.MINUTES)
31-
}
32-
3325
dependencies {
3426
compileOnly group: 'org.apache.kafka', name: 'kafka-clients', version: '0.11.0.0'
3527
implementation project(':dd-java-agent:instrumentation:kafka:kafka-common')

‎dd-java-agent/instrumentation/vertx/vertx-rx-3.5/build.gradle‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,3 @@
1-
import java.time.Duration
2-
import java.time.temporal.ChronoUnit
3-
41
apply from: "$rootDir/gradle/java.gradle"
52

63
addTestSuiteForDir('latestDepTest', 'test')
@@ -12,15 +9,6 @@ latestDepTest {
129
finalizedBy 'latestDepForkedTest'
1310
}
1411

15-
// Increased time needed in CI due to migration from JUnit 4 SpockRunner to JUnit 5 SpockExtension
16-
forkedTest {
17-
timeout = Duration.of(15, ChronoUnit.MINUTES)
18-
}
19-
20-
latestDepForkedTest {
21-
timeout = Duration.of(15, ChronoUnit.MINUTES)
22-
}
23-
2412
dependencies {
2513
testImplementation project(':dd-java-agent:instrumentation:netty:netty-4.1')
2614
testImplementation project(':dd-java-agent:instrumentation:trace-annotation')

‎dd-smoke-tests/gradle/build.gradle‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,7 @@ test {
2525
environment "MAVEN_REPOSITORY_PROXY", project.property("mavenRepositoryProxy")
2626
}
2727

28-
// overriding the default timeout of 9 minutes set in configure_tests.gradle,
29-
// as Gradle smoke tests might run for a longer duration
28+
// Overriding the default timeout as Gradle smoke tests might run for a longer duration.
3029
timeout = Duration.of(20, ChronoUnit.MINUTES)
3130
}
3231

‎dd-smoke-tests/maven/build.gradle‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,7 @@ tasks.withType(Test).configureEach {
2626
dependsOn "shadowJar"
2727
jvmArgs "-Ddatadog.smoketest.maven.jar.path=${tasks.shadowJar.archiveFile.get()}"
2828

29-
// overriding the default timeout set in configure_tests.gradle, as Maven smoke
30-
// tests might run for a longer duration
29+
// Overriding the default timeout as Maven smoke tests might run for a longer duration.
3130
timeout = Duration.of(25, ChronoUnit.MINUTES)
3231

3332
if (project.hasProperty("mavenRepositoryProxy")) {

‎gradle/configure_tests.gradle‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,6 @@ def forkedTestLimit = gradle.sharedServices.registerIfAbsent("forkedTestLimit",
2727
maxParallelUsages = 3
2828
}
2929

30-
// Force timeout after 9 minutes (The timeout is configurable per job, default job timeout is 1h)
31-
def testTimeoutDuration = Duration.of(9, ChronoUnit.MINUTES)
32-
3330
testing {
3431
suites.configureEach {
3532
// Use JUnit 5 to run tests
@@ -39,10 +36,6 @@ testing {
3936

4037
// Go through the Test tasks and configure them
4138
tasks.withType(Test).configureEach {
42-
// All tests must complete within 15 minutes.
43-
// This value is quite big because with lower values (3 mins) we were experiencing large number of false positives
44-
timeout = Duration.ofMinutes(15)
45-
4639
// Disable all tests if skipTests property was specified
4740
onlyIf { !project.rootProject.hasProperty("skipTests") }
4841

@@ -82,7 +75,8 @@ tasks.withType(Test).configureEach {
8275
exclude("**/*ForkedTest*")
8376
}
8477

85-
timeout = testTimeoutDuration
78+
// Set test timeout for 15 minutes. Default job timeout is 1h (configured on CI level).
79+
timeout = Duration.of(15, ChronoUnit.MINUTES)
8680

8781
check.dependsOn(it)
8882
}

‎gradle/dump_hanging_test.gradle‎

Lines changed: 20 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,15 @@ tasks.withType(Test).configureEach { testTask ->
1010
t
1111
})
1212

13-
// Calculate delay for taking dumps as test timeout minus 2 minutes, but no less than 1 minute.
14-
def delayMinutes = Math.max(1L, timeout.get().minusMinutes(2).toMinutes())
13+
// Calculate delay for taking dumps as test timeout minus 1 minutes, but no less than 1 minute.
14+
def delayMinutes = Math.max(1L, timeout.get().minusMinutes(1).toMinutes())
1515

1616
def future = scheduler.schedule({
17+
logger.warn("Taking dumps for: ${testTask.getPath()} after ${delayMinutes} minutes.")
18+
1719
try {
1820
// Use Gradle's build dir and adjust for CI artifacts collection if needed.
19-
def dumpDir = layout.buildDirectory.dir('dumps').map {
21+
def dumpsDir = layout.buildDirectory.dir('dumps').map {
2022
if (providers.environmentVariable("CI").isPresent()) {
2123
// Move reports into the folder collected by the collect_reports.sh script.
2224
new File(it.getAsFile().absolutePath.replace('dd-trace-java/dd-java-agent', 'dd-trace-java/workspace/dd-java-agent'))
@@ -25,7 +27,15 @@ tasks.withType(Test).configureEach { testTask ->
2527
}
2628
}.get()
2729

28-
dumpDir.mkdirs()
30+
dumpsDir.mkdirs()
31+
32+
// For simplicity, use `0` as the PID, which collects all thread dumps across JVMs.
33+
// Single file can be useful for quick search.
34+
def threadDumpsFile = new File(dumpsDir, "all-thread-dumps-${System.currentTimeMillis()}.log")
35+
new ProcessBuilder("jcmd", "0", "Thread.print", "-l")
36+
.redirectErrorStream(true)
37+
.redirectOutput(threadDumpsFile)
38+
.start().waitFor()
2939

3040
// Collect PIDs of all Java processes.
3141
def jvmProcesses = 'jcmd -l'.execute().text.readLines()
@@ -36,20 +46,18 @@ tasks.withType(Test).configureEach { testTask ->
3646
.collect({ it.substring(0, it.indexOf(' ')) })
3747

3848
pids.each { pid ->
39-
logger.warn("Taking dumps for: ${testTask.getPath()}")
49+
// Collect heap dump by pid.
50+
def heapDumpFile = new File(dumpsDir, "${pid}-heap-dump-${System.currentTimeMillis()}.hprof").absolutePath
51+
def cmd = "jcmd ${pid} GC.heap_dump ${heapDumpFile}"
52+
cmd.execute().waitFor()
4053

41-
// Collect thread dump.
42-
def threadDumpFile = new File(dumpDir, "${pid}-thread-dump-${System.currentTimeMillis()}.log")
54+
// Collect thread dump by pid.
55+
def threadDumpFile = new File(dumpsDir, "${pid}-thread-dump-${System.currentTimeMillis()}.log")
4356
new ProcessBuilder('jcmd', pid, 'Thread.print', '-l')
4457
.redirectErrorStream(true)
4558
.redirectOutput(threadDumpFile)
4659
.start()
4760
.waitFor()
48-
49-
// Collect heap dump.
50-
def heapDumpFile = new File(dumpDir, "${pid}-heap-dump-${System.currentTimeMillis()}.hprof").absolutePath
51-
def cmd = "jcmd ${pid} GC.heap_dump ${heapDumpFile}"
52-
cmd.execute().waitFor()
5361
}
5462
} catch (Throwable e) {
5563
logger.warn("Dumping failed: ${e.message}")

0 commit comments

Comments
 (0)