Skip to content

Commit 93bbda1

Browse files
randomandersonrandomanderson
authored
Migrate agent integration tests to gitlab (#8448)
* Add checks, spotless, and test_published_artifact * small fixes * WIP * syntax errors * add gradle daemon logs everywhere * fix c/p error * fix system tests dir * add some ls * always save artifacts * fix workspace path for system tests * move more jobs to the tests stage * split muzzle into 6 jobs * remove system-tests debug output * fix path and index for muzzle * add signing args * 8 parallism * no needs for spotless * java 7 for test published artifacts * use the image for the test jvm * upload ci_app results * when always -> on_success * Cleanup TraceAgentTests to always run * try agent integration test * add the actual target * clean up branch * set keys and use explicit agent value in test * remove DD_* variables from environment before tests * save/restore must be toplevel * some debug logic for keys * yaml * set api keys in after_script * fix DDApiIntegrationTest * remove agent integration tests from circleci * cleanup merge issues * Revert "remove agent integration tests from circleci" This reverts commit 90602a2. * assign CI_AGENT_HOST to localhost * don't skip tests when populating cache * codenarc
1 parent 87b5366 commit 93bbda1

7 files changed

Lines changed: 167 additions & 89 deletions

File tree

‎.circleci/config.continue.yml.j2‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -696,6 +696,8 @@ jobs:
696696
<<: *tests
697697

698698
resource_class: medium
699+
environment:
700+
- CI_AGENT_HOST=localhost
699701

700702
docker:
701703
- image: << pipeline.parameters.docker_image >>:{{ docker_image_prefix }}8

‎.gitlab-ci.yml‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,9 @@ variables:
2525
GRADLE_VERSION: "8.4" # must match gradle-wrapper.properties
2626
JAVA_BUILD_IMAGE_VERSION: "v25.01"
2727
REPO_NOTIFICATION_CHANNEL: "#apm-java-escalations"
28+
PROFILE_TESTS:
29+
description: "Enable profiling of tests"
30+
value: "false"
2831

2932
default:
3033
tags: [ "arch:amd64" ]
@@ -35,6 +38,10 @@ default:
3538
script:
3639
- echo "done"
3740

41+
.set_datadog_api_keys: &set_datadog_api_keys
42+
- export DATADOG_API_KEY_PROD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.DATADOG_API_KEY_PROD --with-decryption --query "Parameter.Value" --out text)
43+
- export DATADOG_API_KEY_DDSTAGING=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.dd_api_key --with-decryption --query "Parameter.Value" --out text)
44+
3845
.gradle_build: &gradle_build
3946
image: ghcr.io/datadog/dd-trace-java-docker-build:${JAVA_BUILD_IMAGE_VERSION}-base
4047
stage: build
@@ -177,12 +184,76 @@ muzzle-dep-report:
177184
- ./reports
178185
- '.gradle/daemon/*/*.out.log'
179186

187+
# In Gitlab, DD_* variables are set because the build runner is instrumented with Datadog telemetry
188+
# To have a pristine environment for the tests, these variables are saved before the test run and restored afterwards
189+
.prepare_test_env: &prepare_test_env
190+
- export gitlabVariables=("DD_SERVICE" "DD_ENTITY_ID" "DD_SITE" "DD_ENV" "DD_DATACENTER" "DD_PARTITION" "DD_CLOUDPROVIDER")
191+
- '[ ! -e pretest.env ] || rm pretest.env'
192+
- |
193+
for VARIABLE in "${gitlabVariables[@]}"
194+
do
195+
echo "export $VARIABLE=${!VARIABLE}" >> pretest.env
196+
unset "$VARIABLE"
197+
done
198+
199+
.restore_pretest_env: &restore_pretest_env
200+
- source pretest.env
201+
202+
.test_job:
203+
extends: .gradle_build
204+
image: ghcr.io/datadog/dd-trace-java-docker-build:$testJvm
205+
needs: [ build ]
206+
stage: tests
207+
variables:
208+
BUILD_CACHE_TYPE: lib
209+
GRADLE_PARAMS: ""
210+
CONTINUE_ON_FAILURE: "false"
211+
script:
212+
- >
213+
if [ "$PROFILE_TESTS" == "true" ] && [ "$testJvm" != "ibm8" ] && [ "$testJvm" != "oracle8" ];
214+
then
215+
export PROFILER_COMMAND="-XX:StartFlightRecording=settings=profile,filename=/tmp/${CI_JOB_NAME_SLUG}.jfr,dumponexit=true";
216+
fi
217+
- *prepare_test_env
218+
- export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xms2G -Xmx2G $PROFILER_COMMAND -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp' -Ddatadog.forkedMaxHeapSize=768M -Ddatadog.forkedMinHeapSize=128M"
219+
- ./gradlew $GRADLE_TARGET $GRADLE_PARAMS -PtestJvm=$testJvm $GRADLE_ARGS --continue || $CONTINUE_ON_FAILURE
220+
after_script:
221+
- *restore_pretest_env
222+
- *set_datadog_api_keys
223+
- .circleci/collect_reports.sh
224+
- if [ "$PROFILE_TESTS" == "true" ]; then .circleci/collect_profiles.sh; fi
225+
- .circleci/collect_results.sh
226+
- .circleci/upload_ciapp.sh tests $testJvm
227+
# TODO Get APM Test Agent Trace Check Results
228+
artifacts:
229+
when: always
230+
paths:
231+
- ./reports.tar
232+
- ./profiles.tar
233+
- ./results
234+
- '.gradle/daemon/*/*.out.log'
235+
236+
agent_integration_tests:
237+
extends: .test_job
238+
variables:
239+
testJvm: "8"
240+
CI_AGENT_HOST: local-agent
241+
GRADLE_TARGET: "traceAgentTest"
242+
services:
243+
- name: datadog/agent:7.34.0
244+
alias: local-agent
245+
variables:
246+
DD_APM_ENABLED: "true"
247+
DD_BIND_HOST: "0.0.0.0"
248+
DD_API_KEY: "invalid_key_but_this_is_fine"
249+
180250
required:
181251
extends: .fan_in
182252
needs:
183253
- spotless
184254
- muzzle
185255
- test_published_artifacts
256+
- agent_integration_tests
186257

187258
deploy_to_profiling_backend:
188259
stage: publish

‎dd-trace-core/src/traceAgentTest/groovy/AbstractTraceAgentTest.groovy‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
import datadog.trace.api.ConfigDefaults
2+
import datadog.trace.api.config.TracerConfig
3+
import datadog.trace.test.util.DDSpecification
4+
import org.testcontainers.containers.GenericContainer
5+
import org.testcontainers.containers.startupcheck.MinimumDurationRunningStartupCheckStrategy
6+
import spock.lang.Shared
7+
8+
import java.time.Duration
9+
10+
abstract class AbstractTraceAgentTest extends DDSpecification {
11+
@Shared
12+
def agentContainer
13+
14+
def setupSpec() {
15+
/*
16+
CI will provide us with agent container running along side our build.
17+
When building locally, however, we need to take matters into our own hands
18+
and we use 'testcontainers' for this.
19+
*/
20+
if ("true" != System.getenv("CI")) {
21+
agentContainer = new GenericContainer("datadog/agent:7.34.0")
22+
.withEnv(["DD_APM_ENABLED": "true",
23+
"DD_BIND_HOST" : "0.0.0.0",
24+
"DD_API_KEY" : "invalid_key_but_this_is_fine",
25+
"DD_LOGS_STDOUT": "yes"])
26+
.withExposedPorts(datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_AGENT_PORT)
27+
.withStartupTimeout(Duration.ofSeconds(120))
28+
// Apparently we need to sleep for a bit so agent's response `{"service:,env:":1}` in rate_by_service.
29+
// This is clearly a race-condition and maybe we should avoid verifying complete response
30+
.withStartupCheckStrategy(new MinimumDurationRunningStartupCheckStrategy(Duration.ofSeconds(10)))
31+
agentContainer.start()
32+
}
33+
}
34+
35+
def setup() {
36+
injectSysConfig(TracerConfig.AGENT_HOST, getAgentContainerHost())
37+
injectSysConfig(TracerConfig.TRACE_AGENT_PORT, getAgentContainerPort())
38+
}
39+
40+
String getAgentContainerHost() {
41+
if (agentContainer) {
42+
return (String) agentContainer.getHost()
43+
}
44+
45+
return System.getenv("CI_AGENT_HOST")
46+
}
47+
48+
String getAgentContainerPort() {
49+
if (agentContainer) {
50+
return (String) agentContainer.getMappedPort(ConfigDefaults.DEFAULT_TRACE_AGENT_PORT)
51+
}
52+
53+
return ConfigDefaults.DEFAULT_TRACE_AGENT_PORT
54+
}
55+
56+
def cleanupSpec() {
57+
agentContainer?.stop()
58+
}
59+
}

‎dd-trace-core/src/traceAgentTest/groovy/DDApiIntegrationTest.groovy‎

Lines changed: 8 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import datadog.communication.http.OkHttpUtils
33
import datadog.communication.serialization.ByteBufferConsumer
44
import datadog.communication.serialization.FlushingBuffer
55
import datadog.communication.serialization.msgpack.MsgPackWriter
6+
import datadog.trace.api.Config
67
import datadog.trace.api.StatsDClient
78
import datadog.trace.common.writer.ListWriter
89
import datadog.trace.common.writer.Payload
@@ -15,39 +16,22 @@ import datadog.trace.common.writer.ddagent.TraceMapperV0_5
1516
import datadog.trace.core.CoreTracer
1617
import datadog.trace.core.DDSpan
1718
import datadog.trace.core.monitor.MonitoringImpl
18-
import datadog.trace.test.util.DDSpecification
1919
import okhttp3.HttpUrl
2020
import okhttp3.OkHttpClient
21-
import org.testcontainers.containers.GenericContainer
22-
import org.testcontainers.containers.startupcheck.MinimumDurationRunningStartupCheckStrategy
23-
import spock.lang.Requires
2421
import spock.lang.Shared
2522

2623
import java.nio.ByteBuffer
27-
import java.time.Duration
2824
import java.util.concurrent.TimeUnit
2925
import java.util.concurrent.atomic.AtomicReference
3026

31-
// It is fine to run on CI because CI provides agent externally, not through testcontainers
32-
@Requires({ "true" == System.getenv("CI") })
33-
class DDApiIntegrationTest extends DDSpecification {
34-
def tracer = CoreTracer.builder().writer(new ListWriter()).build()
27+
class DDApiIntegrationTest extends AbstractTraceAgentTest {
28+
def tracer
3529
DDSpan span
3630

3731
// Looks like okHttp needs to resolve this, even for connection over socket
3832
static final SOMEHOST = "datadoghq.com"
3933
static final SOMEPORT = 123
4034

41-
/*
42-
Note: type here has to stay undefined, otherwise tests will fail in CI in Java 7 because
43-
'testcontainers' are built for Java 8 and Java 7 cannot load this class.
44-
*/
45-
@Shared
46-
def agentContainer
47-
@Shared
48-
def agentContainerHost = "localhost"
49-
@Shared
50-
def agentContainerPort = 8126
5135
@Shared
5236
Process process
5337
@Shared
@@ -69,30 +53,6 @@ class DDApiIntegrationTest extends DDSpecification {
6953
}
7054

7155
def setupSpec() {
72-
/*
73-
CI will provide us with agent container running along side our build.
74-
When building locally, however, we need to take matters into our own hands
75-
and we use 'testcontainers' for this.
76-
*/
77-
if ("true" != System.getenv("CI")) {
78-
agentContainer = new GenericContainer("datadog/agent:7.22.0")
79-
.withEnv(["DD_APM_ENABLED": "true",
80-
"DD_BIND_HOST" : "0.0.0.0",
81-
"DD_API_KEY" : "invalid_key_but_this_is_fine",
82-
"DD_LOGS_STDOUT": "yes"])
83-
.withExposedPorts(datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_AGENT_PORT)
84-
.withStartupTimeout(Duration.ofSeconds(120))
85-
// Apparently we need to sleep for a bit so agent's response `{"service:,env:":1}` in rate_by_service.
86-
// This is clearly a race-condition and maybe we should avoid verifying complete response
87-
.withStartupCheckStrategy(new MinimumDurationRunningStartupCheckStrategy(Duration.ofSeconds(10)))
88-
// .withLogConsumer { output ->
89-
// print output.utf8String
90-
// }
91-
agentContainer.start()
92-
agentContainerHost = agentContainer.getHost()
93-
agentContainerPort = agentContainer.getMappedPort(datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_AGENT_PORT)
94-
}
95-
9656
File tmpDir = File.createTempDir()
9757
tmpDir.deleteOnExit()
9858
socketPath = new File(tmpDir, "socket")
@@ -101,6 +61,7 @@ class DDApiIntegrationTest extends DDSpecification {
10161
}
10262

10363
def setup() {
64+
tracer = CoreTracer.builder().writer(new ListWriter()).build()
10465
span = tracer.buildSpan("fakeOperation").start()
10566
Thread.sleep(1)
10667
span.finish()
@@ -111,15 +72,12 @@ class DDApiIntegrationTest extends DDSpecification {
11172
}
11273

11374
def cleanupSpec() {
114-
if (agentContainer) {
115-
agentContainer.stop()
116-
}
117-
process.destroy()
75+
process?.destroy()
11876
}
11977

12078
def beforeTest(boolean enableV05) {
12179
MonitoringImpl monitoring = new MonitoringImpl(StatsDClient.NO_OP, 1, TimeUnit.SECONDS)
122-
HttpUrl agentUrl = HttpUrl.get(String.format("http://%s:%d", agentContainerHost, agentContainerPort))
80+
HttpUrl agentUrl = HttpUrl.get(Config.get().getAgentUrl())
12381
OkHttpClient httpClient = OkHttpUtils.buildHttpClient(agentUrl, 5000)
12482
discovery = new DDAgentFeaturesDiscovery(httpClient, monitoring, agentUrl, enableV05, true)
12583
api = new DDAgentApi(httpClient, agentUrl, discovery, monitoring, false)
@@ -143,7 +101,7 @@ class DDApiIntegrationTest extends DDSpecification {
143101
assert 200 == response.status()
144102
assert response.success()
145103
assert discovery.getTraceEndpoint() == "${version}/traces"
146-
assert endpoint.get() == "http://${agentContainerHost}:${agentContainerPort}/${version}/traces"
104+
assert endpoint.get() == "${Config.get().getAgentUrl()}/${version}/traces"
147105
assert agentResponse.get()["rate_by_service"] instanceof Map
148106
149107
where:
@@ -166,7 +124,7 @@ class DDApiIntegrationTest extends DDSpecification {
166124
assert 200 == response.status()
167125
assert response.success()
168126
assert discovery.getTraceEndpoint() == "${version}/traces"
169-
assert endpoint.get() == "http://${agentContainerHost}:${agentContainerPort}/${version}/traces"
127+
assert endpoint.get() == "${Config.get().getAgentUrl()}/${version}/traces"
170128
assert agentResponse.get()["rate_by_service"] instanceof Map
171129
172130
where:

‎dd-trace-core/src/traceAgentTest/groovy/DataStreamsIntegrationTest.groovy‎

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -8,21 +8,16 @@ import datadog.trace.api.datastreams.StatsPoint
88
import datadog.trace.common.metrics.EventListener
99
import datadog.trace.common.metrics.OkHttpSink
1010
import datadog.trace.core.datastreams.DefaultDataStreamsMonitoring
11-
import datadog.trace.test.util.DDSpecification
1211
import okhttp3.HttpUrl
1312
import spock.lang.Ignore
14-
import spock.lang.Requires
1513
import spock.util.concurrent.PollingConditions
1614

1715
import java.util.concurrent.CopyOnWriteArrayList
1816

1917
import static datadog.trace.common.metrics.EventListener.EventType.OK
2018

21-
@Requires({
22-
"true" == System.getenv("CI")
23-
})
2419
@Ignore("The agent in CI doesn't have a valid API key. Unlike metrics and traces, data streams fails in this case")
25-
class DataStreamsIntegrationTest extends DDSpecification {
20+
class DataStreamsIntegrationTest extends AbstractTraceAgentTest {
2621

2722
def "Sending stats bucket to agent should notify with OK event"() {
2823
given:
@@ -49,10 +44,10 @@ class DataStreamsIntegrationTest extends DDSpecification {
4944
}
5045

5146
when:
52-
def dataStreams = new DefaultDataStreamsMonitoring(sink, sharedCommunicationObjects.featuresDiscovery, timeSource, { traceConfig }, Config.get())
47+
def dataStreams = new DefaultDataStreamsMonitoring(sink, sharedCommunicationObjects.featuresDiscovery(Config.get()), timeSource, { traceConfig }, Config.get())
5348
dataStreams.start()
54-
dataStreams.accept(new StatsPoint("testType", "testGroup", "testTopic", 1, 2, timeSource.currentTimeNanos, 0, 0, null))
55-
timeSource.advance(DEFAULT_BUCKET_DURATION_NANOS)
49+
dataStreams.add(new StatsPoint(["type:testType", "group:testGroup", "topic:testTopic"], 1, 2, 5, timeSource.currentTimeNanos, 0, 0, 0, null))
50+
timeSource.advance(Config.get().getDataStreamsBucketDurationNanoseconds())
5651
dataStreams.report()
5752

5853
then:

‎dd-trace-core/src/traceAgentTest/groovy/MetricsIntegrationTest.groovy‎

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,34 +1,29 @@
1-
21
import datadog.communication.ddagent.DDAgentFeaturesDiscovery
32
import datadog.communication.http.OkHttpUtils
3+
import datadog.trace.api.Config
44
import datadog.trace.api.WellKnownTags
55
import datadog.trace.common.metrics.AggregateMetric
66
import datadog.trace.common.metrics.EventListener
77
import datadog.trace.common.metrics.MetricKey
88
import datadog.trace.common.metrics.OkHttpSink
99
import datadog.trace.common.metrics.SerializingMetricWriter
10-
import datadog.trace.test.util.DDSpecification
11-
import spock.lang.Requires
10+
import okhttp3.HttpUrl
1211

1312
import java.util.concurrent.CopyOnWriteArrayList
1413
import java.util.concurrent.CountDownLatch
1514
import java.util.concurrent.atomic.AtomicLongArray
1615

1716
import static datadog.trace.common.metrics.EventListener.EventType.OK
1817
import static java.util.concurrent.TimeUnit.SECONDS
19-
import okhttp3.HttpUrl
2018

21-
@Requires({
22-
"true" == System.getenv("CI")
23-
})
24-
class MetricsIntegrationTest extends DDSpecification {
19+
class MetricsIntegrationTest extends AbstractTraceAgentTest {
2520

2621

2722
def "send metrics to trace agent should notify with OK event"() {
2823
setup:
2924
def latch = new CountDownLatch(1)
3025
def listener = new BlockingListener(latch)
31-
def agentUrl = "http://localhost:8126"
26+
def agentUrl = Config.get().getAgentUrl()
3227
OkHttpSink sink = new OkHttpSink(OkHttpUtils.buildHttpClient(HttpUrl.parse(agentUrl), 5000L), agentUrl, DDAgentFeaturesDiscovery.V6_METRICS_ENDPOINT, true, false, [:])
3328
sink.register(listener)
3429

0 commit comments

Comments
 (0)