Skip to content

Commit d23ea24

Browse files
authored
Merge branch 'master' into mario.vidal/fix_replace
2 parents 1c08728 + d3cfbcb commit d23ea24

161 files changed

Lines changed: 2564 additions & 1090 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.circleci/config.continue.yml.j2

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ instrumentation_modules: &instrumentation_modules "dd-java-agent/instrumentation
3636
debugger_modules: &debugger_modules "dd-java-agent/agent-debugger|dd-java-agent/agent-bootstrap|dd-java-agent/agent-builder|internal-api|communication|dd-trace-core"
3737
profiling_modules: &profiling_modules "dd-java-agent/agent-profiling"
3838

39-
default_system_tests_commit: &default_system_tests_commit 8b05076e897fe62206d7704f2e8e650ed83ebd1f
39+
default_system_tests_commit: &default_system_tests_commit c706e333ef06800b866ac300e4b6cdb7566cc5e5
4040

4141
parameters:
4242
nightly:

.github/workflows/update-docker-build-image.yaml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ on:
99
tag:
1010
description: 'The tag to use for the Docker build image'
1111
required: true
12-
default: 'vYY.MM-base'
12+
default: 'vYY.MM'
1313

1414
jobs:
1515
update-docker-build-image:
@@ -46,7 +46,7 @@ jobs:
4646
08|09|10) TAG_DATE="${CURRENT_YEAR}.07" ;;
4747
11|12) TAG_DATE="${CURRENT_YEAR}.10" ;;
4848
esac
49-
TAG="v${TAG_DATE}-base"
49+
TAG="v${TAG_DATE}"
5050
fi
5151
echo "tag=${TAG}" >> "$GITHUB_OUTPUT"
5252
echo "::notice::Using Docker build image tag: ${TAG}"
@@ -55,7 +55,7 @@ jobs:
5555
sed -i 's|DOCKER_IMAGE_VERSION=.*|DOCKER_IMAGE_VERSION="${{ steps.define-tag.outputs.tag }}"|' .circleci/render_config.py
5656
- name: Update the Docker build image in GitLab CI config
5757
run: |
58-
sed -i 's|image: ghcr.io/datadog/dd-trace-java-docker-build:.*|image: ghcr.io/datadog/dd-trace-java-docker-build:${{ steps.define-tag.outputs.tag }}|' .gitlab-ci.yml
58+
sed -i 's|image: ghcr.io/datadog/dd-trace-java-docker-build:.*|image: ghcr.io/datadog/dd-trace-java-docker-build:${{ steps.define-tag.outputs.tag }}-base|' .gitlab-ci.yml
5959
- name: Commit and push changes
6060
env:
6161
GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}

.gitlab-ci.yml

Lines changed: 33 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -20,24 +20,34 @@ stages:
2020
variables:
2121
REGISTRY: 486234852809.dkr.ecr.us-east-1.amazonaws.com
2222
BUILD_JOB_NAME: "build"
23+
DEPENDENCY_CACHE_POLICY: pull
24+
BUILD_CACHE_POLICY: pull
25+
GRADLE_VERSION: "8.4" # must match gradle-wrapper.properties
2326

24-
.common: &common
25-
tags: [ "runner:main", "size:large" ]
27+
default:
28+
tags: [ "arch:amd64" ]
2629

2730
.gradle_build: &gradle_build
28-
<<: *common
29-
image: ghcr.io/datadog/dd-trace-java-docker-build:v23.10-base
31+
image: ghcr.io/datadog/dd-trace-java-docker-build:v24.10-base
3032
variables:
3133
GRADLE_OPTS: "-Dorg.gradle.jvmargs='-Xmx2560M -Xms2560M'"
32-
GRADLE_ARGS: " -PskipTests --build-cache --stacktrace --no-daemon --parallel --max-workers=8"
34+
GRADLE_ARGS: " -PskipTests --build-cache --stacktrace --no-daemon --parallel --max-workers=2"
3335
KUBERNETES_CPU_REQUEST: 8
34-
KUBERNETES_MEMORY_REQUEST: 4Gi
35-
cache: &default_cache
36-
key: '$CI_SERVER_VERSION' # Reset the cache every time gitlab is upgraded. ~Every couple months
37-
paths:
38-
- .gradle/wrapper
39-
- .gradle/caches
40-
policy: pull
36+
KUBERNETES_MEMORY_REQUEST: 6Gi
37+
cache:
38+
- key: '$CI_SERVER_VERSION-v2' # Dependencies cache. Reset the cache every time gitlab is upgraded. ~Every couple months
39+
paths:
40+
# Cached dependencies and wrappers for gradle
41+
- .gradle/wrapper
42+
- .gradle/caches
43+
- .gradle/notifications
44+
policy: $DEPENDENCY_CACHE_POLICY
45+
- key: $CI_PIPELINE_ID # Incremental build cache. Shared by all jobs in the pipeline
46+
paths:
47+
- .gradle/caches/$GRADLE_VERSION
48+
- .gradle/$GRADLE_VERSION/executionHistory
49+
- workspace
50+
policy: $BUILD_CACHE_POLICY
4151
before_script:
4252
- export GRADLE_USER_HOME=`pwd`/.gradle
4353
# for weird reasons, gradle will always "chmod 700" the .gradle folder
@@ -49,9 +59,12 @@ variables:
4959
- mv .gradle-copy .gradle
5060
- ls -la
5161

52-
build: &build
53-
<<: *gradle_build
62+
build:
63+
extends: .gradle_build
5464
stage: build
65+
variables:
66+
BUILD_CACHE_POLICY: push
67+
DEPENDENCY_CACHE_POLICY: pull
5568
script:
5669
- ./gradlew clean :dd-java-agent:shadowJar :dd-trace-api:jar :dd-trace-ot:shadowJar $GRADLE_ARGS
5770
- echo UPSTREAM_TRACER_VERSION=$(java -jar workspace/dd-java-agent/build/libs/*.jar) >> upstream.env
@@ -66,16 +79,16 @@ build: &build
6679
reports:
6780
dotenv: build.env
6881

69-
build_with_cache:
70-
<<: *build
82+
build_and_populate_dep_cache:
83+
extends: build
84+
variables:
85+
BUILD_CACHE_POLICY: push
86+
DEPENDENCY_CACHE_POLICY: push
7187
rules:
7288
- if: '$POPULATE_CACHE'
7389
when: on_success
7490
- when: manual
7591
allow_failure: true
76-
cache:
77-
<<: *default_cache
78-
policy: push
7992

8093
deploy_to_profiling_backend:
8194
stage: publish
@@ -149,7 +162,7 @@ deploy_to_di_backend:manual:
149162
UPSTREAM_COMMIT_SHORT_SHA: $CI_COMMIT_SHORT_SHA
150163

151164
deploy_to_sonatype:
152-
<<: *gradle_build
165+
extends: .gradle_build
153166
stage: publish
154167
needs: [ build ]
155168
rules:
@@ -177,7 +190,6 @@ deploy_to_sonatype:
177190
deploy_artifacts_to_github:
178191
stage: publish
179192
image: registry.ddbuild.io/github-cli:v27480869-eafb11d-2.43.0
180-
tags: [ "arch:amd64" ]
181193
rules:
182194
- if: '$POPULATE_CACHE'
183195
when: never
@@ -234,7 +246,6 @@ create_key:
234246
stage: generate-signing-key
235247
when: manual
236248
needs: [ ]
237-
tags: [ "arch:amd64", "size:large" ]
238249
variables:
239250
PROJECT_NAME: "dd-trace-java"
240251
EXPORT_TO_KEYSERVER: "true"

communication/src/main/java/datadog/communication/ddagent/SharedCommunicationObjects.java

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import datadog.remoteconfig.DefaultConfigurationPoller;
1212
import datadog.trace.api.Config;
1313
import datadog.trace.util.AgentTaskScheduler;
14+
import java.security.Security;
1415
import java.util.ArrayList;
1516
import java.util.List;
1617
import java.util.concurrent.TimeUnit;
@@ -59,6 +60,7 @@ public void createRemaining(Config config) {
5960
}
6061
}
6162

63+
/** Registers a callback to be called when remote communications resume. */
6264
public void whenReady(Runnable callback) {
6365
if (paused) {
6466
synchronized (pausedComponents) {
@@ -71,8 +73,15 @@ public void whenReady(Runnable callback) {
7173
callback.run(); // not paused, run immediately
7274
}
7375

76+
/** Resumes remote communications including any paused callbacks. */
7477
public void resume() {
7578
paused = false;
79+
// attempt discovery first to avoid potential race condition on IBM Java8
80+
if (null != featuresDiscovery) {
81+
featuresDiscovery.discoverIfOutdated();
82+
} else {
83+
Security.getProviders(); // fallback to preloading provider extensions
84+
}
7685
synchronized (pausedComponents) {
7786
for (Runnable callback : pausedComponents) {
7887
try {

dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java

Lines changed: 39 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package datadog.trace.bootstrap;
22

33
import static datadog.trace.api.ConfigDefaults.DEFAULT_STARTUP_LOGS_ENABLED;
4-
import static datadog.trace.api.Platform.getRuntimeVendor;
54
import static datadog.trace.api.Platform.isJavaVersionAtLeast;
65
import static datadog.trace.api.Platform.isOracleJDK8;
76
import static datadog.trace.bootstrap.Library.WILDFLY;
@@ -329,7 +328,7 @@ public void run() {
329328
if (appUsingCustomJMXBuilder) {
330329
log.debug("Custom JMX builder detected. Delaying JMXFetch initialization.");
331330
registerMBeanServerBuilderCallback(new StartJmxCallback(jmxStartDelay));
332-
// one minute fail-safe in case nothing touches JMX and and callback isn't triggered
331+
// one minute fail-safe in case nothing touches JMX and callback isn't triggered
333332
scheduleJmxStart(60 + jmxStartDelay);
334333
} else if (appUsingCustomLogManager) {
335334
log.debug("Custom logger detected. Delaying JMXFetch initialization.");
@@ -339,20 +338,31 @@ public void run() {
339338
}
340339
}
341340

342-
boolean delayOkHttp = appUsingCustomLogManager && okHttpMayIndirectlyLoadJUL();
343-
344341
/*
345342
* Similar thing happens with DatadogTracer on (at least) zulu-8 because it uses OkHttp which indirectly loads JFR
346343
* events which in turn loads LogManager. This is not a problem on newer JDKs because there JFR uses different
347344
* logging facility. Likewise on IBM JDKs OkHttp may indirectly load 'IBMSASL' which in turn loads LogManager.
348345
*/
346+
boolean delayOkHttp = !ciVisibilityEnabled && okHttpMayIndirectlyLoadJUL();
347+
boolean waitForJUL = appUsingCustomLogManager && delayOkHttp;
348+
int okHttpDelayMillis;
349+
if (waitForJUL) {
350+
okHttpDelayMillis = 1_000;
351+
} else if (delayOkHttp) {
352+
okHttpDelayMillis = 100;
353+
} else {
354+
okHttpDelayMillis = 0;
355+
}
356+
349357
InstallDatadogTracerCallback installDatadogTracerCallback =
350-
new InstallDatadogTracerCallback(initTelemetry, inst, delayOkHttp);
351-
if (delayOkHttp) {
358+
new InstallDatadogTracerCallback(initTelemetry, inst, okHttpDelayMillis);
359+
if (waitForJUL) {
352360
log.debug("Custom logger detected. Delaying Datadog Tracer initialization.");
353361
registerLogManagerCallback(installDatadogTracerCallback);
362+
} else if (okHttpDelayMillis > 0) {
363+
installDatadogTracerCallback.run(); // complete on different thread (after premain)
354364
} else {
355-
installDatadogTracerCallback.execute();
365+
installDatadogTracerCallback.execute(); // complete on primordial thread in premain
356366
}
357367

358368
/*
@@ -362,7 +372,7 @@ public void run() {
362372
if (profilingEnabled && !isOracleJDK8()) {
363373
StaticEventLogger.begin("Profiling");
364374

365-
if (delayOkHttp) {
375+
if (waitForJUL) {
366376
log.debug("Custom logger detected. Delaying Profiling initialization.");
367377
registerLogManagerCallback(new StartProfilingAgentCallback(inst));
368378
} else {
@@ -499,18 +509,18 @@ protected static class InstallDatadogTracerCallback extends ClassLoadCallBack {
499509
private final Instrumentation instrumentation;
500510
private final Object sco;
501511
private final Class<?> scoClass;
502-
private final boolean delayOkHttp;
512+
private final int okHttpDelayMillis;
503513

504514
public InstallDatadogTracerCallback(
505515
InitializationTelemetry initTelemetry,
506516
Instrumentation instrumentation,
507-
boolean delayOkHttp) {
508-
this.delayOkHttp = delayOkHttp;
517+
int okHttpDelayMillis) {
518+
this.okHttpDelayMillis = okHttpDelayMillis;
509519
this.instrumentation = instrumentation;
510520
try {
511521
scoClass =
512522
AGENT_CLASSLOADER.loadClass("datadog.communication.ddagent.SharedCommunicationObjects");
513-
sco = scoClass.getConstructor(boolean.class).newInstance(delayOkHttp);
523+
sco = scoClass.getConstructor(boolean.class).newInstance(okHttpDelayMillis > 0);
514524
} catch (ClassNotFoundException
515525
| NoSuchMethodException
516526
| InstantiationException
@@ -521,6 +531,7 @@ public InstallDatadogTracerCallback(
521531

522532
installDatadogTracer(initTelemetry, scoClass, sco);
523533
maybeInstallLogsIntake(scoClass, sco);
534+
maybeStartIast(instrumentation);
524535
}
525536

526537
@Override
@@ -530,12 +541,11 @@ public AgentThread agentThread() {
530541

531542
@Override
532543
public void execute() {
533-
if (delayOkHttp) {
544+
if (okHttpDelayMillis > 0) {
534545
resumeRemoteComponents();
535546
}
536547

537548
maybeStartAppSec(scoClass, sco);
538-
maybeStartIast(instrumentation, scoClass, sco);
539549
maybeStartCiVisibility(instrumentation, scoClass, sco);
540550
// start debugger before remote config to subscribe to it before starting to poll
541551
maybeStartDebugger(instrumentation, scoClass, sco);
@@ -550,7 +560,7 @@ private void resumeRemoteComponents() {
550560
try {
551561
// remote components were paused for custom log-manager/jmx-builder
552562
// add small delay before resuming remote I/O to help stabilization
553-
Thread.sleep(1_000);
563+
Thread.sleep(okHttpDelayMillis);
554564
scoClass.getMethod("resume").invoke(sco);
555565
} catch (InterruptedException ignore) {
556566
} catch (Throwable e) {
@@ -837,14 +847,14 @@ private static boolean isSupportedAppSecArch() {
837847
return true;
838848
}
839849

840-
private static void maybeStartIast(Instrumentation instrumentation, Class<?> scoClass, Object o) {
850+
private static void maybeStartIast(Instrumentation instrumentation) {
841851
if (iastEnabled || !iastFullyDisabled) {
842852

843853
StaticEventLogger.begin("IAST");
844854

845855
try {
846856
SubscriptionService ss = AgentTracer.get().getSubscriptionService(RequestContextSlot.IAST);
847-
startIast(instrumentation, ss, scoClass, o);
857+
startIast(instrumentation, ss);
848858
} catch (Exception e) {
849859
log.error("Error starting IAST subsystem", e);
850860
}
@@ -853,8 +863,7 @@ private static void maybeStartIast(Instrumentation instrumentation, Class<?> sco
853863
}
854864
}
855865

856-
private static void startIast(
857-
Instrumentation instrumentation, SubscriptionService ss, Class<?> scoClass, Object sco) {
866+
private static void startIast(Instrumentation instrumentation, SubscriptionService ss) {
858867
try {
859868
final Class<?> appSecSysClass = AGENT_CLASSLOADER.loadClass("com.datadog.iast.IastSystem");
860869
final Method iastInstallerMethod =
@@ -1339,15 +1348,24 @@ private static String ddGetEnv(final String sysProp) {
13391348
}
13401349

13411350
private static boolean okHttpMayIndirectlyLoadJUL() {
1342-
if ("IBM Corporation".equals(getRuntimeVendor())) {
1343-
return true; // IBM JDKs ship with 'IBMSASL' which will load JUL when OkHttp accesses TLS
1351+
if (isCustomSecurityProviderInstalled() || isIBMSASLInstalled()) {
1352+
return true; // custom security providers may load JUL when OkHttp accesses TLS
13441353
}
13451354
if (isJavaVersionAtLeast(9)) {
13461355
return false; // JDKs since 9 have reworked JFR to use a different logging facility, not JUL
13471356
}
13481357
return isJFRSupported(); // assume OkHttp will indirectly load JUL via its JFR events
13491358
}
13501359

1360+
private static boolean isCustomSecurityProviderInstalled() {
1361+
return ClassLoader.getSystemResource("META-INF/services/java.security.Provider") != null;
1362+
}
1363+
1364+
private static boolean isIBMSASLInstalled() {
1365+
// need explicit check as this is installed without using the service-loader mechanism
1366+
return ClassLoader.getSystemResource("com/ibm/security/sasl/IBMSASL.class") != null;
1367+
}
1368+
13511369
private static boolean isJFRSupported() {
13521370
// FIXME: this is quite a hack because there maybe jfr classes on classpath somehow that have
13531371
// nothing to do with JDK - but this should be safe because only thing this does is to delay

dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/java/concurrent/QueueTimerHelper.java

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,23 +30,31 @@ private static final class RateLimiterHolder {
3030
}
3131

3232
public static <T> void startQueuingTimer(
33-
ContextStore<T, State> taskContextStore, Class<?> schedulerClass, T task) {
33+
ContextStore<T, State> taskContextStore,
34+
Class<?> schedulerClass,
35+
Class<?> queueClass,
36+
int queueLength,
37+
T task) {
3438
State state = taskContextStore.get(task);
35-
startQueuingTimer(state, schedulerClass, task);
39+
startQueuingTimer(state, schedulerClass, queueClass, queueLength, task);
3640
}
3741

38-
public static void startQueuingTimer(State state, Class<?> schedulerClass, Object task) {
42+
public static void startQueuingTimer(
43+
State state, Class<?> schedulerClass, Class<?> queueClass, int queueLength, Object task) {
3944
if (Platform.isNativeImage()) {
4045
// explicitly not supported for Graal native image
4146
return;
4247
}
48+
// TODO consider queue length based sampling here to reduce overhead
4349
// avoid calling this before JFR is initialised because it will lead to reading the wrong
4450
// TSC frequency before JFR has set it up properly
4551
if (task != null && state != null && InstrumentationBasedProfiling.isJFRReady()) {
4652
QueueTiming timing =
4753
(QueueTiming) AgentTracer.get().getProfilingContext().start(Timer.TimerType.QUEUEING);
4854
timing.setTask(task);
4955
timing.setScheduler(schedulerClass);
56+
timing.setQueue(queueClass);
57+
timing.setQueueLength(queueLength);
5058
state.setTiming(timing);
5159
}
5260
}

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ConfigurationApi.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ public CiVisibilitySettings getSettings(TracerEnvironment tracerEnvironment) {
1818

1919
@Override
2020
public SkippableTests getSkippableTests(TracerEnvironment tracerEnvironment) {
21-
return new SkippableTests(null, Collections.emptyMap(), null);
21+
return SkippableTests.EMPTY;
2222
}
2323

2424
@Override

0 commit comments

Comments
 (0)