Skip to content

Commit 7ec2cb4

Browse files
MariovidoMariovido
authored
Merge branch 'master' into mario.vidal/span_rasp_timeout
2 parents 672a97e + d3cfbcb commit 7ec2cb4

106 files changed

Lines changed: 1834 additions & 681 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.gitlab-ci.yml‎

Lines changed: 33 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -20,24 +20,34 @@ stages:
2020
variables:
2121
REGISTRY: 486234852809.dkr.ecr.us-east-1.amazonaws.com
2222
BUILD_JOB_NAME: "build"
23+
DEPENDENCY_CACHE_POLICY: pull
24+
BUILD_CACHE_POLICY: pull
25+
GRADLE_VERSION: "8.4" # must match gradle-wrapper.properties
2326

24-
.common: &common
25-
tags: [ "runner:main", "size:large" ]
27+
default:
28+
tags: [ "arch:amd64" ]
2629

2730
.gradle_build: &gradle_build
28-
<<: *common
29-
image: ghcr.io/datadog/dd-trace-java-docker-build:v23.10-base
31+
image: ghcr.io/datadog/dd-trace-java-docker-build:v24.10-base
3032
variables:
3133
GRADLE_OPTS: "-Dorg.gradle.jvmargs='-Xmx2560M -Xms2560M'"
32-
GRADLE_ARGS: " -PskipTests --build-cache --stacktrace --no-daemon --parallel --max-workers=8"
34+
GRADLE_ARGS: " -PskipTests --build-cache --stacktrace --no-daemon --parallel --max-workers=2"
3335
KUBERNETES_CPU_REQUEST: 8
34-
KUBERNETES_MEMORY_REQUEST: 4Gi
35-
cache: &default_cache
36-
key: '$CI_SERVER_VERSION' # Reset the cache every time gitlab is upgraded. ~Every couple months
37-
paths:
38-
- .gradle/wrapper
39-
- .gradle/caches
40-
policy: pull
36+
KUBERNETES_MEMORY_REQUEST: 6Gi
37+
cache:
38+
- key: '$CI_SERVER_VERSION-v2' # Dependencies cache. Reset the cache every time gitlab is upgraded. ~Every couple months
39+
paths:
40+
# Cached dependencies and wrappers for gradle
41+
- .gradle/wrapper
42+
- .gradle/caches
43+
- .gradle/notifications
44+
policy: $DEPENDENCY_CACHE_POLICY
45+
- key: $CI_PIPELINE_ID # Incremental build cache. Shared by all jobs in the pipeline
46+
paths:
47+
- .gradle/caches/$GRADLE_VERSION
48+
- .gradle/$GRADLE_VERSION/executionHistory
49+
- workspace
50+
policy: $BUILD_CACHE_POLICY
4151
before_script:
4252
- export GRADLE_USER_HOME=`pwd`/.gradle
4353
# for weird reasons, gradle will always "chmod 700" the .gradle folder
@@ -49,9 +59,12 @@ variables:
4959
- mv .gradle-copy .gradle
5060
- ls -la
5161

52-
build: &build
53-
<<: *gradle_build
62+
build:
63+
extends: .gradle_build
5464
stage: build
65+
variables:
66+
BUILD_CACHE_POLICY: push
67+
DEPENDENCY_CACHE_POLICY: pull
5568
script:
5669
- ./gradlew clean :dd-java-agent:shadowJar :dd-trace-api:jar :dd-trace-ot:shadowJar $GRADLE_ARGS
5770
- echo UPSTREAM_TRACER_VERSION=$(java -jar workspace/dd-java-agent/build/libs/*.jar) >> upstream.env
@@ -66,16 +79,16 @@ build: &build
6679
reports:
6780
dotenv: build.env
6881

69-
build_with_cache:
70-
<<: *build
82+
build_and_populate_dep_cache:
83+
extends: build
84+
variables:
85+
BUILD_CACHE_POLICY: push
86+
DEPENDENCY_CACHE_POLICY: push
7187
rules:
7288
- if: '$POPULATE_CACHE'
7389
when: on_success
7490
- when: manual
7591
allow_failure: true
76-
cache:
77-
<<: *default_cache
78-
policy: push
7992

8093
deploy_to_profiling_backend:
8194
stage: publish
@@ -149,7 +162,7 @@ deploy_to_di_backend:manual:
149162
UPSTREAM_COMMIT_SHORT_SHA: $CI_COMMIT_SHORT_SHA
150163

151164
deploy_to_sonatype:
152-
<<: *gradle_build
165+
extends: .gradle_build
153166
stage: publish
154167
needs: [ build ]
155168
rules:
@@ -177,7 +190,6 @@ deploy_to_sonatype:
177190
deploy_artifacts_to_github:
178191
stage: publish
179192
image: registry.ddbuild.io/github-cli:v27480869-eafb11d-2.43.0
180-
tags: [ "arch:amd64" ]
181193
rules:
182194
- if: '$POPULATE_CACHE'
183195
when: never
@@ -234,7 +246,6 @@ create_key:
234246
stage: generate-signing-key
235247
when: manual
236248
needs: [ ]
237-
tags: [ "arch:amd64", "size:large" ]
238249
variables:
239250
PROJECT_NAME: "dd-trace-java"
240251
EXPORT_TO_KEYSERVER: "true"

‎dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java‎

Lines changed: 11 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -531,6 +531,7 @@ public InstallDatadogTracerCallback(
531531

532532
installDatadogTracer(initTelemetry, scoClass, sco);
533533
maybeInstallLogsIntake(scoClass, sco);
534+
maybeStartIast(instrumentation);
534535
}
535536

536537
@Override
@@ -545,7 +546,6 @@ public void execute() {
545546
}
546547

547548
maybeStartAppSec(scoClass, sco);
548-
maybeStartIast(instrumentation, scoClass, sco);
549549
maybeStartCiVisibility(instrumentation, scoClass, sco);
550550
// start debugger before remote config to subscribe to it before starting to poll
551551
maybeStartDebugger(instrumentation, scoClass, sco);
@@ -847,14 +847,14 @@ private static boolean isSupportedAppSecArch() {
847847
return true;
848848
}
849849

850-
private static void maybeStartIast(Instrumentation instrumentation, Class<?> scoClass, Object o) {
850+
private static void maybeStartIast(Instrumentation instrumentation) {
851851
if (iastEnabled || !iastFullyDisabled) {
852852

853853
StaticEventLogger.begin("IAST");
854854

855855
try {
856856
SubscriptionService ss = AgentTracer.get().getSubscriptionService(RequestContextSlot.IAST);
857-
startIast(instrumentation, ss, scoClass, o);
857+
startIast(instrumentation, ss);
858858
} catch (Exception e) {
859859
log.error("Error starting IAST subsystem", e);
860860
}
@@ -863,8 +863,7 @@ private static void maybeStartIast(Instrumentation instrumentation, Class<?> sco
863863
}
864864
}
865865

866-
private static void startIast(
867-
Instrumentation instrumentation, SubscriptionService ss, Class<?> scoClass, Object sco) {
866+
private static void startIast(Instrumentation instrumentation, SubscriptionService ss) {
868867
try {
869868
final Class<?> appSecSysClass = AGENT_CLASSLOADER.loadClass("com.datadog.iast.IastSystem");
870869
final Method iastInstallerMethod =
@@ -1349,23 +1348,22 @@ private static String ddGetEnv(final String sysProp) {
13491348
}
13501349

13511350
private static boolean okHttpMayIndirectlyLoadJUL() {
1352-
if (isIBMSASLInstalled() || isACCPInstalled()) {
1353-
return true; // 'IBMSASL' and 'ACCP' crypto providers can load JUL when OkHttp accesses TLS
1351+
if (isCustomSecurityProviderInstalled() || isIBMSASLInstalled()) {
1352+
return true; // custom security providers may load JUL when OkHttp accesses TLS
13541353
}
13551354
if (isJavaVersionAtLeast(9)) {
13561355
return false; // JDKs since 9 have reworked JFR to use a different logging facility, not JUL
13571356
}
13581357
return isJFRSupported(); // assume OkHttp will indirectly load JUL via its JFR events
13591358
}
13601359

1361-
private static boolean isIBMSASLInstalled() {
1362-
return ClassLoader.getSystemResource("com/ibm/security/sasl/IBMSASL.class") != null;
1360+
private static boolean isCustomSecurityProviderInstalled() {
1361+
return ClassLoader.getSystemResource("META-INF/services/java.security.Provider") != null;
13631362
}
13641363

1365-
private static boolean isACCPInstalled() {
1366-
return ClassLoader.getSystemResource(
1367-
"com/amazon/corretto/crypto/provider/AmazonCorrettoCryptoProvider.class")
1368-
!= null;
1364+
private static boolean isIBMSASLInstalled() {
1365+
// need explicit check as this is installed without using the service-loader mechanism
1366+
return ClassLoader.getSystemResource("com/ibm/security/sasl/IBMSASL.class") != null;
13691367
}
13701368

13711369
private static boolean isJFRSupported() {

‎dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/java/concurrent/QueueTimerHelper.java‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,23 +30,31 @@ private static final class RateLimiterHolder {
3030
}
3131

3232
public static <T> void startQueuingTimer(
33-
ContextStore<T, State> taskContextStore, Class<?> schedulerClass, T task) {
33+
ContextStore<T, State> taskContextStore,
34+
Class<?> schedulerClass,
35+
Class<?> queueClass,
36+
int queueLength,
37+
T task) {
3438
State state = taskContextStore.get(task);
35-
startQueuingTimer(state, schedulerClass, task);
39+
startQueuingTimer(state, schedulerClass, queueClass, queueLength, task);
3640
}
3741

38-
public static void startQueuingTimer(State state, Class<?> schedulerClass, Object task) {
42+
public static void startQueuingTimer(
43+
State state, Class<?> schedulerClass, Class<?> queueClass, int queueLength, Object task) {
3944
if (Platform.isNativeImage()) {
4045
// explicitly not supported for Graal native image
4146
return;
4247
}
48+
// TODO consider queue length based sampling here to reduce overhead
4349
// avoid calling this before JFR is initialised because it will lead to reading the wrong
4450
// TSC frequency before JFR has set it up properly
4551
if (task != null && state != null && InstrumentationBasedProfiling.isJFRReady()) {
4652
QueueTiming timing =
4753
(QueueTiming) AgentTracer.get().getProfilingContext().start(Timer.TimerType.QUEUEING);
4854
timing.setTask(task);
4955
timing.setScheduler(schedulerClass);
56+
timing.setQueue(queueClass);
57+
timing.setQueueLength(queueLength);
5058
state.setTiming(timing);
5159
}
5260
}

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ConfigurationApi.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ public CiVisibilitySettings getSettings(TracerEnvironment tracerEnvironment) {
1818

1919
@Override
2020
public SkippableTests getSkippableTests(TracerEnvironment tracerEnvironment) {
21-
return new SkippableTests(null, Collections.emptyMap(), null);
21+
return SkippableTests.EMPTY;
2222
}
2323

2424
@Override

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ConfigurationApiImpl.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import java.util.Base64;
3030
import java.util.BitSet;
3131
import java.util.Collection;
32+
import java.util.Collections;
3233
import java.util.HashMap;
3334
import java.util.HashSet;
3435
import java.util.List;
@@ -191,7 +192,9 @@ public SkippableTests getSkippableTests(TracerEnvironment tracerEnvironment) thr
191192

192193
String correlationId = response.meta != null ? response.meta.correlation_id : null;
193194
Map<String, BitSet> coveredLinesByRelativeSourcePath =
194-
response.meta != null ? response.meta.coverage : null;
195+
response.meta != null && response.meta.coverage != null
196+
? response.meta.coverage
197+
: Collections.emptyMap();
195198
return new SkippableTests(
196199
correlationId, testIdentifiersByModule, coveredLinesByRelativeSourcePath);
197200
}

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettings.java‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ public class ExecutionSettings {
4040
@Nonnull private final EarlyFlakeDetectionSettings earlyFlakeDetectionSettings;
4141
@Nullable private final String itrCorrelationId;
4242
@Nonnull private final Map<TestIdentifier, TestMetadata> skippableTests;
43-
@Nullable private final Map<String, BitSet> skippableTestsCoverage;
43+
@Nonnull private final Map<String, BitSet> skippableTestsCoverage;
4444
@Nullable private final Collection<TestIdentifier> flakyTests;
4545
@Nullable private final Collection<TestIdentifier> knownTests;
4646
@Nonnull private final Diff pullRequestDiff;
@@ -54,7 +54,7 @@ public ExecutionSettings(
5454
@Nonnull EarlyFlakeDetectionSettings earlyFlakeDetectionSettings,
5555
@Nullable String itrCorrelationId,
5656
@Nonnull Map<TestIdentifier, TestMetadata> skippableTests,
57-
@Nullable Map<String, BitSet> skippableTestsCoverage,
57+
@Nonnull Map<String, BitSet> skippableTestsCoverage,
5858
@Nullable Collection<TestIdentifier> flakyTests,
5959
@Nullable Collection<TestIdentifier> knownTests,
6060
@Nonnull Diff pullRequestDiff) {
@@ -107,7 +107,7 @@ public String getItrCorrelationId() {
107107
}
108108

109109
/** A bit vector of covered lines by relative source file path. */
110-
@Nullable
110+
@Nonnull
111111
public Map<String, BitSet> getSkippableTestsCoverage() {
112112
return skippableTestsCoverage;
113113
}
@@ -126,6 +126,10 @@ public Collection<TestIdentifier> getKnownTests() {
126126
return knownTests;
127127
}
128128

129+
/**
130+
* @return the list of flaky tests for the given module (can be empty), or {@code null} if flaky
131+
* tests could not be obtained
132+
*/
129133
@Nullable
130134
public Collection<TestIdentifier> getFlakyTests() {
131135
return flakyTests;

0 commit comments

Comments
 (0)