Merge branch 'master' into malvarez/rasp-ssrf-with-url

smola · web-flow · commit 4d8ccf9360c7 · 2024-08-21T09:40:33.000+02:00
diff --git a/.circleci/config.continue.yml.j2 b/.circleci/config.continue.yml.j2
@@ -1057,7 +1057,7 @@ build_test_jobs: &build_test_jobs
       name: z_test_<< matrix.testJvm >>_base
       triggeredBy: *core_modules
       gradleTarget: ":baseTest"
-      gradleParameters: "-PskipFlakyTests -PskipInstTests -PskipSmokeTests -PskipProfilingTests -PskipDebuggerTests"
+      gradleParameters: "-PskipFlakyTests"
       stage: core
       cacheType: base
       parallelism: 4
@@ -1070,8 +1070,8 @@ build_test_jobs: &build_test_jobs
         - ok_to_test
       name: z_test_8_base
       triggeredBy: *core_modules
-      gradleTarget: :baseTest jacocoTestReport jacocoTestCoverageVerification
-      gradleParameters: "-PskipFlakyTests -PskipInstTests -PskipSmokeTests -PskipProfilingTests -PskipDebuggerTests"
+      gradleTarget: :baseTest
+      gradleParameters: "-PskipFlakyTests -PcheckCoverage"
       stage: core
       cacheType: base
       parallelism: 4
diff --git a/.circleci/upload_ciapp.sh b/.circleci/upload_ciapp.sh
@@ -15,6 +15,7 @@ java_prop () {
 
 # based on tracer implementation: https://github.com/DataDog/dd-trace-java/blob/master/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/TestDecorator.java#L55-L77
 datadog-ci junit upload --service $SERVICE_NAME \
+    --logs \
     --tags "test.traits:{\"marker\":[\"$1\"]}" \
     --tags "runtime.name:$(java_prop java.runtime.name)" \
     --tags "runtime.vendor:$(java_prop java.vendor)" \
diff --git a/.github/workflows/README.md b/.github/workflows/README.md
@@ -98,7 +98,7 @@ _Action:_ Run GitHub CodeQL action, upload result to GitHub security tab and Dat
 
 _Trigger:_ Every week or manually.
 
-_Action:_ Update the Grade dependencies and their locking files.
+_Action:_ Create a PR updating the Grade dependencies and their locking files.
 
 _Recovery:_ Manually trigger the action again.
 
diff --git a/.github/workflows/add-milestone-to-pull-requests.yaml b/.github/workflows/add-milestone-to-pull-requests.yaml
@@ -57,13 +57,14 @@ jobs:
                   return a.version.patch - b.version.patch
                 })
                 .pop()?.number
-            } else if (base.startsWith('release/v')) {
-              // Pick the milestone with the same title as the base branch
-              const version = base.substring(9)
+            } else if (base.startsWith('release/v') && base.endsWith('.x')) {
+              // Extract the minor version related to the base branch (e.g. "release/v1.2.x" -> "1.2.")
+              const version = base.substring(9, base.length - 1)
+              // Pick the milestone with title matching the extracted version
               const versionMilestone = response.data
-                .find(milestone => milestone.title == version)
+                .find(milestone => milestone.title.startsWith(version))
               if (!versionMilestone) {
-                core.setFailed(`Version milestone not found: ${version}`)
+                core.setFailed(`Milestone not found for minor version: ${version}`)
               } else {
                 milestoneNumber = versionMilestone.number
               }
diff --git a/.github/workflows/add-release-to-cloudfoundry.yaml b/.github/workflows/add-release-to-cloudfoundry.yaml
@@ -26,7 +26,7 @@ jobs:
           MAX_TRIES=60 # Wait up to 30 minutes
           DELAY=30
           while [ $TRY -lt $MAX_TRIES ]; do
-            if curl -s -I ${{ steps.get-release-url.outputs.URL }} | grep -q "200 OK"; then
+            if curl -s -I ${{ steps.get-release-url.outputs.URL }} | grep -q "^HTTP/.* 200"; then
               break
             fi
             echo "Waiting for the release to be available..."
diff --git a/.github/workflows/update-gradle-dependencies.yaml b/.github/workflows/update-gradle-dependencies.yaml
@@ -70,7 +70,7 @@ jobs:
           GH_TOKEN: ${{ github.token }}
         run: |
           gh pr create --title "Update Gradle dependencies" \
-            --body "This PR updates the Gradle dependencies.  \n:warning: Don't forget to squash commits before merging :warning:" \
+            --body "This PR updates the Gradle dependencies. ⚠️ Don't forget to squash commits before merging. ⚠️" \
             --base master \
             --head $BRANCH_NAME \
             --label "tag: dependencies" \
diff --git a/build.gradle b/build.gradle
@@ -165,8 +165,9 @@ allprojects { project ->
 }
 
 
-def testAggregate(String baseTaskName, includePrefixes, excludePrefixes, boolean coverage = false) {
+def testAggregate(String baseTaskName, includePrefixes, excludePrefixes, boolean forceCoverage = false) {
   def createRootTask = { rootTaskName, subProjTaskName ->
+    def coverage = forceCoverage || rootProject.hasProperty("checkCoverage")
     tasks.register(rootTaskName) { aggTest ->
       subprojects { subproject ->
         if (subproject.property("activePartition") && includePrefixes.any { subproject.path.startsWith(it) } && !excludePrefixes.any { subproject.path.startsWith(it) }) {
diff --git a/dd-java-agent/agent-debugger/build.gradle b/dd-java-agent/agent-debugger/build.gradle
@@ -73,15 +73,6 @@ jar {
   from sourceSets.main.output
 }
 
-tasks.withType(Test).configureEach {
-  onlyIf { !project.rootProject.hasProperty("skipDebuggerTests") }
-}
-
 // we want to test with no special reflective access (no --add-opens)
 ext.allowReflectiveAccessToJdk = false
 
-subprojects { Project subProj ->
-  subProj.tasks.withType(Test).configureEach { subTask ->
-    onlyIf { !project.rootProject.hasProperty("skipDebuggerTests") }
-  }
-}
diff --git a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/agent/ClassesToRetransformFinder.java b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/agent/ClassesToRetransformFinder.java
@@ -89,7 +89,10 @@ Trie getAllChangedClasses(ConfigurationComparer comparer) {
 
   private void processAdditionalClasses(String sourceFile, Trie changedClasses) {
     sourceFile = stripPackagePath(sourceFile);
-    List<String> additionalClasses = classNamesBySourceFile.get(sourceFile);
+    // need to clone the list to avoid concurrent modification during iteration
+    List<String> additionalClasses =
+        classNamesBySourceFile.computeIfPresent(
+            sourceFile, (k, classNames) -> new ArrayList<>(classNames));
     if (additionalClasses == null) {
       return;
     }
diff --git a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/symbol/SymDBEnablement.java b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/symbol/SymDBEnablement.java
@@ -13,6 +13,7 @@
 import datadog.trace.util.Strings;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
+import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.lang.instrument.Instrumentation;
@@ -160,11 +161,16 @@ private void extractSymbolForLoadedClasses() {
       if (!Files.exists(jarPath)) {
         continue;
       }
+      File jarPathFile = jarPath.toFile();
+      if (jarPathFile.isDirectory()) {
+        // we are not supporting class directories (classpath) but only jar files
+        continue;
+      }
       if (alreadyScannedJars.contains(jarPath.toString())) {
         continue;
       }
       try {
-        try (JarFile jarFile = new JarFile(jarPath.toFile())) {
+        try (JarFile jarFile = new JarFile(jarPathFile)) {
           jarFile.stream()
               .filter(jarEntry -> jarEntry.getName().endsWith(".class"))
               .filter(
diff --git a/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/trace/OtelTracer.java b/dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/trace/OtelTracer.java
@@ -9,7 +9,7 @@
 
 @ParametersAreNonnullByDefault
 class OtelTracer implements Tracer {
-  private static final String INSTRUMENTATION_NAME = "otel";
+  private static final String INSTRUMENTATION_NAME = otelInstrumentationName();
   private final AgentTracer.TracerAPI tracer;
   private final String instrumentationScopeName;
 
@@ -24,4 +24,12 @@ public SpanBuilder spanBuilder(String spanName) {
         this.tracer.buildSpan(INSTRUMENTATION_NAME, SPAN_KIND_INTERNAL).withResourceName(spanName);
     return new OtelSpanBuilder(delegate);
   }
+
+  @SuppressWarnings("ConstantConditions")
+  private static String otelInstrumentationName() {
+    // is this the bootstrap shim for drop-in support, or the shim for manual instrumentation?
+    return OtelTracer.class.getName().startsWith("datadog.trace.bootstrap")
+        ? "otel.library"
+        : "otel";
+  }
 }
diff --git a/dd-java-agent/agent-profiling/build.gradle b/dd-java-agent/agent-profiling/build.gradle
@@ -30,12 +30,6 @@ dependencies {
   testImplementation libs.bundles.mockito
 }
 
-subprojects { Project subProj ->
-  subProj.tasks.withType(Test).configureEach { subTask ->
-    onlyIf { !project.rootProject.hasProperty("skipProfilingTests") }
-  }
-}
-
 configurations {
   // exclude bootstrap dependencies from shadowJar
   runtime {
diff --git a/dd-java-agent/instrumentation/build.gradle b/dd-java-agent/instrumentation/build.gradle
@@ -97,8 +97,6 @@ subprojects { Project subProj ->
         useJUnit()
       }
 
-      onlyIf { !project.rootProject.hasProperty("skipInstTests") }
-
       if (subTask.name in ['latestDepTest', 'latestDepForkedTest']) {
         subTask.jvmArgs '-Dtest.dd.latestDepTest=true'
       }
diff --git a/dd-java-agent/instrumentation/couchbase/couchbase-3.1/build.gradle b/dd-java-agent/instrumentation/couchbase/couchbase-3.1/build.gradle
@@ -5,7 +5,7 @@ muzzle {
     module = 'java-client'
     versions = '[3.1,3.2)'
     // Version 2.7.5 and 2.7.8 were not released properly
-    skipVersions = ['2.7.5', '2.7.8']
+    skipVersions = ['2.7.5', '2.7.8', '3.6.3']
     assertInverse = true
   }
 }
diff --git a/dd-java-agent/instrumentation/couchbase/couchbase-3.2/build.gradle b/dd-java-agent/instrumentation/couchbase/couchbase-3.2/build.gradle
@@ -5,7 +5,7 @@ muzzle {
     module = 'java-client'
     versions = '[3.2,)'
     // Version 2.7.5 and 2.7.8 were not released properly
-    skipVersions = ['2.7.5', '2.7.8']
+    skipVersions = ['2.7.5', '2.7.8', '3.6.3']
     assertInverse = true
   }
 }
diff --git a/dd-java-agent/instrumentation/jdbc/src/main/java/datadog/trace/instrumentation/jdbc/AbstractPreparedStatementInstrumentation.java b/dd-java-agent/instrumentation/jdbc/src/main/java/datadog/trace/instrumentation/jdbc/AbstractPreparedStatementInstrumentation.java
@@ -102,14 +102,14 @@ public static AgentScope onEnter(@Advice.This final Statement statement) {
     @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class)
     public static void stopSpan(
         @Advice.Enter final AgentScope scope, @Advice.Thrown final Throwable throwable) {
+      CallDepthThreadLocalMap.decrementCallDepth(Statement.class);
       if (scope == null) {
         return;
       }
       DECORATE.onError(scope.span(), throwable);
       DECORATE.beforeFinish(scope.span());
       scope.close();
       scope.span().finish();
-      CallDepthThreadLocalMap.reset(Statement.class);
     }
   }
 }
diff --git a/dd-java-agent/instrumentation/jdbc/src/main/java/datadog/trace/instrumentation/jdbc/StatementInstrumentation.java b/dd-java-agent/instrumentation/jdbc/src/main/java/datadog/trace/instrumentation/jdbc/StatementInstrumentation.java
@@ -132,7 +132,6 @@ public static AgentScope onEnter(
         // if we can't get the connection for any reason
         return null;
       } catch (BlockingException e) {
-        CallDepthThreadLocalMap.reset(Statement.class);
         // re-throw blocking exceptions
         throw e;
       } catch (Throwable e) {
@@ -146,14 +145,14 @@ public static AgentScope onEnter(
     @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class)
     public static void stopSpan(
         @Advice.Enter final AgentScope scope, @Advice.Thrown final Throwable throwable) {
+      CallDepthThreadLocalMap.decrementCallDepth(Statement.class);
       if (scope == null) {
         return;
       }
       DECORATE.onError(scope.span(), throwable);
       DECORATE.beforeFinish(scope.span());
       scope.close();
       scope.span().finish();
-      CallDepthThreadLocalMap.reset(Statement.class);
     }
   }
 }
diff --git a/dd-smoke-tests/build.gradle b/dd-smoke-tests/build.gradle
@@ -19,8 +19,6 @@ subprojects { Project subProj ->
   subProj.tasks.withType(Test).configureEach { subTask ->
     dependsOn project(':dd-java-agent').tasks.named("shadowJar")
 
-    onlyIf { !project.rootProject.hasProperty("skipSmokeTests") }
-
     // Tests depend on this to know where to run things and what agent jar to use
     jvmArgs "-Ddatadog.smoketest.builddir=${buildDir}"
     jvmArgs "-Ddatadog.smoketest.agent.shadowJar.path=${project(':dd-java-agent').tasks.shadowJar.archiveFile.get()}"
diff --git a/docs/how_to_test.md b/docs/how_to_test.md
@@ -44,7 +44,7 @@ You can run the whole project test suite using `./gradlew test` but expect it to
 Instead, you can run test for a specific module (ex. `:dd-java-agent:instrumentation:opentelemetry:opentelemetry-1.4`) using the test command for this module only: `./gradlew :dd-java-agent:instrumentation:opentelemetry:opentelemetry-1.4:test`.
 
 > [!TIP]
-> Flaky tests can be disabled by setting the Gradle property `skipInstTests` (ex. `./gradlew -PskipFlakyTests <task>`).
+> Flaky tests can be disabled by setting the Gradle property `skipFlakyTests` (ex. `./gradlew -PskipFlakyTests <task>`).
 
 ### Running tests on another JVM
 
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
@@ -29,7 +29,7 @@ moshi = '1.11.0'
 testcontainers = '1.19.3'
 jmc = "8.1.0"
 autoservice = "1.0-rc7"
-ddprof = "1.12.0"
+ddprof = "1.13.0"
 asm = "9.7"
 cafe_crypto = "0.1.0"
 lz4 = "1.7.1"

Original file line number	Diff line number	Diff line change
`@@ -165,8 +165,9 @@ allprojects { project ->`
`165`	`165`	`}`
`166`	`166`
`167`	`167`
`168`		`-def testAggregate(String baseTaskName, includePrefixes, excludePrefixes, boolean coverage = false) {`
	`168`	`+def testAggregate(String baseTaskName, includePrefixes, excludePrefixes, boolean forceCoverage = false) {`
`169`	`169`	`def createRootTask = { rootTaskName, subProjTaskName ->`
	`170`	`+ def coverage = forceCoverage \|\| rootProject.hasProperty("checkCoverage")`
`170`	`171`	`tasks.register(rootTaskName) { aggTest ->`
`171`	`172`	`subprojects { subproject ->`
`172`	`173`	`if (subproject.property("activePartition") && includePrefixes.any { subproject.path.startsWith(it) } && !excludePrefixes.any { subproject.path.startsWith(it) }) {`