Skip to content

Commit 89a5993

Browse files
evanchoolyevanchooly
authored
Merge branch 'master' into evanchooly/DEBUG-3354
2 parents 7376748 + ac29d6c commit 89a5993

584 files changed

Lines changed: 60916 additions & 52486 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.circleci/collect_results.sh‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
# This folder will be saved by circleci and available after test runs.
55

66
set -e
7-
#Enable '**' support
7+
# Enable '**' support
88
shopt -s globstar
99

1010
TEST_RESULTS_DIR=results
@@ -19,13 +19,36 @@ if [[ ${#TEST_RESULT_DIRS[@]} -eq 0 ]]; then
1919
exit 0
2020
fi
2121

22+
function get_source_file () {
23+
file_path="${RESULT_XML_FILE%%"/build"*}"
24+
file_path="${file_path/#"$WORKSPACE_DIR"\//}/src"
25+
if ! [[ $RESULT_XML_FILE == *"#"* ]]; then
26+
class="${RESULT_XML_FILE%.xml}"
27+
class="${class##*"TEST-"}"
28+
class="${class##*"."}"
29+
common_root=$(grep -rl "class $class" "$file_path" | head -n 1)
30+
while IFS= read -r line; do
31+
while [[ $line != "$common_root"* ]]; do
32+
common_root=$(dirname "$common_root")
33+
if [[ "$common_root" == "$common_root/.." ]]; then
34+
break
35+
fi
36+
done
37+
done < <(grep -rl "class $class" "$file_path")
38+
file_path="$common_root"
39+
fi
40+
}
41+
2242
echo "Saving test results:"
2343
while IFS= read -r -d '' RESULT_XML_FILE
2444
do
2545
echo -n "- $RESULT_XML_FILE"
2646
AGGREGATED_FILE_NAME=$(echo "$RESULT_XML_FILE" | rev | cut -d "/" -f 1,2,5 | rev | tr "/" "_")
2747
echo -n " as $AGGREGATED_FILE_NAME"
2848
cp "$RESULT_XML_FILE" "$TEST_RESULTS_DIR/$AGGREGATED_FILE_NAME"
49+
# Insert file attribute to testcase XML nodes
50+
get_source_file
51+
sed -i "/<testcase/ s|\(time=\"[^\"]*\"\)|\1 file=\"$file_path\"|g" "$TEST_RESULTS_DIR/$AGGREGATED_FILE_NAME"
2952
# Replace Java Object hashCode by marker in testcase XML nodes to get stable test names
3053
sed -i '/<testcase/ s/@[0-9a-f]\{5,\}/@HASHCODE/g' "$TEST_RESULTS_DIR/$AGGREGATED_FILE_NAME"
3154
# Replace random port numbers by marker in testcase XML nodes to get stable test names

‎.circleci/config.continue.yml.j2‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ instrumentation_modules: &instrumentation_modules "dd-java-agent/instrumentation
3636
debugger_modules: &debugger_modules "dd-java-agent/agent-debugger|dd-java-agent/agent-bootstrap|dd-java-agent/agent-builder|internal-api|communication|dd-trace-core"
3737
profiling_modules: &profiling_modules "dd-java-agent/agent-profiling"
3838

39-
default_system_tests_commit: &default_system_tests_commit 2cb5033072c1813eccdce332153a378607e73d85
39+
default_system_tests_commit: &default_system_tests_commit 8b05076e897fe62206d7704f2e8e650ed83ebd1f
4040

4141
parameters:
4242
nightly:

‎components/context/src/main/java/datadog/context/Context.java‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,7 @@ static Context detachFrom(Object carrier) {
114114
/**
115115
* Creates a copy of this context with the given key-value set.
116116
*
117-
* <p>Existing value with the given key will be replaced, and mapping to a {@code null} value will
117+
* <p>Existing value with the given key will be replaced. Mapping to a {@code null} value will
118118
* remove the key-value from the context copy.
119119
*
120120
* @param <T> the type of the value.
@@ -124,6 +124,28 @@ static Context detachFrom(Object carrier) {
124124
*/
125125
<T> Context with(ContextKey<T> key, @Nullable T value);
126126

127+
/**
128+
* Creates a copy of this context with the given pair of key-values.
129+
*
130+
* <p>Existing values with the given keys will be replaced. Mapping to a {@code null} value will
131+
* remove the key-value from the context copy.
132+
*
133+
* @param <T> the type of the first value.
134+
* @param <U> the type of the second value.
135+
* @param firstKey the first key to store the first value.
136+
* @param firstValue the first value to store.
137+
* @param secondKey the second key to store the second value.
138+
* @param secondValue the second value to store.
139+
* @return a new context with the pair of key-values set.
140+
*/
141+
default <T, U> Context with(
142+
ContextKey<T> firstKey,
143+
@Nullable T firstValue,
144+
ContextKey<U> secondKey,
145+
@Nullable U secondValue) {
146+
return with(firstKey, firstValue).with(secondKey, secondValue);
147+
}
148+
127149
/**
128150
* Creates a copy of this context with the implicit key is mapped to the value.
129151
*

‎components/context/src/test/java/datadog/context/ContextTest.java‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,41 @@ void testWith(Context context) {
6363
assertDoesNotThrow(() -> context.with(null), "Null implicitly keyed value not throw exception");
6464
}
6565

66+
@ParameterizedTest
67+
@MethodSource("contextImplementations")
68+
void testWithPair(Context context) {
69+
// Test retrieving value
70+
String stringValue = "value";
71+
Context context1 = context.with(BOOLEAN_KEY, false, STRING_KEY, stringValue);
72+
assertEquals(stringValue, context1.get(STRING_KEY));
73+
assertEquals(false, context1.get(BOOLEAN_KEY));
74+
// Test overriding value
75+
String stringValue2 = "value2";
76+
Context context2 = context1.with(STRING_KEY, stringValue2, BOOLEAN_KEY, true);
77+
assertEquals(stringValue2, context2.get(STRING_KEY));
78+
assertEquals(true, context2.get(BOOLEAN_KEY));
79+
// Test clearing value
80+
Context context3 = context2.with(BOOLEAN_KEY, null, STRING_KEY, null);
81+
assertNull(context3.get(STRING_KEY));
82+
assertNull(context3.get(BOOLEAN_KEY));
83+
// Test null key handling
84+
assertThrows(
85+
NullPointerException.class,
86+
() -> context.with(null, "test", STRING_KEY, "test"),
87+
"Context forbids null keys");
88+
assertThrows(
89+
NullPointerException.class,
90+
() -> context.with(STRING_KEY, "test", null, "test"),
91+
"Context forbids null keys");
92+
// Test null value handling
93+
assertDoesNotThrow(
94+
() -> context.with(BOOLEAN_KEY, null, STRING_KEY, "test"),
95+
"Null value should not throw exception");
96+
assertDoesNotThrow(
97+
() -> context.with(STRING_KEY, "test", BOOLEAN_KEY, null),
98+
"Null value should not throw exception");
99+
}
100+
66101
@ParameterizedTest
67102
@MethodSource("contextImplementations")
68103
void testGet(Context original) {

‎dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Constants.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ public final class Constants {
1515
*/
1616
public static final String[] BOOTSTRAP_PACKAGE_PREFIXES = {
1717
"datadog.slf4j",
18+
"datadog.context",
1819
"datadog.appsec.api",
1920
"datadog.trace.api",
2021
"datadog.trace.bootstrap",

‎dd-java-agent/agent-ci-visibility/build.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ dependencies {
4343
testFixturesApi project(':utils:test-utils')
4444

4545
testFixturesApi group: 'org.skyscreamer', name: 'jsonassert', version: '1.5.1'
46-
testFixturesApi group: 'org.freemarker', name: 'freemarker', version: '2.3.30'
46+
testFixturesApi group: 'org.freemarker', name: 'freemarker', version: '2.3.31'
4747
testFixturesApi group: 'com.jayway.jsonpath', name: 'json-path', version: '2.8.0'
4848
testFixturesApi group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.16.0'
4949
testFixturesApi group: 'org.msgpack', name: 'jackson-dataformat-msgpack', version: '0.9.6'

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilityRepoServices.java‎

Lines changed: 72 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import datadog.trace.civisibility.ci.CIInfo;
99
import datadog.trace.civisibility.ci.CIProviderInfo;
1010
import datadog.trace.civisibility.ci.CITagsProvider;
11+
import datadog.trace.civisibility.ci.PullRequestInfo;
1112
import datadog.trace.civisibility.codeowners.Codeowners;
1213
import datadog.trace.civisibility.codeowners.CodeownersProvider;
1314
import datadog.trace.civisibility.codeowners.NoCodeowners;
@@ -22,6 +23,7 @@
2223
import datadog.trace.civisibility.git.tree.GitDataApi;
2324
import datadog.trace.civisibility.git.tree.GitDataUploader;
2425
import datadog.trace.civisibility.git.tree.GitDataUploaderImpl;
26+
import datadog.trace.civisibility.git.tree.GitRepoUnshallow;
2527
import datadog.trace.civisibility.ipc.ExecutionSettingsRequest;
2628
import datadog.trace.civisibility.ipc.ExecutionSettingsResponse;
2729
import datadog.trace.civisibility.ipc.SignalClient;
@@ -31,6 +33,8 @@
3133
import datadog.trace.civisibility.source.SourcePathResolver;
3234
import datadog.trace.civisibility.source.index.RepoIndexProvider;
3335
import datadog.trace.civisibility.source.index.RepoIndexSourcePathResolver;
36+
import datadog.trace.util.Strings;
37+
import java.io.File;
3438
import java.nio.file.Path;
3539
import java.nio.file.Paths;
3640
import java.util.Map;
@@ -59,16 +63,26 @@ public class CiVisibilityRepoServices {
5963
ciProvider = ciProviderInfo.getProvider();
6064

6165
CIInfo ciInfo = ciProviderInfo.buildCIInfo();
62-
repoRoot = ciInfo.getNormalizedCiWorkspace();
63-
moduleName = getModuleName(services.config, path, ciInfo);
64-
ciTags = new CITagsProvider().getCiTags(ciInfo);
66+
PullRequestInfo pullRequestInfo = ciProviderInfo.buildPullRequestInfo();
67+
68+
if (pullRequestInfo.isNotEmpty()) {
69+
LOGGER.info("PR detected: {}", pullRequestInfo);
70+
}
71+
72+
repoRoot = appendSlashIfNeeded(getRepoRoot(ciInfo, services.gitClientFactory));
73+
moduleName = getModuleName(services.config, repoRoot, path);
74+
ciTags = new CITagsProvider().getCiTags(ciInfo, pullRequestInfo);
75+
76+
GitClient gitClient = services.gitClientFactory.create(repoRoot);
77+
GitRepoUnshallow gitRepoUnshallow = new GitRepoUnshallow(services.config, gitClient);
6578

6679
gitDataUploader =
6780
buildGitDataUploader(
6881
services.config,
6982
services.metricCollector,
7083
services.gitInfoProvider,
71-
services.gitClientFactory,
84+
gitClient,
85+
gitRepoUnshallow,
7286
services.backendApi,
7387
repoRoot);
7488
repoIndexProvider = services.repoIndexProviderFactory.create(repoRoot);
@@ -84,18 +98,49 @@ public class CiVisibilityRepoServices {
8498
services.config,
8599
services.metricCollector,
86100
services.backendApi,
101+
gitClient,
102+
gitRepoUnshallow,
87103
gitDataUploader,
104+
pullRequestInfo,
88105
repoRoot);
89106
}
90107
}
91108

92-
static String getModuleName(Config config, Path path, CIInfo ciInfo) {
109+
private static String getRepoRoot(CIInfo ciInfo, GitClient.Factory gitClientFactory) {
110+
String ciWorkspace = ciInfo.getNormalizedCiWorkspace();
111+
if (Strings.isNotBlank(ciWorkspace)) {
112+
return ciWorkspace;
113+
114+
} else {
115+
try {
116+
return gitClientFactory.create(".").getRepoRoot();
117+
118+
} catch (InterruptedException e) {
119+
Thread.currentThread().interrupt();
120+
LOGGER.error("Interrupted while getting repo root", e);
121+
return null;
122+
123+
} catch (Exception e) {
124+
LOGGER.error("Error while getting repo root", e);
125+
return null;
126+
}
127+
}
128+
}
129+
130+
private static String appendSlashIfNeeded(String repoRoot) {
131+
if (repoRoot != null && !repoRoot.endsWith(File.separator)) {
132+
return repoRoot + File.separator;
133+
} else {
134+
return repoRoot;
135+
}
136+
}
137+
138+
static String getModuleName(Config config, String repoRoot, Path path) {
93139
// if parent process is instrumented, it will provide build system's module name
94140
String parentModuleName = config.getCiVisibilityModuleName();
95141
if (parentModuleName != null) {
96142
return parentModuleName;
97143
}
98-
String repoRoot = ciInfo.getNormalizedCiWorkspace();
99144
if (repoRoot != null && path.startsWith(repoRoot)) {
100145
String relativePath = Paths.get(repoRoot).relativize(path).toString();
101146
if (!relativePath.isEmpty()) {
@@ -126,7 +171,10 @@ private static ExecutionSettingsFactory buildExecutionSettingsFactory(
126171
Config config,
127172
CiVisibilityMetricCollector metricCollector,
128173
BackendApi backendApi,
174+
GitClient gitClient,
175+
GitRepoUnshallow gitRepoUnshallow,
129176
GitDataUploader gitDataUploader,
177+
PullRequestInfo pullRequestInfo,
130178
String repoRoot) {
131179
ConfigurationApi configurationApi;
132180
if (backendApi == null) {
@@ -138,7 +186,14 @@ private static ExecutionSettingsFactory buildExecutionSettingsFactory(
138186
}
139187

140188
ExecutionSettingsFactoryImpl factory =
141-
new ExecutionSettingsFactoryImpl(config, configurationApi, gitDataUploader, repoRoot);
189+
new ExecutionSettingsFactoryImpl(
190+
config,
191+
configurationApi,
192+
gitClient,
193+
gitRepoUnshallow,
194+
gitDataUploader,
195+
pullRequestInfo,
196+
repoRoot);
142197
if (processHierarchy.isHeadless()) {
143198
return factory;
144199
} else {
@@ -150,7 +205,8 @@ private static GitDataUploader buildGitDataUploader(
150205
Config config,
151206
CiVisibilityMetricCollector metricCollector,
152207
GitInfoProvider gitInfoProvider,
153-
GitClient.Factory gitClientFactory,
208+
GitClient gitClient,
209+
GitRepoUnshallow gitRepoUnshallow,
154210
BackendApi backendApi,
155211
String repoRoot) {
156212
if (!config.isCiVisibilityGitUploadEnabled()) {
@@ -171,9 +227,15 @@ private static GitDataUploader buildGitDataUploader(
171227

172228
String remoteName = config.getCiVisibilityGitRemoteName();
173229
GitDataApi gitDataApi = new GitDataApi(backendApi, metricCollector);
174-
GitClient gitClient = gitClientFactory.create(repoRoot);
175230
return new GitDataUploaderImpl(
176-
config, metricCollector, gitDataApi, gitClient, gitInfoProvider, repoRoot, remoteName);
231+
config,
232+
metricCollector,
233+
gitDataApi,
234+
gitClient,
235+
gitRepoUnshallow,
236+
gitInfoProvider,
237+
repoRoot,
238+
remoteName);
177239
}
178240

179241
private static SourcePathResolver buildSourcePathResolver(

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilityServices.java‎

Lines changed: 33 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,11 @@
88
import datadog.communication.ddagent.SharedCommunicationObjects;
99
import datadog.communication.http.HttpRetryPolicy;
1010
import datadog.communication.http.OkHttpUtils;
11+
import datadog.communication.util.IOUtils;
1112
import datadog.trace.api.Config;
13+
import datadog.trace.api.civisibility.telemetry.CiVisibilityCountMetric;
1214
import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector;
15+
import datadog.trace.api.civisibility.telemetry.tag.Command;
1316
import datadog.trace.api.git.GitInfoProvider;
1417
import datadog.trace.civisibility.ci.CIProviderInfoFactory;
1518
import datadog.trace.civisibility.ci.env.CiEnvironment;
@@ -22,24 +25,28 @@
2225
import datadog.trace.civisibility.git.CIProviderGitInfoBuilder;
2326
import datadog.trace.civisibility.git.GitClientGitInfoBuilder;
2427
import datadog.trace.civisibility.git.tree.GitClient;
28+
import datadog.trace.civisibility.git.tree.NoOpGitClient;
29+
import datadog.trace.civisibility.git.tree.ShellGitClient;
2530
import datadog.trace.civisibility.ipc.SignalClient;
2631
import datadog.trace.civisibility.source.BestEffortLinesResolver;
2732
import datadog.trace.civisibility.source.ByteCodeLinesResolver;
2833
import datadog.trace.civisibility.source.CompilerAidedLinesResolver;
2934
import datadog.trace.civisibility.source.LinesResolver;
3035
import datadog.trace.civisibility.source.index.*;
36+
import datadog.trace.civisibility.utils.ShellCommandExecutor;
37+
import java.io.File;
3138
import java.lang.reflect.Type;
3239
import java.net.InetSocketAddress;
3340
import java.nio.file.FileSystem;
3441
import java.nio.file.FileSystems;
3542
import java.nio.file.Path;
3643
import java.util.Collections;
3744
import java.util.Map;
45+
import javax.annotation.Nonnull;
3846
import javax.annotation.Nullable;
3947
import okhttp3.HttpUrl;
4048
import okhttp3.OkHttpClient;
4149
import okhttp3.Request;
42-
import org.jetbrains.annotations.NotNull;
4350
import org.slf4j.Logger;
4451
import org.slf4j.LoggerFactory;
4552

@@ -78,7 +85,7 @@ public class CiVisibilityServices {
7885
this.backendApi =
7986
new BackendApiFactory(config, sco).createBackendApi(BackendApiFactory.Intake.API);
8087
this.jvmInfoFactory = new CachingJvmInfoFactory(config, new JvmInfoFactoryImpl());
81-
this.gitClientFactory = new GitClient.Factory(config, metricCollector);
88+
this.gitClientFactory = buildGitClientFactory(config, metricCollector);
8289

8390
CiEnvironment environment = buildCiEnvironment(config, sco);
8491
this.ciProviderInfoFactory = new CIProviderInfoFactory(config, environment);
@@ -111,7 +118,30 @@ public class CiVisibilityServices {
111118
}
112119
}
113120

114-
@NotNull
121+
private static GitClient.Factory buildGitClientFactory(
122+
Config config, CiVisibilityMetricCollector metricCollector) {
123+
if (!config.isCiVisibilityGitClientEnabled()) {
124+
return r -> NoOpGitClient.INSTANCE;
125+
}
126+
try {
127+
ShellCommandExecutor shellCommandExecutor =
128+
new ShellCommandExecutor(new File("."), config.getCiVisibilityGitCommandTimeoutMillis());
129+
String gitVersion = shellCommandExecutor.executeCommand(IOUtils::readFully, "git", "version");
130+
logger.debug("Detected git executable version {}", gitVersion);
131+
return new ShellGitClient.Factory(config, metricCollector);
132+
133+
} catch (Exception e) {
134+
metricCollector.add(
135+
CiVisibilityCountMetric.GIT_COMMAND_ERRORS,
136+
1,
137+
Command.OTHER,
138+
ShellCommandExecutor.getExitCode(e));
139+
logger.info("No git executable detected, some features will not be available");
140+
return r -> NoOpGitClient.INSTANCE;
141+
}
142+
}
143+
144+
@Nonnull
115145
private static CiEnvironment buildCiEnvironment(Config config, SharedCommunicationObjects sco) {
116146
String remoteEnvVarsProviderUrl = config.getCiVisibilityRemoteEnvVarsProviderUrl();
117147
if (remoteEnvVarsProviderUrl != null) {

0 commit comments

Comments
 (0)