Skip to content

Commit 2f0211b

Browse files
nikita-tkachenko-datadognikita-tkachenko-datadog
authored
Implement early flakiness detection (#6727)
1 parent ef26b91 commit 2f0211b

283 files changed

Lines changed: 20626 additions & 992 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/CiVisibilitySettings.java‎

Lines changed: 62 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,52 +1,90 @@
11
package datadog.trace.civisibility.config;
22

3+
import com.squareup.moshi.FromJson;
4+
import datadog.trace.api.civisibility.config.EarlyFlakeDetectionSettings;
35
import java.nio.file.Path;
6+
import java.util.Map;
47

58
public class CiVisibilitySettings {
69

710
public static final CiVisibilitySettings DEFAULT =
8-
new CiVisibilitySettings(false, false, false, false, false);
9-
10-
private final boolean itr_enabled;
11-
private final boolean code_coverage;
12-
private final boolean tests_skipping;
13-
private final boolean require_git;
14-
private final boolean flaky_test_retries_enabled;
15-
16-
public CiVisibilitySettings(
17-
boolean itr_enabled,
18-
boolean code_coverage,
19-
boolean tests_skipping,
20-
boolean require_git,
21-
boolean flaky_test_retries_enabled) {
22-
this.itr_enabled = itr_enabled;
23-
this.code_coverage = code_coverage;
24-
this.tests_skipping = tests_skipping;
25-
this.require_git = require_git;
26-
this.flaky_test_retries_enabled = flaky_test_retries_enabled;
11+
new CiVisibilitySettings(
12+
false, false, false, false, false, EarlyFlakeDetectionSettings.DEFAULT);
13+
14+
private final boolean itrEnabled;
15+
private final boolean codeCoverage;
16+
private final boolean testsSkipping;
17+
private final boolean requireGit;
18+
private final boolean flakyTestRetriesEnabled;
19+
private final EarlyFlakeDetectionSettings earlyFlakeDetectionSettings;
20+
21+
private CiVisibilitySettings(
22+
boolean itrEnabled,
23+
boolean codeCoverage,
24+
boolean testsSkipping,
25+
boolean requireGit,
26+
boolean flakyTestRetriesEnabled,
27+
EarlyFlakeDetectionSettings earlyFlakeDetectionSettings) {
28+
this.itrEnabled = itrEnabled;
29+
this.codeCoverage = codeCoverage;
30+
this.testsSkipping = testsSkipping;
31+
this.requireGit = requireGit;
32+
this.flakyTestRetriesEnabled = flakyTestRetriesEnabled;
33+
this.earlyFlakeDetectionSettings = earlyFlakeDetectionSettings;
2734
}
2835

2936
public boolean isItrEnabled() {
30-
return itr_enabled;
37+
return itrEnabled;
3138
}
3239

3340
public boolean isCodeCoverageEnabled() {
34-
return code_coverage;
41+
return codeCoverage;
3542
}
3643

3744
public boolean isTestsSkippingEnabled() {
38-
return tests_skipping;
45+
return testsSkipping;
3946
}
4047

4148
public boolean isGitUploadRequired() {
42-
return require_git;
49+
return requireGit;
4350
}
4451

4552
public boolean isFlakyTestRetriesEnabled() {
46-
return flaky_test_retries_enabled;
53+
return flakyTestRetriesEnabled;
54+
}
55+
56+
public EarlyFlakeDetectionSettings getEarlyFlakeDetectionSettings() {
57+
return earlyFlakeDetectionSettings;
4758
}
4859

4960
public interface Factory {
5061
CiVisibilitySettings create(Path path);
5162
}
63+
64+
public static final class JsonAdapter {
65+
66+
public static final JsonAdapter INSTANCE = new JsonAdapter();
67+
68+
@FromJson
69+
public CiVisibilitySettings fromJson(Map<String, Object> json) {
70+
if (json == null) {
71+
return DEFAULT;
72+
}
73+
74+
return new CiVisibilitySettings(
75+
getBoolean(json, "itr_enabled", false),
76+
getBoolean(json, "code_coverage", false),
77+
getBoolean(json, "tests_skipping", false),
78+
getBoolean(json, "require_git", false),
79+
getBoolean(json, "flaky_test_retries_enabled", false),
80+
EarlyFlakeDetectionSettingsJsonAdapter.INSTANCE.fromJson(
81+
(Map<String, Object>) json.get("early_flake_detection")));
82+
}
83+
84+
private static boolean getBoolean(
85+
Map<String, Object> json, String fieldName, boolean defaultValue) {
86+
Object value = json.get(fieldName);
87+
return value instanceof Boolean ? (Boolean) value : defaultValue;
88+
}
89+
}
5290
}

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ConfigurationApi.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import java.io.IOException;
55
import java.util.Collection;
66
import java.util.Collections;
7+
import java.util.Map;
78

89
public interface ConfigurationApi {
910

@@ -23,11 +24,20 @@ public SkippableTests getSkippableTests(TracerEnvironment tracerEnvironment) {
2324
public Collection<TestIdentifier> getFlakyTests(TracerEnvironment tracerEnvironment) {
2425
return Collections.emptyList();
2526
}
27+
28+
@Override
29+
public Map<String, Collection<TestIdentifier>> getKnownTestsByModuleName(
30+
TracerEnvironment tracerEnvironment) throws IOException {
31+
return Collections.emptyMap();
32+
}
2633
};
2734

2835
CiVisibilitySettings getSettings(TracerEnvironment tracerEnvironment) throws IOException;
2936

3037
SkippableTests getSkippableTests(TracerEnvironment tracerEnvironment) throws IOException;
3138

3239
Collection<TestIdentifier> getFlakyTests(TracerEnvironment tracerEnvironment) throws IOException;
40+
41+
Map<String, Collection<TestIdentifier>> getKnownTestsByModuleName(
42+
TracerEnvironment tracerEnvironment) throws IOException;
3343
}

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ConfigurationApiImpl.java‎

Lines changed: 64 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,11 @@
1616
import datadog.trace.civisibility.communication.TelemetryListener;
1717
import java.io.IOException;
1818
import java.lang.reflect.ParameterizedType;
19+
import java.util.ArrayList;
1920
import java.util.Collection;
21+
import java.util.HashMap;
2022
import java.util.List;
23+
import java.util.Map;
2124
import java.util.UUID;
2225
import java.util.function.Supplier;
2326
import java.util.stream.Collectors;
@@ -33,6 +36,7 @@ public class ConfigurationApiImpl implements ConfigurationApi {
3336
private static final String SETTINGS_URI = "libraries/tests/services/setting";
3437
private static final String SKIPPABLE_TESTS_URI = "ci/tests/skippable";
3538
private static final String FLAKY_TESTS_URI = "ci/libraries/tests/flaky";
39+
private static final String KNOWN_TESTS_URI = "ci/libraries/tests";
3640

3741
private final BackendApi backendApi;
3842
private final CiVisibilityMetricCollector metricCollector;
@@ -41,6 +45,7 @@ public class ConfigurationApiImpl implements ConfigurationApi {
4145
private final JsonAdapter<EnvelopeDto<TracerEnvironment>> requestAdapter;
4246
private final JsonAdapter<EnvelopeDto<CiVisibilitySettings>> settingsResponseAdapter;
4347
private final JsonAdapter<MultiEnvelopeDto<TestIdentifier>> testIdentifiersResponseAdapter;
48+
private final JsonAdapter<EnvelopeDto<KnownTestsDto>> testFullNamesResponseAdapter;
4449

4550
public ConfigurationApiImpl(BackendApi backendApi, CiVisibilityMetricCollector metricCollector) {
4651
this(backendApi, metricCollector, () -> UUID.randomUUID().toString());
@@ -55,7 +60,11 @@ public ConfigurationApiImpl(BackendApi backendApi, CiVisibilityMetricCollector m
5560
this.uuidGenerator = uuidGenerator;
5661

5762
Moshi moshi =
58-
new Moshi.Builder().add(new ConfigurationsJson.ConfigurationsJsonAdapter()).build();
63+
new Moshi.Builder()
64+
.add(ConfigurationsJson.JsonAdapter.INSTANCE)
65+
.add(CiVisibilitySettings.JsonAdapter.INSTANCE)
66+
.add(EarlyFlakeDetectionSettingsJsonAdapter.INSTANCE)
67+
.build();
5968

6069
ParameterizedType requestType =
6170
Types.newParameterizedTypeWithOwner(
@@ -71,6 +80,11 @@ public ConfigurationApiImpl(BackendApi backendApi, CiVisibilityMetricCollector m
7180
Types.newParameterizedTypeWithOwner(
7281
ConfigurationApiImpl.class, MultiEnvelopeDto.class, TestIdentifier.class);
7382
testIdentifiersResponseAdapter = moshi.adapter(testIdentifiersResponseType);
83+
84+
ParameterizedType testFullNamesResponseType =
85+
Types.newParameterizedTypeWithOwner(
86+
ConfigurationApiImpl.class, EnvelopeDto.class, KnownTestsDto.class);
87+
testFullNamesResponseAdapter = moshi.adapter(testFullNamesResponseType);
7488
}
7589

7690
@Override
@@ -156,6 +170,47 @@ public Collection<TestIdentifier> getFlakyTests(TracerEnvironment tracerEnvironm
156170
return response.stream().map(DataDto::getAttributes).collect(Collectors.toList());
157171
}
158172

173+
@Override
174+
public Map<String, Collection<TestIdentifier>> getKnownTestsByModuleName(
175+
TracerEnvironment tracerEnvironment) throws IOException {
176+
String uuid = uuidGenerator.get();
177+
EnvelopeDto<TracerEnvironment> request =
178+
new EnvelopeDto<>(new DataDto<>(uuid, "ci_app_libraries_tests_request", tracerEnvironment));
179+
String json = requestAdapter.toJson(request);
180+
RequestBody requestBody = RequestBody.create(JSON, json);
181+
KnownTestsDto knownTests =
182+
backendApi.post(
183+
KNOWN_TESTS_URI,
184+
requestBody,
185+
is ->
186+
testFullNamesResponseAdapter.fromJson(Okio.buffer(Okio.source(is))).data.attributes,
187+
null);
188+
return parseTestIdentifiers(knownTests);
189+
}
190+
191+
private static Map<String, Collection<TestIdentifier>> parseTestIdentifiers(
192+
KnownTestsDto knownTests) {
193+
Map<String, Collection<TestIdentifier>> knownTestsByModuleName =
194+
new HashMap<>(knownTests.tests.size() * 4 / 3);
195+
for (Map.Entry<String, Map<String, List<String>>> e : knownTests.tests.entrySet()) {
196+
String moduleName = e.getKey();
197+
Map<String, List<String>> testsBySuiteName = e.getValue();
198+
199+
ArrayList<TestIdentifier> testIdentifiers = new ArrayList<>();
200+
for (Map.Entry<String, List<String>> se : testsBySuiteName.entrySet()) {
201+
String suiteName = se.getKey();
202+
List<String> testNames = se.getValue();
203+
testIdentifiers.ensureCapacity(testIdentifiers.size() + testNames.size());
204+
for (String testName : testNames) {
205+
testIdentifiers.add(new TestIdentifier(suiteName, testName, null, null));
206+
}
207+
}
208+
209+
knownTestsByModuleName.put(moduleName, testIdentifiers);
210+
}
211+
return knownTestsByModuleName;
212+
}
213+
159214
private static final class EnvelopeDto<T> {
160215
private final DataDto<T> data;
161216

@@ -197,4 +252,12 @@ private MetaDto(String correlation_id) {
197252
this.correlation_id = correlation_id;
198253
}
199254
}
255+
256+
private static final class KnownTestsDto {
257+
private final Map<String, Map<String, List<String>>> tests;
258+
259+
private KnownTestsDto(Map<String, Map<String, List<String>>> tests) {
260+
this.tests = tests;
261+
}
262+
}
200263
}

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ConfigurationsJson.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,9 @@ public ConfigurationsJson(
5959
this.custom = custom;
6060
}
6161

62-
public static final class ConfigurationsJsonAdapter {
62+
public static final class JsonAdapter {
63+
public static final JsonAdapter INSTANCE = new JsonAdapter();
64+
6365
@FromJson
6466
public Configurations fromJson(ConfigurationsJson configurationsJson) {
6567
return new Configurations(

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/EarlyFlakeDetectionSettingsJsonAdapter.java‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
package datadog.trace.civisibility.config;
2+
3+
import com.squareup.moshi.FromJson;
4+
import datadog.trace.api.civisibility.config.EarlyFlakeDetectionSettings;
5+
import java.util.ArrayList;
6+
import java.util.Collections;
7+
import java.util.Comparator;
8+
import java.util.List;
9+
import java.util.Map;
10+
import java.util.concurrent.TimeUnit;
11+
12+
public final class EarlyFlakeDetectionSettingsJsonAdapter {
13+
public static final EarlyFlakeDetectionSettingsJsonAdapter INSTANCE =
14+
new EarlyFlakeDetectionSettingsJsonAdapter();
15+
16+
@FromJson
17+
public EarlyFlakeDetectionSettings fromJson(Map<String, Object> json) {
18+
if (json == null) {
19+
return EarlyFlakeDetectionSettings.DEFAULT;
20+
}
21+
22+
Boolean enabled = (Boolean) json.get("enabled");
23+
Double faultySessionThreshold = (Double) json.get("faulty_session_threshold");
24+
25+
List<EarlyFlakeDetectionSettings.ExecutionsByDuration> executionsByDuration;
26+
Map<String, Double> slowTestRetries = (Map<String, Double>) json.get("slow_test_retries");
27+
if (slowTestRetries != null) {
28+
executionsByDuration = new ArrayList<>(slowTestRetries.size());
29+
for (Map.Entry<String, Double> e : slowTestRetries.entrySet()) {
30+
long durationMillis = parseDuration(e.getKey());
31+
int retries = e.getValue().intValue();
32+
executionsByDuration.add(
33+
new EarlyFlakeDetectionSettings.ExecutionsByDuration(durationMillis, retries));
34+
}
35+
executionsByDuration.sort(Comparator.comparingLong(r -> r.durationMillis));
36+
} else {
37+
executionsByDuration = Collections.emptyList();
38+
}
39+
40+
return new EarlyFlakeDetectionSettings(
41+
enabled != null ? enabled : false,
42+
executionsByDuration,
43+
faultySessionThreshold != null ? faultySessionThreshold.intValue() : -1);
44+
}
45+
46+
private static long parseDuration(String duration) {
47+
char lastCharacter = duration.charAt(duration.length() - 1);
48+
int numericValue = Integer.parseInt(duration.substring(0, duration.length() - 1));
49+
TimeUnit timeUnit;
50+
switch (lastCharacter) {
51+
case 's':
52+
timeUnit = TimeUnit.SECONDS;
53+
break;
54+
case 'm':
55+
timeUnit = TimeUnit.MINUTES;
56+
break;
57+
case 'h':
58+
timeUnit = TimeUnit.HOURS;
59+
break;
60+
default:
61+
throw new IllegalArgumentException("Unexpected duration unit: " + lastCharacter);
62+
}
63+
return timeUnit.toMillis(numericValue);
64+
}
65+
}

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/EarlyFlakeDetectionSettingsSerializer.java‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
package datadog.trace.civisibility.config;
2+
3+
import datadog.trace.api.civisibility.config.EarlyFlakeDetectionSettings;
4+
import datadog.trace.civisibility.ipc.Serializer;
5+
import java.nio.ByteBuffer;
6+
import java.util.List;
7+
8+
public final class EarlyFlakeDetectionSettingsSerializer {
9+
10+
public static void serialize(Serializer serializer, EarlyFlakeDetectionSettings settings) {
11+
if (!settings.isEnabled()) {
12+
serializer.write((byte) 0);
13+
return;
14+
}
15+
serializer.write((byte) 1);
16+
serializer.write(settings.getFaultySessionThreshold());
17+
serializer.write(
18+
settings.getExecutionsByDuration(),
19+
EarlyFlakeDetectionSettingsSerializer::serializeExecutionsByDuration);
20+
}
21+
22+
public static EarlyFlakeDetectionSettings deserialize(ByteBuffer buf) {
23+
boolean enabled = Serializer.readByte(buf) != 0;
24+
if (!enabled) {
25+
return EarlyFlakeDetectionSettings.DEFAULT;
26+
}
27+
28+
int faultySessionThreshold = Serializer.readInt(buf);
29+
List<EarlyFlakeDetectionSettings.ExecutionsByDuration> executionsByDuration =
30+
Serializer.readList(
31+
buf, EarlyFlakeDetectionSettingsSerializer::deserializeExecutionsByDuration);
32+
return new EarlyFlakeDetectionSettings(enabled, executionsByDuration, faultySessionThreshold);
33+
}
34+
35+
private static void serializeExecutionsByDuration(
36+
Serializer serializer,
37+
EarlyFlakeDetectionSettings.ExecutionsByDuration executionsByDuration) {
38+
serializer.write(executionsByDuration.durationMillis);
39+
serializer.write(executionsByDuration.executions);
40+
}
41+
42+
private static EarlyFlakeDetectionSettings.ExecutionsByDuration deserializeExecutionsByDuration(
43+
ByteBuffer buf) {
44+
return new EarlyFlakeDetectionSettings.ExecutionsByDuration(
45+
Serializer.readLong(buf), Serializer.readInt(buf));
46+
}
47+
}

0 commit comments

Comments
 (0)