Skip to content

Commit 0b24171

Browse files
mccullsmcculls
authored
Merge pull request #6887 from DataDog/mcculls/support-deferred-instrumentation
[EXPERIMENTAL] Support deferred matching and transformation for particular class-loaders
2 parents 5c8d55c + efe7fe0 commit 0b24171

10 files changed

Lines changed: 267 additions & 24 deletions

File tree

‎dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/CombiningMatcher.java‎

Lines changed: 133 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,21 @@
11
package datadog.trace.agent.tooling;
22

3+
import static datadog.trace.api.config.TraceInstrumentationConfig.EXPERIMENTAL_DEFER_INTEGRATIONS_UNTIL;
4+
import static datadog.trace.util.AgentThreadFactory.AgentThread.RETRANSFORMER;
5+
6+
import datadog.trace.agent.tooling.bytebuddy.matcher.CustomExcludes;
7+
import datadog.trace.agent.tooling.bytebuddy.matcher.ProxyClassIgnores;
8+
import datadog.trace.api.InstrumenterConfig;
9+
import datadog.trace.api.time.TimeUtils;
10+
import datadog.trace.util.AgentTaskScheduler;
11+
import java.lang.instrument.Instrumentation;
312
import java.security.ProtectionDomain;
13+
import java.util.ArrayList;
414
import java.util.BitSet;
15+
import java.util.Iterator;
516
import java.util.List;
17+
import java.util.Set;
18+
import java.util.concurrent.TimeUnit;
619
import net.bytebuddy.agent.builder.AgentBuilder;
720
import net.bytebuddy.description.type.TypeDescription;
821
import net.bytebuddy.utility.JavaModule;
@@ -13,6 +26,14 @@
1326
final class CombiningMatcher implements AgentBuilder.RawMatcher {
1427
private static final Logger log = LoggerFactory.getLogger(CombiningMatcher.class);
1528

29+
private static final boolean DEFER_MATCHING =
30+
null != InstrumenterConfig.get().deferIntegrationsUntil();
31+
32+
private static final Set<String> DEFERRED_CLASSLOADER_NAMES =
33+
InstrumenterConfig.get().getDeferredClassLoaders();
34+
35+
private static final boolean DEFER_ALL = DEFERRED_CLASSLOADER_NAMES.isEmpty();
36+
1637
// optimization to avoid repeated allocations inside BitSet as matched ids are set
1738
static final int MAX_COMBINED_ID_HINT = 512;
1839

@@ -25,9 +46,16 @@ final class CombiningMatcher implements AgentBuilder.RawMatcher {
2546
private final BitSet knownTypesMask;
2647
private final MatchRecorder[] matchers;
2748

28-
CombiningMatcher(BitSet knownTypesMask, List<MatchRecorder> matchers) {
49+
private volatile boolean deferring;
50+
51+
CombiningMatcher(
52+
Instrumentation instrumentation, BitSet knownTypesMask, List<MatchRecorder> matchers) {
2953
this.knownTypesMask = knownTypesMask;
3054
this.matchers = matchers.toArray(new MatchRecorder[0]);
55+
56+
if (DEFER_MATCHING) {
57+
scheduleResumeMatching(instrumentation, InstrumenterConfig.get().deferIntegrationsUntil());
58+
}
3159
}
3260

3361
@Override
@@ -38,6 +66,11 @@ public boolean matches(
3866
Class<?> classBeingRedefined,
3967
ProtectionDomain pd) {
4068

69+
// check initial requests to see if we should defer matching until retransformation
70+
if (DEFER_MATCHING && null == classBeingRedefined && deferring && isDeferred(classLoader)) {
71+
return false;
72+
}
73+
4174
BitSet ids = recordedMatches.get();
4275
ids.clear();
4376

@@ -63,4 +96,103 @@ public boolean matches(
6396

6497
return !ids.isEmpty();
6598
}
99+
100+
/** Arranges for any deferred matching to resume at the requested trigger point. */
101+
private void scheduleResumeMatching(Instrumentation instrumentation, String untilTrigger) {
102+
if (null != untilTrigger && !untilTrigger.isEmpty()) {
103+
long delay = TimeUtils.parseSimpleDelay(untilTrigger);
104+
if (delay < 0) {
105+
log.info(
106+
"Unrecognized value for dd.{}: {}",
107+
EXPERIMENTAL_DEFER_INTEGRATIONS_UNTIL,
108+
untilTrigger);
109+
} else if (delay >= 5) { // don't bother deferring small delays
110+
111+
new AgentTaskScheduler(RETRANSFORMER)
112+
.schedule(this::resumeMatching, instrumentation, delay, TimeUnit.SECONDS);
113+
114+
deferring = true;
115+
}
116+
}
117+
}
118+
119+
/**
120+
* Scans loaded classes to find which ones we should retransform to resume matching them.
121+
*
122+
* <p>We try to only trigger retransformations for classes we know would match. Caching and
123+
* memoization means running matching twice is cheaper than unnecessary retransformations.
124+
*/
125+
void resumeMatching(Instrumentation instrumentation) {
126+
if (!deferring) {
127+
return;
128+
}
129+
130+
deferring = false;
131+
132+
Iterator<Iterable<Class<?>>> rediscovery =
133+
AgentStrategies.rediscoveryStrategy().resolve(instrumentation).iterator();
134+
135+
List<Class<?>> resuming = new ArrayList<>();
136+
while (rediscovery.hasNext()) {
137+
for (Class<?> clazz : rediscovery.next()) {
138+
ClassLoader classLoader = clazz.getClassLoader();
139+
if (isDeferred(classLoader)
140+
&& !wouldIgnore(clazz.getName())
141+
&& instrumentation.isModifiableClass(clazz)
142+
&& wouldMatch(classLoader, clazz)) {
143+
resuming.add(clazz);
144+
}
145+
}
146+
}
147+
148+
try {
149+
log.debug("Resuming deferred matching for {}", resuming);
150+
instrumentation.retransformClasses(resuming.toArray(new Class[0]));
151+
} catch (Throwable e) {
152+
log.debug("Problem resuming deferred matching", e);
153+
}
154+
}
155+
156+
/**
157+
* Tests whether matches involving this class-loader should be deferred until later.
158+
*
159+
* <p>The bootstrap class-loader is never deferred.
160+
*/
161+
private static boolean isDeferred(ClassLoader classLoader) {
162+
return null != classLoader
163+
&& (DEFER_ALL || DEFERRED_CLASSLOADER_NAMES.contains(classLoader.getClass().getName()));
164+
}
165+
166+
/** Tests whether this class would be ignored on retransformation. */
167+
private static boolean wouldIgnore(String name) {
168+
return name.indexOf('/') >= 0 // don't retransform lambdas
169+
|| CustomExcludes.isExcluded(name)
170+
|| ProxyClassIgnores.isIgnored(name);
171+
}
172+
173+
/** Tests whether this class would be matched at least once on retransformation. */
174+
private boolean wouldMatch(ClassLoader classLoader, Class<?> clazz) {
175+
BitSet ids = recordedMatches.get();
176+
ids.clear();
177+
178+
knownTypesIndex.apply(clazz.getName(), knownTypesMask, ids);
179+
if (!ids.isEmpty()) {
180+
return true;
181+
}
182+
183+
TypeDescription target = new TypeDescription.ForLoadedType(clazz);
184+
185+
for (MatchRecorder matcher : matchers) {
186+
try {
187+
matcher.record(target, classLoader, clazz, ids);
188+
if (!ids.isEmpty()) {
189+
return true;
190+
}
191+
} catch (Throwable ignore) {
192+
// skip misbehaving matchers
193+
}
194+
}
195+
196+
return false;
197+
}
66198
}

‎dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/CombiningTransformerBuilder.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -257,7 +257,7 @@ public ClassFileTransformer installOn(Instrumentation instrumentation) {
257257
}
258258

259259
return agentBuilder
260-
.type(new CombiningMatcher(knownTypesMask, matchers))
260+
.type(new CombiningMatcher(instrumentation, knownTypesMask, matchers))
261261
.and(NOT_DECORATOR_MATCHER)
262262
.transform(defaultTransformers())
263263
.transform(new SplittingTransformer(transformers))

‎dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/matcher/ClassLoaderMatchers.java‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import java.util.HashMap;
1515
import java.util.List;
1616
import java.util.Map;
17+
import java.util.Set;
1718
import net.bytebuddy.matcher.ElementMatcher;
1819
import org.slf4j.Logger;
1920
import org.slf4j.LoggerFactory;
@@ -25,8 +26,10 @@ public final class ClassLoaderMatchers {
2526

2627
private static final ClassLoader BOOTSTRAP_CLASSLOADER = null;
2728

28-
private static final boolean HAS_CLASSLOADER_EXCLUDES =
29-
!InstrumenterConfig.get().getExcludedClassLoaders().isEmpty();
29+
private static final Set<String> EXCLUDED_CLASSLOADER_NAMES =
30+
InstrumenterConfig.get().getExcludedClassLoaders();
31+
32+
private static final boolean CHECK_EXCLUDES = !EXCLUDED_CLASSLOADER_NAMES.isEmpty();
3033

3134
/** A private constructor that must not be invoked. */
3235
private ClassLoaderMatchers() {
@@ -45,8 +48,8 @@ public static boolean canSkipClassLoaderByName(final ClassLoader loader) {
4548
case "datadog.trace.bootstrap.DatadogClassLoader":
4649
return true;
4750
}
48-
if (HAS_CLASSLOADER_EXCLUDES) {
49-
return InstrumenterConfig.get().getExcludedClassLoaders().contains(classLoaderName);
51+
if (CHECK_EXCLUDES) {
52+
return EXCLUDED_CLASSLOADER_NAMES.contains(classLoaderName);
5053
}
5154
return false;
5255
}

‎dd-trace-api/src/main/java/datadog/trace/api/config/TraceInstrumentationConfig.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,10 @@ public final class TraceInstrumentationConfig {
2929
public static final String TRACE_CLASSES_EXCLUDE_FILE = "trace.classes.exclude.file";
3030
public static final String TRACE_CLASSLOADERS_EXCLUDE = "trace.classloaders.exclude";
3131
public static final String TRACE_CODESOURCES_EXCLUDE = "trace.codesources.exclude";
32+
public static final String TRACE_CLASSLOADERS_DEFER = "trace.classloaders.defer";
33+
34+
public static final String EXPERIMENTAL_DEFER_INTEGRATIONS_UNTIL =
35+
"experimental.defer.integrations.until";
3236

3337
@SuppressWarnings("unused")
3438
public static final String TRACE_TESTS_ENABLED = "trace.tests.enabled";

‎dd-trace-core/src/main/java/datadog/trace/core/flare/TracerFlareService.java‎

Lines changed: 5 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import datadog.trace.api.Config;
77
import datadog.trace.api.DynamicConfig;
88
import datadog.trace.api.flare.TracerFlare;
9+
import datadog.trace.api.time.TimeUtils;
910
import datadog.trace.core.CoreTracer;
1011
import datadog.trace.core.DDTraceCoreInfo;
1112
import datadog.trace.logging.GlobalLogLevelSwitcher;
@@ -27,8 +28,6 @@
2728
import java.time.ZonedDateTime;
2829
import java.util.Collections;
2930
import java.util.concurrent.TimeUnit;
30-
import java.util.regex.Matcher;
31-
import java.util.regex.Pattern;
3231
import java.util.zip.ZipOutputStream;
3332
import okhttp3.HttpUrl;
3433
import okhttp3.MediaType;
@@ -49,8 +48,6 @@ final class TracerFlareService {
4948

5049
private static final MediaType OCTET_STREAM = MediaType.get("application/octet-stream");
5150

52-
private static final Pattern DELAY_TRIGGER = Pattern.compile("(\\d+)([HhMmSs]?)");
53-
5451
private final AgentTaskScheduler scheduler = new AgentTaskScheduler(TRACER_FLARE);
5552

5653
private final Config config;
@@ -81,20 +78,11 @@ final class TracerFlareService {
8178

8279
private void applyTriageReportTrigger(String triageTrigger) {
8380
if (null != triageTrigger && !triageTrigger.isEmpty()) {
84-
Matcher delayMatcher = DELAY_TRIGGER.matcher(triageTrigger);
85-
if (delayMatcher.matches()) {
86-
long delay = Integer.parseInt(delayMatcher.group(1));
87-
String unit = delayMatcher.group(2);
88-
if ("H".equalsIgnoreCase(unit)) {
89-
delay = TimeUnit.HOURS.toSeconds(delay);
90-
} else if ("M".equalsIgnoreCase(unit)) {
91-
delay = TimeUnit.MINUTES.toSeconds(delay);
92-
} else {
93-
// already in seconds
94-
}
95-
scheduleTriageReport(delay);
96-
} else {
81+
long delay = TimeUtils.parseSimpleDelay(triageTrigger);
82+
if (delay < 0) {
9783
log.info("Unrecognized triage trigger {}", triageTrigger);
84+
} else {
85+
scheduleTriageReport(delay);
9886
}
9987
}
10088
}

‎internal-api/src/main/java/datadog/trace/api/InstrumenterConfig.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import static datadog.trace.api.config.ProfilingConfig.PROFILING_ENABLED;
3131
import static datadog.trace.api.config.ProfilingConfig.PROFILING_ENABLED_DEFAULT;
3232
import static datadog.trace.api.config.TraceInstrumentationConfig.AXIS_TRANSPORT_CLASS_NAME;
33+
import static datadog.trace.api.config.TraceInstrumentationConfig.EXPERIMENTAL_DEFER_INTEGRATIONS_UNTIL;
3334
import static datadog.trace.api.config.TraceInstrumentationConfig.HTTP_URL_CONNECTION_CLASS_NAME;
3435
import static datadog.trace.api.config.TraceInstrumentationConfig.INTEGRATIONS_ENABLED;
3536
import static datadog.trace.api.config.TraceInstrumentationConfig.JAX_RS_ADDITIONAL_ANNOTATIONS;
@@ -50,6 +51,7 @@
5051
import static datadog.trace.api.config.TraceInstrumentationConfig.TRACE_ANNOTATION_ASYNC;
5152
import static datadog.trace.api.config.TraceInstrumentationConfig.TRACE_CLASSES_EXCLUDE;
5253
import static datadog.trace.api.config.TraceInstrumentationConfig.TRACE_CLASSES_EXCLUDE_FILE;
54+
import static datadog.trace.api.config.TraceInstrumentationConfig.TRACE_CLASSLOADERS_DEFER;
5355
import static datadog.trace.api.config.TraceInstrumentationConfig.TRACE_CLASSLOADERS_EXCLUDE;
5456
import static datadog.trace.api.config.TraceInstrumentationConfig.TRACE_CODESOURCES_EXCLUDE;
5557
import static datadog.trace.api.config.TraceInstrumentationConfig.TRACE_ENABLED;
@@ -120,6 +122,9 @@ public class InstrumenterConfig {
120122
private final String excludedClassesFile;
121123
private final Set<String> excludedClassLoaders;
122124
private final List<String> excludedCodeSources;
125+
private final Set<String> deferredClassLoaders;
126+
127+
private final String deferIntegrationsUntil;
123128

124129
private final ResolverCacheConfig resolverCacheConfig;
125130
private final String resolverCacheDir;
@@ -206,6 +211,9 @@ private InstrumenterConfig() {
206211
excludedClassesFile = configProvider.getString(TRACE_CLASSES_EXCLUDE_FILE);
207212
excludedClassLoaders = tryMakeImmutableSet(configProvider.getList(TRACE_CLASSLOADERS_EXCLUDE));
208213
excludedCodeSources = tryMakeImmutableList(configProvider.getList(TRACE_CODESOURCES_EXCLUDE));
214+
deferredClassLoaders = tryMakeImmutableSet(configProvider.getList(TRACE_CLASSLOADERS_DEFER));
215+
216+
deferIntegrationsUntil = configProvider.getString(EXPERIMENTAL_DEFER_INTEGRATIONS_UNTIL);
209217

210218
resolverCacheConfig =
211219
configProvider.getEnum(
@@ -353,6 +361,14 @@ public List<String> getExcludedCodeSources() {
353361
return excludedCodeSources;
354362
}
355363

364+
public Set<String> getDeferredClassLoaders() {
365+
return deferredClassLoaders;
366+
}
367+
368+
public String deferIntegrationsUntil() {
369+
return deferIntegrationsUntil;
370+
}
371+
356372
public int getResolverNoMatchesSize() {
357373
return resolverCacheConfig.noMatchesSize();
358374
}
@@ -512,6 +528,10 @@ public String toString() {
512528
+ excludedClassLoaders
513529
+ ", excludedCodeSources="
514530
+ excludedCodeSources
531+
+ ", deferredClassLoaders="
532+
+ deferredClassLoaders
533+
+ ", deferIntegrationsUntil="
534+
+ deferIntegrationsUntil
515535
+ ", resolverCacheConfig="
516536
+ resolverCacheConfig
517537
+ ", resolverCacheDir="

‎internal-api/src/main/java/datadog/trace/util/AgentThreadFactory.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,9 @@ public enum AgentThread {
5151
CI_GIT_DATA_SHUTDOWN_HOOK("dd-ci-git-data-shutdown-hook"),
5252
CI_TEST_EVENTS_SHUTDOWN_HOOK("dd-ci-test-events-shutdown-hook"),
5353
CI_PROJECT_CONFIGURATOR("dd-ci-project-configurator"),
54-
CI_SIGNAL_SERVER("dd-ci-signal-server");
54+
CI_SIGNAL_SERVER("dd-ci-signal-server"),
55+
56+
RETRANSFORMER("dd-retransformer");
5557

5658
public final String threadName;
5759

‎utils/time-utils/build.gradle‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,12 @@
11
apply from: "$rootDir/gradle/java.gradle"
22

3+
ext {
4+
excludedClassesCoverage = [
5+
'datadog.trace.api.time.ControllableTimeSource:',
6+
'datadog.trace.api.time.SystemTimeSource'
7+
]
8+
}
9+
310
dependencies {
411
testImplementation project(':utils:test-utils')
512
}

‎utils/time-utils/src/main/java/datadog/trace/api/time/TimeUtils.java‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
package datadog.trace.api.time;
2+
3+
import java.util.concurrent.TimeUnit;
4+
import java.util.regex.Matcher;
5+
import java.util.regex.Pattern;
6+
7+
public abstract class TimeUtils {
8+
9+
/** Number followed by an optional time unit of hours (h), minutes (m), or seconds (s). */
10+
private static final Pattern SIMPLE_DELAY_PATTERN = Pattern.compile("(\\d+)([HhMmSs]?)");
11+
12+
/**
13+
* Parses the string as a simple delay, such as "30s" or "10m".
14+
*
15+
* @param delayString number followed by an optional time unit
16+
* @return delay in seconds; -1 if the string cannot be parsed
17+
*/
18+
public static long parseSimpleDelay(String delayString) {
19+
if (null != delayString) {
20+
Matcher delayMatcher = SIMPLE_DELAY_PATTERN.matcher(delayString);
21+
if (delayMatcher.matches()) {
22+
long delay = Integer.parseInt(delayMatcher.group(1));
23+
String unit = delayMatcher.group(2);
24+
if ("H".equalsIgnoreCase(unit)) {
25+
return TimeUnit.HOURS.toSeconds(delay);
26+
} else if ("M".equalsIgnoreCase(unit)) {
27+
return TimeUnit.MINUTES.toSeconds(delay);
28+
} else {
29+
return delay; // already in seconds
30+
}
31+
}
32+
}
33+
return -1; // unrecognized
34+
}
35+
36+
private TimeUtils() {}
37+
}

0 commit comments

Comments
 (0)