Skip to content

Commit 0d8268a

Browse files
jpbempeljpbempel
authored
Add support for scanning jar from loaded class (#8370)
when a class is loaded we are now locating the jar containing the class and push the jar to a queue to be scanned by a background thread like for the initial process of SymDB enablement Add more information into SymDB report of total class count processed total jars scanned and histogram of class count by scanned jar
1 parent b8f5fb8 commit 0d8268a

10 files changed

Lines changed: 346 additions & 166 deletions

File tree

‎dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/agent/DebuggerAgent.java‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -147,13 +147,14 @@ private static void startDynamicInstrumentation(
147147
configurationPoller = sco.configurationPoller(config);
148148
if (configurationPoller != null) {
149149
if (config.isSymbolDatabaseEnabled()) {
150+
SymbolAggregator symbolAggregator =
151+
new SymbolAggregator(
152+
classNameFilter,
153+
debuggerSink.getSymbolSink(),
154+
config.getSymbolDatabaseFlushThreshold());
155+
symbolAggregator.start();
150156
symDBEnablement =
151-
new SymDBEnablement(
152-
instrumentation,
153-
config,
154-
new SymbolAggregator(
155-
debuggerSink.getSymbolSink(), config.getSymbolDatabaseFlushThreshold()),
156-
classNameFilter);
157+
new SymDBEnablement(instrumentation, config, symbolAggregator, classNameFilter);
157158
if (config.isSymbolDatabaseForceUpload()) {
158159
symDBEnablement.startSymbolExtraction();
159160
}

‎dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/symbol/BasicSymDBReport.java‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
package com.datadog.debugger.symbol;
2+
3+
import java.io.IOException;
4+
import java.util.ArrayList;
5+
import java.util.HashMap;
6+
import java.util.HashSet;
7+
import java.util.List;
8+
import java.util.Map;
9+
import java.util.Set;
10+
import org.slf4j.Logger;
11+
import org.slf4j.LoggerFactory;
12+
13+
public class BasicSymDBReport implements SymDBReport {
14+
private static final Logger LOGGER = LoggerFactory.getLogger(BasicSymDBReport.class);
15+
16+
private final Set<String> missingJars = new HashSet<>();
17+
private final Map<String, String> ioExceptions = new HashMap<>();
18+
private final List<String> locationErrors = new ArrayList<>();
19+
private final Map<String, Integer> classCountByJar = new HashMap<>();
20+
private final List<String> scannedJars = new ArrayList<>();
21+
22+
public void addMissingJar(String jarPath) {
23+
missingJars.add(jarPath);
24+
}
25+
26+
public void addIOException(String jarPath, IOException e) {
27+
ioExceptions.put(jarPath, e.toString());
28+
}
29+
30+
public void addLocationError(String locationStr) {
31+
locationErrors.add(locationStr);
32+
}
33+
34+
public void incClassCount(String jarPath) {
35+
classCountByJar.compute(jarPath, (k, v) -> v == null ? 1 : v + 1);
36+
}
37+
38+
public void addScannedJar(String jarPath) {
39+
scannedJars.add(jarPath);
40+
}
41+
42+
public void report() {
43+
int totalClasses = classCountByJar.values().stream().mapToInt(Integer::intValue).sum();
44+
String content =
45+
String.format(
46+
"SymDB Report: Scanned jar count=%d, Total class count=%d, class count by jar: %s, Scanned jars: %s, Location errors: %s Missing jars: %s IOExceptions: %s",
47+
scannedJars.size(),
48+
totalClasses,
49+
classCountByJar,
50+
scannedJars,
51+
locationErrors,
52+
missingJars,
53+
ioExceptions);
54+
LOGGER.info(content);
55+
}
56+
}

‎dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/symbol/JarScanner.java‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,9 +51,7 @@ public static Path extractJarPath(ProtectionDomain protectionDomain, SymDBReport
5151
} else if (locationStr.startsWith(FILE_PREFIX)) {
5252
return getPathFromPrefixedFileName(locationStr, FILE_PREFIX, locationStr.length());
5353
}
54-
if (symDBReport != null) {
55-
symDBReport.addLocationError(locationStr);
56-
}
54+
symDBReport.addLocationError(locationStr);
5755
return null;
5856
}
5957

‎dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/symbol/SymDBEnablement.java‎

Lines changed: 2 additions & 94 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
package com.datadog.debugger.symbol;
22

3-
import static com.datadog.debugger.symbol.JarScanner.trimPrefixes;
4-
53
import com.datadog.debugger.util.MoshiHelper;
64
import com.squareup.moshi.JsonAdapter;
75
import datadog.remoteconfig.PollingRateHinter;
@@ -10,34 +8,24 @@
108
import datadog.trace.api.Config;
119
import datadog.trace.bootstrap.debugger.DebuggerContext.ClassNameFilter;
1210
import datadog.trace.util.AgentTaskScheduler;
13-
import datadog.trace.util.Strings;
1411
import java.io.ByteArrayInputStream;
1512
import java.io.ByteArrayOutputStream;
16-
import java.io.File;
1713
import java.io.IOException;
18-
import java.io.InputStream;
1914
import java.lang.instrument.Instrumentation;
2015
import java.net.URISyntaxException;
2116
import java.nio.file.Files;
22-
import java.nio.file.LinkOption;
2317
import java.nio.file.Path;
2418
import java.time.Instant;
2519
import java.time.LocalDateTime;
2620
import java.time.ZoneId;
2721
import java.util.Arrays;
28-
import java.util.HashSet;
29-
import java.util.Set;
3022
import java.util.concurrent.atomic.AtomicBoolean;
31-
import java.util.jar.JarEntry;
32-
import java.util.jar.JarFile;
33-
import java.util.regex.Pattern;
3423
import okio.Okio;
3524
import org.slf4j.Logger;
3625
import org.slf4j.LoggerFactory;
3726

3827
public class SymDBEnablement implements ProductListener {
3928
private static final Logger LOGGER = LoggerFactory.getLogger(SymDBEnablement.class);
40-
private static final Pattern COMMA_PATTERN = Pattern.compile(",");
4129
private static final JsonAdapter<SymDbRemoteConfigRecord> SYM_DB_JSON_ADAPTER =
4230
MoshiHelper.createMoshiConfig().adapter(SymDbRemoteConfigRecord.class);
4331
private static final String SYM_DB_RC_KEY = "symDb";
@@ -120,7 +108,7 @@ public void startSymbolExtraction() {
120108
symbolExtractionTransformer =
121109
new SymbolExtractionTransformer(symbolAggregator, classNameFilter);
122110
instrumentation.addTransformer(symbolExtractionTransformer);
123-
SymDBReport symDBReport = new SymDBReport();
111+
SymDBReport symDBReport = new BasicSymDBReport();
124112
extractSymbolForLoadedClasses(symDBReport);
125113
symDBReport.report();
126114
lastUploadTimestamp = System.currentTimeMillis();
@@ -145,7 +133,6 @@ private void extractSymbolForLoadedClasses(SymDBReport symDBReport) {
145133
LOGGER.debug("Failed to get all loaded classes", ex);
146134
return;
147135
}
148-
Set<String> alreadyScannedJars = new HashSet<>();
149136
byte[] buffer = new byte[READ_BUFFER_SIZE];
150137
ByteArrayOutputStream baos = new ByteArrayOutputStream(CLASSFILE_BUFFER_SIZE);
151138
for (Class<?> clazz : classesToExtract) {
@@ -162,86 +149,7 @@ private void extractSymbolForLoadedClasses(SymDBReport symDBReport) {
162149
symDBReport.addMissingJar(jarPath.toString());
163150
continue;
164151
}
165-
File jarPathFile = jarPath.toFile();
166-
if (jarPathFile.isDirectory()) {
167-
scanDirectory(jarPath, alreadyScannedJars, baos, buffer, symDBReport);
168-
alreadyScannedJars.add(jarPath.toString());
169-
continue;
170-
}
171-
if (alreadyScannedJars.contains(jarPath.toString())) {
172-
continue;
173-
}
174-
try {
175-
try (JarFile jarFile = new JarFile(jarPathFile)) {
176-
jarFile.stream()
177-
.filter(jarEntry -> jarEntry.getName().endsWith(".class"))
178-
.filter(
179-
jarEntry ->
180-
!classNameFilter.isExcluded(
181-
Strings.getClassName(trimPrefixes(jarEntry.getName()))))
182-
.forEach(jarEntry -> parseJarEntry(jarEntry, jarFile, jarPath, baos, buffer));
183-
}
184-
alreadyScannedJars.add(jarPath.toString());
185-
} catch (IOException e) {
186-
symDBReport.addIOException(jarPath.toString(), e);
187-
throw new RuntimeException(e);
188-
}
189-
}
190-
}
191-
192-
private void scanDirectory(
193-
Path jarPath,
194-
Set<String> alreadyScannedJars,
195-
ByteArrayOutputStream baos,
196-
byte[] buffer,
197-
SymDBReport symDBReport) {
198-
try {
199-
Files.walk(jarPath)
200-
// explicitly no follow links walking the directory to avoid cycles
201-
.filter(path -> Files.isRegularFile(path, LinkOption.NOFOLLOW_LINKS))
202-
.filter(path -> path.toString().endsWith(".class"))
203-
.filter(
204-
path ->
205-
!classNameFilter.isExcluded(
206-
Strings.getClassName(trimPrefixes(jarPath.relativize(path).toString()))))
207-
.forEach(path -> parseFileEntry(path, jarPath, baos, buffer));
208-
alreadyScannedJars.add(jarPath.toString());
209-
} catch (IOException e) {
210-
symDBReport.addIOException(jarPath.toString(), e);
211-
throw new RuntimeException(e);
212-
}
213-
}
214-
215-
private void parseFileEntry(Path path, Path jarPath, ByteArrayOutputStream baos, byte[] buffer) {
216-
LOGGER.debug("parsing file class: {}", path.toString());
217-
try {
218-
try (InputStream inputStream = Files.newInputStream(path)) {
219-
int readBytes;
220-
baos.reset();
221-
while ((readBytes = inputStream.read(buffer)) != -1) {
222-
baos.write(buffer, 0, readBytes);
223-
}
224-
symbolAggregator.parseClass(
225-
path.getFileName().toString(), baos.toByteArray(), jarPath.toString());
226-
}
227-
} catch (IOException ex) {
228-
LOGGER.debug("Exception during parsing file class: {}", path, ex);
229-
}
230-
}
231-
232-
private void parseJarEntry(
233-
JarEntry jarEntry, JarFile jarFile, Path jarPath, ByteArrayOutputStream baos, byte[] buffer) {
234-
LOGGER.debug("parsing jarEntry class: {}", jarEntry.getName());
235-
try {
236-
InputStream inputStream = jarFile.getInputStream(jarEntry);
237-
int readBytes;
238-
baos.reset();
239-
while ((readBytes = inputStream.read(buffer)) != -1) {
240-
baos.write(buffer, 0, readBytes);
241-
}
242-
symbolAggregator.parseClass(jarEntry.getName(), baos.toByteArray(), jarPath.toString());
243-
} catch (IOException ex) {
244-
LOGGER.debug("Exception during parsing jarEntry class: {}", jarEntry.getName(), ex);
152+
symbolAggregator.scanJar(symDBReport, jarPath, baos, buffer);
245153
}
246154
}
247155
}

‎dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/symbol/SymDBReport.java‎

Lines changed: 35 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -1,42 +1,39 @@
11
package com.datadog.debugger.symbol;
22

33
import java.io.IOException;
4-
import java.util.ArrayList;
5-
import java.util.HashMap;
6-
import java.util.HashSet;
7-
import java.util.List;
8-
import java.util.Map;
9-
import java.util.Set;
10-
import org.slf4j.Logger;
11-
import org.slf4j.LoggerFactory;
12-
13-
public class SymDBReport {
14-
private static final Logger LOGGER = LoggerFactory.getLogger(SymDBReport.class);
15-
16-
private final Set<String> missingJars = new HashSet<>();
17-
private final Map<String, String> ioExceptions = new HashMap<>();
18-
private final List<String> locationErrors = new ArrayList<>();
19-
20-
public void addMissingJar(String jarPath) {
21-
missingJars.add(jarPath);
22-
}
23-
24-
public void addIOException(String jarPath, IOException e) {
25-
ioExceptions.put(jarPath, e.toString());
26-
}
27-
28-
public void addLocationError(String locationStr) {
29-
locationErrors.add(locationStr);
30-
}
31-
32-
public void report() {
33-
String content =
34-
"== SymDB Report == Location errors:"
35-
+ locationErrors
36-
+ " Missing jars: "
37-
+ missingJars
38-
+ " IOExceptions: "
39-
+ ioExceptions;
40-
LOGGER.info(content);
41-
}
4+
5+
public interface SymDBReport {
6+
7+
void addMissingJar(String jarPath);
8+
9+
void addIOException(String jarPath, IOException e);
10+
11+
void addLocationError(String locationStr);
12+
13+
void incClassCount(String jarPath);
14+
15+
void addScannedJar(String jarPath);
16+
17+
void report();
18+
19+
SymDBReport NO_OP =
20+
new SymDBReport() {
21+
@Override
22+
public void addMissingJar(String jarPath) {}
23+
24+
@Override
25+
public void addIOException(String jarPath, IOException e) {}
26+
27+
@Override
28+
public void addLocationError(String locationStr) {}
29+
30+
@Override
31+
public void incClassCount(String jarPath) {}
32+
33+
@Override
34+
public void addScannedJar(String jarPath) {}
35+
36+
@Override
37+
public void report() {}
38+
};
4239
}

0 commit comments

Comments
 (0)