Skip to content

Commit 5e32dc2

Browse files
jbachorikjbachorik
authored
Merge branch 'master' into jb/j9_jfr_check
2 parents 38dd56a + e6c6eaf commit 5e32dc2

43 files changed

Lines changed: 913 additions & 118 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.circleci/config.continue.yml.j2‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -858,6 +858,9 @@ jobs:
858858
APPSEC_RASP
859859
APPSEC_RUNTIME_ACTIVATION
860860
APPSEC_WAF_TELEMETRY
861+
APPSEC_STANDALONE_V2
862+
IAST_STANDALONE_V2
863+
SCA_STANDALONE_V2
861864
REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD
862865
"
863866
fi

‎communication/src/main/java/datadog/communication/ddagent/DDAgentFeaturesDiscovery.java‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
package datadog.communication.ddagent;
22

3+
import static datadog.communication.serialization.msgpack.MsgPackWriter.FIXARRAY;
4+
import static java.util.Collections.singletonList;
5+
36
import com.squareup.moshi.JsonAdapter;
47
import com.squareup.moshi.Moshi;
58
import com.squareup.moshi.Types;
@@ -11,7 +14,6 @@
1114
import datadog.trace.util.Strings;
1215
import java.nio.ByteBuffer;
1316
import java.security.NoSuchAlgorithmException;
14-
import java.util.Collections;
1517
import java.util.HashSet;
1618
import java.util.List;
1719
import java.util.Map;
@@ -32,6 +34,12 @@ public class DDAgentFeaturesDiscovery implements DroppingPolicy {
3234
.build()
3335
.adapter(Types.newParameterizedType(Map.class, String.class, Object.class));
3436

37+
// Currently all the endpoints that we probe expect a msgpack body of an array of arrays, v3/v4
38+
// arbitrary size and v5 two elements, so let's give them a two element array of empty arrays
39+
private static final byte[] PROBE_MESSAGE = {
40+
(byte) FIXARRAY | 2, (byte) FIXARRAY, (byte) FIXARRAY
41+
};
42+
3543
public static final String V3_ENDPOINT = "v0.3/traces";
3644
public static final String V4_ENDPOINT = "v0.4/traces";
3745
public static final String V5_ENDPOINT = "v0.5/traces";
@@ -191,7 +199,9 @@ private String probeTracesEndpoint(String[] endpoints) {
191199
client
192200
.newCall(
193201
new Request.Builder()
194-
.put(OkHttpUtils.msgpackRequestBodyOf(Collections.<ByteBuffer>emptyList()))
202+
.put(
203+
OkHttpUtils.msgpackRequestBodyOf(
204+
singletonList(ByteBuffer.wrap(PROBE_MESSAGE))))
195205
.url(agentBaseUrl.resolve(candidate))
196206
.build())
197207
.execute()) {

‎dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/java/concurrent/ExecutorInstrumentationUtils.java‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
package datadog.trace.bootstrap.instrumentation.java.concurrent;
22

3-
import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activeScope;
3+
import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activeSpan;
4+
import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.isAsyncPropagationEnabled;
45
import static datadog.trace.bootstrap.instrumentation.java.concurrent.ExcludeFilter.ExcludeType;
56

67
import datadog.trace.bootstrap.ContextStore;
7-
import datadog.trace.bootstrap.instrumentation.api.AgentScope;
88
import datadog.trace.bootstrap.instrumentation.api.AgentSpan;
99
import java.util.concurrent.Executor;
1010
import org.slf4j.Logger;
@@ -26,14 +26,10 @@ public static boolean shouldAttachStateToTask(final Object task, final Executor
2626
if (task == null) {
2727
return false;
2828
}
29-
3029
if (ExcludeFilter.exclude(ExcludeType.EXECUTOR, task)) {
3130
return false;
3231
}
33-
34-
final AgentScope scope = activeScope();
35-
36-
return scope != null && scope.isAsyncPropagating();
32+
return activeSpan() != null && isAsyncPropagationEnabled();
3733
}
3834

3935
/**

‎dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/probe/LogProbe.java‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -701,15 +701,19 @@ public void commit(CapturedContext lineContext, int line) {
701701
shouldCommit = true;
702702
}
703703
if (shouldCommit) {
704-
if (isCaptureSnapshot()) {
705-
// freeze context just before commit because line probes have only one context
706-
Duration timeout =
707-
Duration.of(Config.get().getDynamicInstrumentationCaptureTimeout(), ChronoUnit.MILLIS);
708-
lineContext.freeze(new TimeoutChecker(timeout));
709-
snapshot.addLine(lineContext, line);
704+
incrementBudget();
705+
if (inBudget()) {
706+
if (isCaptureSnapshot()) {
707+
// freeze context just before commit because line probes have only one context
708+
Duration timeout =
709+
Duration.of(
710+
Config.get().getDynamicInstrumentationCaptureTimeout(), ChronoUnit.MILLIS);
711+
lineContext.freeze(new TimeoutChecker(timeout));
712+
snapshot.addLine(lineContext, line);
713+
}
714+
commitSnapshot(snapshot, sink);
715+
return;
710716
}
711-
commitSnapshot(snapshot, sink);
712-
return;
713717
}
714718
sink.skipSnapshot(id, DebuggerContext.SkipCause.CONDITION);
715719
}

‎dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/probe/LogProbeTest.java‎

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,34 @@ public void budgets() {
102102
assertEquals(runs * LogProbe.NON_CAPTURING_PROBE_BUDGET, sink.highRate);
103103
}
104104

105+
@Test
106+
public void budgetsOnLineProbes() {
107+
BudgetSink sink = new BudgetSink(getConfig(), mock(ProbeStatusSink.class));
108+
DebuggerAgentHelper.injectSink(sink);
109+
110+
TracerAPI tracer =
111+
CoreTracer.builder().idGenerationStrategy(IdGenerationStrategy.fromName("random")).build();
112+
AgentTracer.registerIfAbsent(tracer);
113+
int runs = 100;
114+
for (int i = 0; i < runs; i++) {
115+
runTrace(tracer, true, 100);
116+
}
117+
assertEquals(runs * LogProbe.CAPTURING_PROBE_BUDGET, sink.captures);
118+
119+
sink = new BudgetSink(getConfig(), mock(ProbeStatusSink.class));
120+
DebuggerAgentHelper.injectSink(sink);
121+
runs = 1010;
122+
for (int i = 0; i < runs; i++) {
123+
runTrace(tracer, false, 100);
124+
}
125+
assertEquals(runs * LogProbe.NON_CAPTURING_PROBE_BUDGET, sink.highRate);
126+
}
127+
105128
private void runTrace(TracerAPI tracer, boolean captureSnapshot) {
129+
runTrace(tracer, captureSnapshot, null);
130+
}
131+
132+
private void runTrace(TracerAPI tracer, boolean captureSnapshot, Integer line) {
106133
AgentSpan span = tracer.startSpan("budget testing", "test span");
107134
span.setTag(Tags.PROPAGATED_DEBUG, "12345:1");
108135
try (AgentScope scope = tracer.activateSpan(span, ScopeSource.MANUAL)) {
@@ -125,7 +152,11 @@ private void runTrace(TracerAPI tracer, boolean captureSnapshot) {
125152
int runs = budget + 20;
126153

127154
for (int i = 0; i < runs; i++) {
128-
logProbe.commit(entryContext, exitContext, emptyList());
155+
if (line == null) {
156+
logProbe.commit(entryContext, exitContext, emptyList());
157+
} else {
158+
logProbe.commit(entryContext, line);
159+
}
129160
}
130161
assertEquals(runs, span.getLocalRootSpan().getTag(format("_dd.ld.probe_id.%s", logProbe.id)));
131162
}

‎dd-java-agent/agent-iast/src/main/java/com/datadog/iast/Reporter.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,9 @@ private void reportVulnerability(
7777
final VulnerabilityBatch batch = getOrCreateVulnerabilityBatch(span);
7878
if (batch != null) {
7979
batch.add(vulnerability);
80-
if (Config.get().isIastStackTraceEnabled() && batch.getVulnerabilities() != null) {
80+
if (Config.get().isIastStackTraceEnabled()
81+
&& batch.getVulnerabilities() != null
82+
&& vulnerability.getLocation().getStackId() == null) {
8183
String stackId =
8284
addVulnerabilityStackTrace(span, String.valueOf(batch.getVulnerabilities().size()));
8385
if (stackId != null) {

‎dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/ReporterTest.groovy‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -533,6 +533,80 @@ class ReporterTest extends DDSpecification {
533533
batch.vulnerabilities.size() == 2
534534
}
535535

536+
void 'two cookie vulnerabilities that share location share stackId and only generate one stack'() {
537+
given:
538+
final Reporter reporter = new Reporter()
539+
final traceSegment = Mock(TraceSegment)
540+
final ctx = new IastRequestContext(noOpTaintedObjects())
541+
final reqCtx = Mock(RequestContext)
542+
final spanId = 123456
543+
VulnerabilityBatch batch = null
544+
Map stackTraceBatch = new ConcurrentHashMap()
545+
546+
final span = Stub(AgentSpan)
547+
span.getRequestContext() >> reqCtx
548+
span.getSpanId() >> spanId
549+
550+
final location = Location.forSpanAndStack(span, new StackTraceElement("foo", "foo", "foo", 1))
551+
552+
final v1 = new Vulnerability(
553+
VulnerabilityType.INSECURE_COOKIE,
554+
location,
555+
new Evidence("JSESSIONID")
556+
)
557+
final v2 = new Vulnerability(
558+
VulnerabilityType.NO_SAMESITE_COOKIE,
559+
location,
560+
new Evidence("JSESSIONID")
561+
)
562+
563+
when:
564+
reporter.report(span, v1)
565+
reporter.report(span, v2)
566+
567+
then:
568+
1 * reqCtx.getData(RequestContextSlot.IAST) >> ctx
569+
2 * reqCtx.getTraceSegment() >> traceSegment
570+
// first vulnerability
571+
1 * traceSegment.getDataTop('iast') >> null
572+
1 * traceSegment.setDataTop('iast', _) >> { batch = it[1] as VulnerabilityBatch }
573+
1 * reqCtx.getOrCreateMetaStructTop('_dd.stack', _) >> { stackTraceBatch }
574+
// second vulnerability
575+
1 * traceSegment.getDataTop('iast') >> { return batch } // second vulnerability
576+
JSONAssert.assertEquals('''{
577+
"vulnerabilities": [
578+
{
579+
"evidence": { "value":"JSESSIONID" },
580+
"hash":1156210466,
581+
"location": {
582+
"spanId":123456,
583+
"line":1,
584+
"path":"foo",
585+
"method": "foo",
586+
"stackId":"1"
587+
},
588+
"type":"INSECURE_COOKIE"
589+
},
590+
{
591+
"evidence": {"value":"JSESSIONID"},
592+
"hash":1090504969,
593+
"location": {
594+
"spanId":123456,
595+
"line":1,
596+
"path":"foo",
597+
"method": "foo",
598+
"stackId":"1"
599+
},
600+
"type":"NO_SAMESITE_COOKIE"
601+
}
602+
]
603+
}''', batch.toString(), true)
604+
1 * traceSegment.setTagTop('asm.keep', true)
605+
1 * traceSegment.setTagTop('_dd.p.ts', ProductTraceSource.ASM)
606+
assert stackTraceBatch.get("vulnerability").size() == 1
607+
0 * _
608+
}
609+
536610
private AgentSpan spanWithBatch(final VulnerabilityBatch batch) {
537611
final traceSegment = Mock(TraceSegment) {
538612
getDataTop('iast') >> batch

‎dd-java-agent/agent-profiling/profiling-controller/src/main/java/com/datadog/profiling/controller/TempLocationManager.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -294,7 +294,7 @@ private TempLocationManager() {
294294
tempDir = baseTempDir.resolve(TEMPDIR_PREFIX + pid);
295295
if (runStartupCleanup) {
296296
// do not execute the background cleanup task when running in tests
297-
AgentTaskScheduler.INSTANCE.execute(() -> cleanup(false));
297+
AgentTaskScheduler.INSTANCE.execute(cleanupTask);
298298
}
299299

300300
Thread selfCleanup =

‎dd-java-agent/appsec/src/main/java/com/datadog/appsec/gateway/AppSecRequestContext.java‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,9 @@ public class AppSecRequestContext implements DataBundle, Closeable {
7777
public static final Set<String> RESPONSE_HEADERS_ALLOW_LIST =
7878
new TreeSet<>(
7979
Arrays.asList("content-length", "content-type", "content-encoding", "content-language"));
80+
public static final int DD_WAF_RUN_INTERNAL_ERROR = -3;
81+
public static final int DD_WAF_RUN_INVALID_OBJECT_ERROR = -2;
82+
public static final int DD_WAF_RUN_INVALID_ARGUMENT_ERROR = -1;
8083

8184
static {
8285
REQUEST_HEADERS_ALLOW_LIST.addAll(DEFAULT_REQUEST_HEADERS_ALLOW_LIST);
@@ -124,6 +127,12 @@ public class AppSecRequestContext implements DataBundle, Closeable {
124127
private volatile boolean blocked;
125128
private volatile int wafTimeouts;
126129
private volatile int raspTimeouts;
130+
private volatile int raspInternalErrors;
131+
private volatile int raspInvalidObjectErrors;
132+
private volatile int raspInvalidArgumentErrors;
133+
private volatile int wafInternalErrors;
134+
private volatile int wafInvalidObjectErrors;
135+
private volatile int wafInvalidArgumentErrors;
127136

128137
// keep a reference to the last published usr.id
129138
private volatile String userId;
@@ -139,6 +148,29 @@ public class AppSecRequestContext implements DataBundle, Closeable {
139148
private static final AtomicIntegerFieldUpdater<AppSecRequestContext> RASP_TIMEOUTS_UPDATER =
140149
AtomicIntegerFieldUpdater.newUpdater(AppSecRequestContext.class, "raspTimeouts");
141150

151+
private static final AtomicIntegerFieldUpdater<AppSecRequestContext>
152+
RASP_INTERNAL_ERRORS_UPDATER =
153+
AtomicIntegerFieldUpdater.newUpdater(AppSecRequestContext.class, "raspInternalErrors");
154+
private static final AtomicIntegerFieldUpdater<AppSecRequestContext>
155+
RASP_INVALID_OBJECT_ERRORS_UPDATER =
156+
AtomicIntegerFieldUpdater.newUpdater(
157+
AppSecRequestContext.class, "raspInvalidObjectErrors");
158+
private static final AtomicIntegerFieldUpdater<AppSecRequestContext>
159+
RASP_INVALID_ARGUMENT_ERRORS_UPDATER =
160+
AtomicIntegerFieldUpdater.newUpdater(
161+
AppSecRequestContext.class, "raspInvalidArgumentErrors");
162+
163+
private static final AtomicIntegerFieldUpdater<AppSecRequestContext> WAF_INTERNAL_ERRORS_UPDATER =
164+
AtomicIntegerFieldUpdater.newUpdater(AppSecRequestContext.class, "wafInternalErrors");
165+
private static final AtomicIntegerFieldUpdater<AppSecRequestContext>
166+
WAF_INVALID_OBJECT_ERRORS_UPDATER =
167+
AtomicIntegerFieldUpdater.newUpdater(
168+
AppSecRequestContext.class, "wafInvalidObjectErrors");
169+
private static final AtomicIntegerFieldUpdater<AppSecRequestContext>
170+
WAF_INVALID_ARGUMENT_ERRORS_UPDATER =
171+
AtomicIntegerFieldUpdater.newUpdater(
172+
AppSecRequestContext.class, "wafInvalidArgumentErrors");
173+
142174
// to be called by the Event Dispatcher
143175
public void addAll(DataBundle newData) {
144176
for (Map.Entry<Address<?>, Object> entry : newData) {
@@ -188,6 +220,38 @@ public void increaseRaspTimeouts() {
188220
RASP_TIMEOUTS_UPDATER.incrementAndGet(this);
189221
}
190222

223+
public void increaseRaspErrorCode(int code) {
224+
switch (code) {
225+
case DD_WAF_RUN_INTERNAL_ERROR:
226+
RASP_INTERNAL_ERRORS_UPDATER.incrementAndGet(this);
227+
break;
228+
case DD_WAF_RUN_INVALID_OBJECT_ERROR:
229+
RASP_INVALID_OBJECT_ERRORS_UPDATER.incrementAndGet(this);
230+
break;
231+
case DD_WAF_RUN_INVALID_ARGUMENT_ERROR:
232+
RASP_INVALID_ARGUMENT_ERRORS_UPDATER.incrementAndGet(this);
233+
break;
234+
default:
235+
break;
236+
}
237+
}
238+
239+
public void increaseWafErrorCode(int code) {
240+
switch (code) {
241+
case DD_WAF_RUN_INTERNAL_ERROR:
242+
WAF_INTERNAL_ERRORS_UPDATER.incrementAndGet(this);
243+
break;
244+
case DD_WAF_RUN_INVALID_OBJECT_ERROR:
245+
WAF_INVALID_OBJECT_ERRORS_UPDATER.incrementAndGet(this);
246+
break;
247+
case DD_WAF_RUN_INVALID_ARGUMENT_ERROR:
248+
WAF_INVALID_ARGUMENT_ERRORS_UPDATER.incrementAndGet(this);
249+
break;
250+
default:
251+
break;
252+
}
253+
}
254+
191255
public int getWafTimeouts() {
192256
return wafTimeouts;
193257
}
@@ -196,6 +260,32 @@ public int getRaspTimeouts() {
196260
return raspTimeouts;
197261
}
198262

263+
public int getRaspError(int code) {
264+
switch (code) {
265+
case DD_WAF_RUN_INTERNAL_ERROR:
266+
return raspInternalErrors;
267+
case DD_WAF_RUN_INVALID_OBJECT_ERROR:
268+
return raspInvalidObjectErrors;
269+
case DD_WAF_RUN_INVALID_ARGUMENT_ERROR:
270+
return raspInvalidArgumentErrors;
271+
default:
272+
return 0;
273+
}
274+
}
275+
276+
public int getWafError(int code) {
277+
switch (code) {
278+
case DD_WAF_RUN_INTERNAL_ERROR:
279+
return wafInternalErrors;
280+
case DD_WAF_RUN_INVALID_OBJECT_ERROR:
281+
return wafInvalidObjectErrors;
282+
case DD_WAF_RUN_INVALID_ARGUMENT_ERROR:
283+
return wafInvalidArgumentErrors;
284+
default:
285+
return 0;
286+
}
287+
}
288+
199289
public Additive getOrCreateAdditive(PowerwafContext ctx, boolean createMetrics, boolean isRasp) {
200290

201291
if (createMetrics) {

0 commit comments

Comments
 (0)