Skip to content

Commit 89c5a54

Browse files
funyjanefunyjane
authored
Merge branch 'main' into config-revamp/span-attribute-schema-version
2 parents 17d37a9 + 625c602 commit 89c5a54

12 files changed

Lines changed: 332 additions & 31 deletions

File tree

‎internal/appsec/config/wafmanager.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,7 @@ func (m *WAFManager) RestoreDefaultConfig() error {
176176
}
177177

178178
func logLocalDiagnosticMessages(name string, feature *libddwaf.Feature) {
179-
logger := telemetryLog.With(telemetry.WithTags([]string{"appsec_config_key:" + name, "log_type:local::diagnostic"}))
179+
logger := telemetryLog.With(telemetry.WithTags([]string{"appsec_config_key:" + name, "log_type:appsec::waf::ruleset::diagnostic"}))
180180

181181
if feature.Error != "" {
182182
logger.Error("feature error", slog.String("message", feature.Error))

‎internal/appsec/emitter/waf/metrics.go‎

Lines changed: 102 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,10 @@ package waf
77

88
import (
99
"errors"
10+
"fmt"
1011
"log/slog"
1112
"strconv"
13+
"strings"
1214
"sync"
1315
"sync/atomic"
1416
"time"
@@ -31,7 +33,7 @@ var changeToWafUpdates sync.Once
3133
// RequestMilestones is a list of things that can happen as a result of a waf call. They are stacked for each requests
3234
// and used as tags to the telemetry metric `waf.requests`.
3335
// this struct can be modified concurrently.
34-
// TODO: add request_excluded and block_failure to the mix once we have the capability to track them
36+
// TODO: add request_excluded to the mix once we have the capability to track it (blocked on libddwaf)
3537
type RequestMilestones struct {
3638
requestBlocked bool
3739
ruleTriggered bool
@@ -70,6 +72,8 @@ type HandleMetrics struct {
7072
raspTimeout [len(addresses.RASPRuleTypes)]telemetry.MetricHandle
7173
// raspRuleEval holds the telemetry metrics for the `rasp.rule_eval` metric by rule type
7274
raspRuleEval [len(addresses.RASPRuleTypes)]telemetry.MetricHandle
75+
// raspRuleSkipped holds the `rasp.rule.skipped` count metric, lazily filled by rule_type+reason
76+
raspRuleSkipped *xsync.MapOf[raspMetricKey[string], telemetry.MetricHandle]
7377

7478
// Rare metric types
7579

@@ -132,6 +136,7 @@ func NewMetricsInstance(newHandle *libddwaf.Handle, eventRulesVersion string) Ha
132136
wafErrorCount: xsync.NewMapOf[int, telemetry.MetricHandle](xsync.WithGrowOnly(), xsync.WithPresize(2^3)),
133137
raspErrorCount: xsync.NewMapOf[raspMetricKey[int], telemetry.MetricHandle](xsync.WithGrowOnly(), xsync.WithPresize(2^3)),
134138
raspRuleMatch: xsync.NewMapOf[raspMetricKey[string], telemetry.MetricHandle](xsync.WithGrowOnly(), xsync.WithPresize(2^3)),
139+
raspRuleSkipped: xsync.NewMapOf[raspMetricKey[string], telemetry.MetricHandle](xsync.WithGrowOnly(), xsync.WithPresize(2^3)),
135140
}
136141

137142
for ruleType := range metrics.baseRASPTags {
@@ -179,10 +184,19 @@ type ContextMetrics struct {
179184

180185
// SumRASPCalls is the sum of all the RASP calls made by the WAF whatever the rasp rule type it is.
181186
SumRASPCalls atomic.Uint32
182-
// SumWAFErrors is the sum of all the WAF errors that happened not in the RASP scope.
183-
SumWAFErrors atomic.Uint32
184-
// SumRASPErrors is the sum of all the RASP errors that happened in the RASP scope.
185-
SumRASPErrors atomic.Uint32
187+
// WAFErrorCode is the closest-to-zero (least-negative) ddwaf_run error code seen in the WAF scope.
188+
// Zero means no error occurred. See RFC-1012.
189+
WAFErrorCode atomic.Int32
190+
// RASPErrorCodes holds the closest-to-zero ddwaf_run error code per RASP rule type.
191+
// Zero means no error occurred for that rule type. See RFC-1012.
192+
RASPErrorCodes [len(addresses.RASPRuleTypes)]atomic.Int32
193+
194+
// exceptionOnce + exception store the first exception atomically.
195+
// sync.Once serialises the write; atomic.Pointer makes the stored value
196+
// visible to readers (e.g. AddWAFMonitoringTags) without a happens-before
197+
// relationship to the Once.Do call.
198+
exceptionOnce sync.Once
199+
exception atomic.Pointer[exceptionRecord]
186200

187201
// SumWAFTimeouts is the sum of all the WAF timeouts that happened not in the RASP scope.
188202
SumWAFTimeouts atomic.Uint32
@@ -304,7 +318,7 @@ func (m *ContextMetrics) RegisterWafRun(addrs libddwaf.RunAddressData, timerStat
304318
}
305319
if tags.ruleTriggered {
306320
blockTag := "block:irrelevant"
307-
if tags.requestBlocked { // TODO: add block:failure to the mix
321+
if tags.requestBlocked {
308322
blockTag = "block:success"
309323
}
310324

@@ -351,56 +365,128 @@ func (m *ContextMetrics) IncWafError(addrs libddwaf.RunAddressData, in error) {
351365
}
352366

353367
if !errors.Is(in, waferrors.ErrTimeout) {
354-
logger := m.logger.With(telemetry.WithTags(m.baseTags))
355-
// This a known error origin all the ways to the tip of the error chain and since it impact WAF
356-
// behavior we really want to log it so we can investigate it so we don't wrap it in a safe error
357-
logger.Error("unexpected WAF error", slog.Any("error", telemetrylog.NewSafeError(in)))
368+
switch addrs.TimerKey {
369+
case addresses.RASPScope:
370+
m.RecordException(ExceptionTypeRASP, in)
371+
default:
372+
m.RecordException(ExceptionTypeWAF, in)
373+
}
358374
}
359375

360376
switch addrs.TimerKey {
361377
case addresses.RASPScope:
362378
ruleType, ok := addresses.RASPRuleTypeFromAddressSet(addrs)
363379
if !ok {
364-
m.logger.Error("unexpected call to RASPRuleTypeFromAddressSet", slog.Any("error", telemetrylog.NewSafeError(in)))
380+
m.RecordException(ExceptionTypeInstrumentation, fmt.Errorf("unknown RASP rule type for addresses %v: %w", addrs, in))
381+
return
365382
}
366383
m.raspError(in, ruleType)
367384
case addresses.WAFScope, "":
368385
m.wafError(in)
369386
default:
370-
m.logger.Error("unexpected scope name", slog.String("scope", string(addrs.TimerKey)))
387+
m.RecordException(ExceptionTypeInstrumentation, fmt.Errorf("unexpected WAF error scope %q: %w", addrs.TimerKey, in))
371388
}
372389
}
373390

374391
// defaultWafErrorCode is the default error code if the error does not implement [libddwaf.RunError]
375392
// meaning if the error actual come for the bindings and not from the WAF itself
376393
const defaultWafErrorCode = -127
377394

395+
// updateClosestToZero atomically sets target to the closer-to-zero of its current
396+
// value and code. Codes are always ≤ 0; zero is the sentinel "no error".
397+
func updateClosestToZero(target *atomic.Int32, code int32) {
398+
for {
399+
current := target.Load()
400+
if current != 0 && code <= current {
401+
return // current is already closer to (or as close as) zero
402+
}
403+
if target.CompareAndSwap(current, code) {
404+
return
405+
}
406+
}
407+
}
408+
409+
// Exception log_type constants per RFC-1012.
410+
const (
411+
ExceptionTypeWAF = "appsec::waf::exception"
412+
ExceptionTypeRASP = "appsec::rasp::exception"
413+
ExceptionTypeInstrumentation = "appsec::instrumentation::exception"
414+
)
415+
416+
const maxExceptionMsgBytes = 512
417+
418+
type exceptionRecord struct{ typ, msg string }
419+
420+
// RecordException records the first exception that occurred during a request's WAF execution.
421+
// The span tags _dd.appsec.error.type and _dd.appsec.error.message are set once per request
422+
// (first-error-wins). All calls emit a telemetry log with the RFC-1012 log_type and a stack trace.
423+
func (m *ContextMetrics) RecordException(errType string, err error) {
424+
m.exceptionOnce.Do(func() {
425+
msg := err.Error()
426+
if len(msg) > maxExceptionMsgBytes {
427+
msg = strings.ToValidUTF8(msg[:maxExceptionMsgBytes], "")
428+
}
429+
m.exception.Store(&exceptionRecord{typ: errType, msg: msg})
430+
})
431+
logger := m.logger.With(
432+
telemetry.WithTags([]string{"log_type:" + errType}),
433+
telemetry.WithStacktrace(),
434+
)
435+
logger.Error("appsec exception", slog.Any("error", telemetrylog.NewSafeError(err)))
436+
}
437+
438+
// ExceptionType returns the errType from the first RecordException call, or empty string if none.
439+
func (m *ContextMetrics) ExceptionType() string {
440+
if r := m.exception.Load(); r != nil {
441+
return r.typ
442+
}
443+
return ""
444+
}
445+
446+
// ExceptionMsg returns the (truncated) error message from the first RecordException call.
447+
func (m *ContextMetrics) ExceptionMsg() string {
448+
if r := m.exception.Load(); r != nil {
449+
return r.msg
450+
}
451+
return ""
452+
}
378453
func (m *ContextMetrics) wafError(in error) {
379-
m.SumWAFErrors.Add(1)
380454
errCode := defaultWafErrorCode
381455
if code := waferrors.ToWafErrorCode(in); code != 0 {
382456
errCode = code
383457
}
458+
updateClosestToZero(&m.WAFErrorCode, int32(errCode))
384459

385460
handle, _ := m.wafErrorCount.LoadOrCompute(errCode, func() telemetry.MetricHandle {
386461
return telemetry.Count(telemetry.NamespaceAppSec, "waf.error", append([]string{
387-
"error_code:" + strconv.Itoa(errCode),
462+
"waf_error:" + strconv.Itoa(errCode),
388463
}, m.baseTags...))
389464
})
390465

391466
handle.Submit(1)
392467
}
393468

469+
// SkipRASPRule records a skipped RASP rule evaluation for the given rule type and reason.
470+
// Valid reasons per RFC-1012: "app-startup", "before-request", "after-request", "out-of-request".
471+
func (m *ContextMetrics) SkipRASPRule(ruleType addresses.RASPRuleType, reason string) {
472+
handle, _ := m.raspRuleSkipped.LoadOrCompute(raspMetricKey[string]{typ: ruleType, additionalTag: reason}, func() telemetry.MetricHandle {
473+
return telemetry.Count(telemetry.NamespaceAppSec, "rasp.rule.skipped", append([]string{
474+
"reason:" + reason,
475+
}, m.baseRASPTags[ruleType]...))
476+
})
477+
handle.Submit(1)
478+
}
479+
394480
func (m *ContextMetrics) raspError(in error, ruleType addresses.RASPRuleType) {
395-
m.SumRASPErrors.Add(1)
396481
errCode := defaultWafErrorCode
397482
if code := waferrors.ToWafErrorCode(in); code != 0 {
398483
errCode = code
399484
}
485+
updateClosestToZero(&m.RASPErrorCodes[ruleType], int32(errCode))
400486

401487
handle, _ := m.raspErrorCount.LoadOrCompute(raspMetricKey[int]{typ: ruleType, additionalTag: errCode}, func() telemetry.MetricHandle {
402488
return telemetry.Count(telemetry.NamespaceAppSec, "rasp.error", append([]string{
403-
"error_code:" + strconv.Itoa(errCode),
489+
"waf_error:" + strconv.Itoa(errCode),
404490
}, m.baseRASPTags[ruleType]...))
405491
})
406492

‎internal/appsec/emitter/waf/metrics_test.go‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
// Unless explicitly stated otherwise all files in this repository are licensed
2+
// under the Apache License Version 2.0.
3+
// This product includes software developed at Datadog (https://www.datadoghq.com/).
4+
// Copyright 2025 Datadog, Inc.
5+
6+
package waf
7+
8+
import (
9+
"sync/atomic"
10+
"testing"
11+
12+
"github.com/stretchr/testify/require"
13+
)
14+
15+
func TestUpdateClosestToZero(t *testing.T) {
16+
t.Run("first_error_sets_code", func(t *testing.T) {
17+
var target atomic.Int32
18+
updateClosestToZero(&target, -2)
19+
require.Equal(t, int32(-2), target.Load())
20+
})
21+
22+
t.Run("closer_to_zero_replaces", func(t *testing.T) {
23+
var target atomic.Int32
24+
updateClosestToZero(&target, -127)
25+
updateClosestToZero(&target, -2)
26+
updateClosestToZero(&target, -1)
27+
require.Equal(t, int32(-1), target.Load())
28+
})
29+
30+
t.Run("further_from_zero_ignored", func(t *testing.T) {
31+
var target atomic.Int32
32+
updateClosestToZero(&target, -1)
33+
updateClosestToZero(&target, -127)
34+
require.Equal(t, int32(-1), target.Load())
35+
})
36+
37+
t.Run("equal_code_no_change", func(t *testing.T) {
38+
var target atomic.Int32
39+
updateClosestToZero(&target, -1)
40+
updateClosestToZero(&target, -1)
41+
require.Equal(t, int32(-1), target.Load())
42+
})
43+
44+
t.Run("sentinel_zero_always_replaced", func(t *testing.T) {
45+
var target atomic.Int32
46+
// initial zero sentinel — any error code should overwrite it
47+
updateClosestToZero(&target, -127)
48+
require.Equal(t, int32(-127), target.Load())
49+
})
50+
}

‎internal/appsec/emitter/waf/run.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717
"github.com/DataDog/dd-trace-go/v2/ddtrace/ext"
1818
"github.com/DataDog/dd-trace-go/v2/instrumentation/appsec/dyngo"
1919
"github.com/DataDog/dd-trace-go/v2/instrumentation/appsec/emitter/waf/actions"
20+
"github.com/DataDog/dd-trace-go/v2/instrumentation/appsec/emitter/waf/addresses"
2021
"github.com/DataDog/dd-trace-go/v2/internal/log"
2122
"github.com/DataDog/dd-trace-go/v2/internal/samplernames"
2223
)
@@ -27,6 +28,11 @@ import (
2728
func (op *ContextOperation) Run(eventReceiver dyngo.Operation, addrs libddwaf.RunAddressData) {
2829
ctx := op.context.Load()
2930
if ctx == nil { // Context was closed concurrently
31+
if addrs.TimerKey == addresses.RASPScope && op.metrics != nil {
32+
if ruleType, ok := addresses.RASPRuleTypeFromAddressSet(addrs); ok {
33+
op.metrics.SkipRASPRule(ruleType, "after-request")
34+
}
35+
}
3036
return
3137
}
3238

‎internal/appsec/listener/httpsec/http.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,14 @@ func (feature *Feature) OnResponse(op *httpsec.HandlerOperation, resp httpsec.Ha
102102
}
103103
}
104104
telemetry.Count(telemetry.NamespaceAppSec, "api_security.request."+metric, []string{"framework:" + op.Framework()}).Submit(1)
105+
106+
// Only emit for 2xx: redirects (3xx) and errors (4xx/5xx) legitimately have
107+
// no route in routing-capable frameworks (e.g. gin trailing-slash redirects,
108+
// 404 no-match). A framework that genuinely lacks routing will still trigger
109+
// this metric for its successful responses.
110+
if feature.APISec.Enabled && op.Route() == "" && resp.StatusCode >= 200 && resp.StatusCode < 300 {
111+
telemetry.Count(telemetry.NamespaceAppSec, "api_security.missing_route", []string{"framework:" + op.Framework()}).Submit(1)
112+
}
105113
}
106114

107115
// shouldExtractShema checks that API Security is enabled and that sampling rate

‎internal/appsec/listener/httpsec/roundtripper.go‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,10 @@ import (
1616
"github.com/DataDog/dd-trace-go/v2/instrumentation/appsec/emitter/waf/addresses"
1717
"github.com/DataDog/dd-trace-go/v2/internal/appsec/body"
1818
"github.com/DataDog/dd-trace-go/v2/internal/appsec/config"
19+
wafemitter "github.com/DataDog/dd-trace-go/v2/internal/appsec/emitter/waf"
1920
"github.com/DataDog/dd-trace-go/v2/internal/appsec/listener"
2021
"github.com/DataDog/dd-trace-go/v2/internal/log"
22+
"github.com/DataDog/dd-trace-go/v2/internal/telemetry"
2123
telemetrylog "github.com/DataDog/dd-trace-go/v2/internal/telemetry/log"
2224
)
2325

@@ -93,7 +95,10 @@ func (feature *DownwardRequestFeature) OnStart(op *httpsec.RoundTripOperation, a
9395
encodable, err := body.NewEncodable(http.Header(args.Headers).Get("Content-Type"), args.Body, maxBodyParseSize)
9496
if err != nil {
9597
log.Debug("Unsupported response body content type or error reading body: %s", err.Error())
96-
telemetrylog.Warn("Unsupported request body content type or error reading body", slog.Any("error", telemetrylog.NewSafeError(err)))
98+
telemetrylog.With(
99+
telemetry.WithTags([]string{"log_type:" + wafemitter.ExceptionTypeInstrumentation}),
100+
telemetry.WithStacktrace(),
101+
).Warn("Unsupported request body content type or error reading body", slog.Any("error", telemetrylog.NewSafeError(err)))
97102
}
98103
op.SetRequestBody(encodable)
99104
builder = builder.WithDownwardRequestBody(encodable)
@@ -153,7 +158,10 @@ func (feature *DownwardRequestFeature) OnFinish(op *httpsec.RoundTripOperation,
153158
encodable, err := body.NewEncodable(http.Header(args.Headers).Get("Content-Type"), args.Body, maxBodyParseSize)
154159
if err != nil {
155160
log.Debug("Unsupported response body content type or error reading body: %s", err.Error())
156-
telemetrylog.Warn("Unsupported response body content type or error reading body", slog.Any("error", telemetrylog.NewSafeError(err)))
161+
telemetrylog.With(
162+
telemetry.WithTags([]string{"log_type:" + wafemitter.ExceptionTypeInstrumentation}),
163+
telemetry.WithStacktrace(),
164+
).Warn("Unsupported response body content type or error reading body", slog.Any("error", telemetrylog.NewSafeError(err)))
157165
}
158166
builder = builder.WithDownwardResponseBody(encodable)
159167
}

‎internal/appsec/listener/usersec/usec.go‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,13 @@ package usersec
77

88
import (
99
"github.com/DataDog/dd-trace-go/v2/instrumentation/appsec/dyngo"
10+
emitterhttpsec "github.com/DataDog/dd-trace-go/v2/instrumentation/appsec/emitter/httpsec"
1011
"github.com/DataDog/dd-trace-go/v2/instrumentation/appsec/emitter/waf/addresses"
1112
"github.com/DataDog/dd-trace-go/v2/internal/appsec/config"
1213
"github.com/DataDog/dd-trace-go/v2/internal/appsec/emitter/usersec"
1314
"github.com/DataDog/dd-trace-go/v2/internal/appsec/emitter/waf"
1415
"github.com/DataDog/dd-trace-go/v2/internal/appsec/listener"
16+
"github.com/DataDog/dd-trace-go/v2/internal/telemetry"
1517
)
1618

1719
type Feature struct{}
@@ -38,7 +40,45 @@ func NewUserSecFeature(cfg *config.Config, rootOp dyngo.Operation) (listener.Fea
3840
return feature, nil
3941
}
4042

43+
// eventTypeName maps UserEventType to the RFC-1012 event_type tag value.
44+
func eventTypeName(t usersec.UserEventType) string {
45+
switch t {
46+
case usersec.UserLoginSuccess:
47+
return "login_success"
48+
case usersec.UserLoginFailure:
49+
return "login_failure"
50+
case usersec.UserSet:
51+
return "authenticated_request"
52+
default:
53+
return "unknown"
54+
}
55+
}
56+
57+
// frameworkFromOp walks the operation tree upward to find a HandlerOperation and returns its
58+
// framework name. Falls back to "unknown" if none is found.
59+
func frameworkFromOp(op dyngo.Operation) string {
60+
for current := op.Parent(); current != nil; current = current.Parent() {
61+
if h, ok := current.(*emitterhttpsec.HandlerOperation); ok {
62+
return h.Framework()
63+
}
64+
}
65+
return "unknown"
66+
}
67+
4168
func (*Feature) OnFinish(op *usersec.UserLoginOperation, res usersec.UserLoginOperationRes) {
69+
eventType := eventTypeName(op.EventType)
70+
framework := frameworkFromOp(op)
71+
tags := []string{"event_type:" + eventType, "framework:" + framework}
72+
73+
if res.UserLogin == "" &&
74+
(op.EventType == usersec.UserLoginSuccess || op.EventType == usersec.UserLoginFailure) {
75+
telemetry.Count(telemetry.NamespaceAppSec, "instrum.user_auth.missing_user_login", tags).Submit(1)
76+
}
77+
if res.UserID == "" &&
78+
(op.EventType == usersec.UserLoginSuccess || op.EventType == usersec.UserSet) {
79+
telemetry.Count(telemetry.NamespaceAppSec, "instrum.user_auth.missing_user_id", tags).Submit(1)
80+
}
81+
4282
builder := addresses.NewAddressesBuilder().
4383
WithUserID(res.UserID).
4484
WithUserLogin(res.UserLogin).

0 commit comments

Comments
 (0)