Skip to content

Commit d54f5f9

Browse files
gabedosgabedos
authored
[workloadfilter]: Refactor Filter Catalog (#41958)
### What does this PR do? Replaces the `FilterProgram` implementation of `CELProgram` with `LegacyProgram` backed by `container.Filter` when available. Cleans up unneeded code flows. ### Motivation For the last few Agent versions, the rule engine underlying workload filtering was `cel-go` programs. We were converting legacy input format into CEL expressions. However, it is more efficient to use regex directly than pay for the CEL overhead. Hence, for legacy input that only define simple regex, we can continue to use the `containers.Filter` as the evaluator. ### Describe how you validated your changes CI ### Additional Notes
1 parent 8ca9b5c commit d54f5f9

File tree

16 files changed

+182
-510
lines changed

16 files changed

+182
-510
lines changed

‎comp/core/workloadfilter/catalog/common.go‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,9 @@ package catalog
99
// This file contains filter programs that can be shared across different entity types.
1010

1111
import (
12+
log "github.com/DataDog/datadog-agent/comp/core/log/def"
1213
"github.com/DataDog/datadog-agent/comp/core/workloadfilter/program"
14+
"github.com/DataDog/datadog-agent/pkg/util/containers"
1315
)
1416

1517
// AutodiscoveryAnnotations creates a CEL program for autodiscovery annotations.
@@ -35,3 +37,49 @@ func AutodiscoveryLogsAnnotations() program.FilterProgram {
3537
ExcludePrefix: "logs_",
3638
}
3739
}
40+
41+
// LegacyContainerGlobalProgram creates a legacy filter program for global containerized filtering
42+
func LegacyContainerGlobalProgram(cfg *FilterConfig, logger log.Component) program.FilterProgram {
43+
return createLegacyContainerProgram("LegacyContainerGlobalProgram", cfg.ContainerInclude, cfg.ContainerExclude, logger)
44+
}
45+
46+
// LegacyContainerMetricsProgram creates a legacy filter program for containerized metrics filtering
47+
func LegacyContainerMetricsProgram(cfg *FilterConfig, logger log.Component) program.FilterProgram {
48+
return createLegacyContainerProgram("LegacyContainerMetricsProgram", cfg.ContainerIncludeMetrics, cfg.ContainerExcludeMetrics, logger)
49+
}
50+
51+
// LegacyContainerLogsProgram creates a legacy filter program for containerized logs filtering
52+
func LegacyContainerLogsProgram(cfg *FilterConfig, logger log.Component) program.FilterProgram {
53+
return createLegacyContainerProgram("LegacyContainerLogsProgram", cfg.ContainerIncludeLogs, cfg.ContainerExcludeLogs, logger)
54+
}
55+
56+
// LegacyContainerACExcludeProgram creates a legacy filter program for containerized AC exclusion filtering
57+
func LegacyContainerACExcludeProgram(cfg *FilterConfig, logger log.Component) program.FilterProgram {
58+
return createLegacyContainerProgram("LegacyContainerACExcludeProgram", nil, cfg.ACExclude, logger)
59+
}
60+
61+
// LegacyContainerACIncludeProgram creates a legacy filter program for containerized AC inclusion filtering
62+
func LegacyContainerACIncludeProgram(cfg *FilterConfig, logger log.Component) program.FilterProgram {
63+
return createLegacyContainerProgram("LegacyContainerACIncludeProgram", cfg.ACInclude, nil, logger)
64+
}
65+
66+
// LegacyContainerSBOMProgram creates a legacy filter program for containerized SBOM filtering
67+
func LegacyContainerSBOMProgram(cfg *FilterConfig, logger log.Component) program.FilterProgram {
68+
return createLegacyContainerProgram("LegacyContainerSBOMProgram", cfg.SBOMContainerInclude, cfg.SBOMContainerExclude, logger)
69+
}
70+
71+
func createLegacyContainerProgram(programName string, include, exclude []string, logger log.Component) program.FilterProgram {
72+
var initErrors []error
73+
74+
filter, err := containers.NewFilter(containers.GlobalFilter, include, exclude)
75+
if err != nil {
76+
initErrors = append(initErrors, err)
77+
logger.Warnf("Failed to create filter '%s': %v", programName, err)
78+
}
79+
80+
return program.LegacyFilterProgram{
81+
Name: programName,
82+
Filter: filter,
83+
InitializationErrors: initErrors,
84+
}
85+
}

‎comp/core/workloadfilter/catalog/container.go‎

Lines changed: 1 addition & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -7,92 +7,15 @@
77
package catalog
88

99
import (
10-
"regexp"
11-
"strings"
12-
1310
log "github.com/DataDog/datadog-agent/comp/core/log/def"
1411
workloadfilter "github.com/DataDog/datadog-agent/comp/core/workloadfilter/def"
1512
"github.com/DataDog/datadog-agent/comp/core/workloadfilter/program"
1613
"github.com/DataDog/datadog-agent/pkg/util/containers"
1714
)
1815

19-
// LegacyContainerMetricsProgram creates a program for filtering container metrics
20-
func LegacyContainerMetricsProgram(filterConfig *FilterConfig, logger log.Component) program.FilterProgram {
21-
programName := "LegacyContainerMetricsProgram"
22-
include := filterConfig.ContainerIncludeMetrics
23-
exclude := filterConfig.ContainerExcludeMetrics
24-
return createFromOldFilters(programName, include, exclude, workloadfilter.ContainerType, logger)
25-
}
26-
27-
// LegacyContainerLogsProgram creates a program for filtering container logs
28-
func LegacyContainerLogsProgram(filterConfig *FilterConfig, logger log.Component) program.FilterProgram {
29-
programName := "LegacyContainerLogsProgram"
30-
include := filterConfig.ContainerIncludeLogs
31-
exclude := filterConfig.ContainerExcludeLogs
32-
return createFromOldFilters(programName, include, exclude, workloadfilter.ContainerType, logger)
33-
}
34-
35-
// LegacyContainerACExcludeProgram creates a program for excluding containers via legacy `AC` filters
36-
func LegacyContainerACExcludeProgram(filterConfig *FilterConfig, logger log.Component) program.FilterProgram {
37-
programName := "LegacyContainerACExcludeProgram"
38-
exclude := filterConfig.ACExclude
39-
return createFromOldFilters(programName, nil, exclude, workloadfilter.ContainerType, logger)
40-
}
41-
42-
// LegacyContainerACIncludeProgram creates a program for including containers via legacy `AC` filters
43-
func LegacyContainerACIncludeProgram(filterConfig *FilterConfig, logger log.Component) program.FilterProgram {
44-
programName := "LegacyContainerACIncludeProgram"
45-
include := filterConfig.ACInclude
46-
return createFromOldFilters(programName, include, nil, workloadfilter.ContainerType, logger)
47-
}
48-
49-
// LegacyContainerGlobalProgram creates a program for filtering container globally
50-
func LegacyContainerGlobalProgram(filterConfig *FilterConfig, logger log.Component) program.FilterProgram {
51-
programName := "LegacyContainerGlobalProgram"
52-
include := filterConfig.ContainerInclude
53-
exclude := filterConfig.ContainerExclude
54-
return createFromOldFilters(programName, include, exclude, workloadfilter.ContainerType, logger)
55-
}
56-
57-
// LegacyContainerSBOMProgram creates a program for filtering container SBOMs
58-
func LegacyContainerSBOMProgram(filterConfig *FilterConfig, logger log.Component) program.FilterProgram {
59-
programName := "LegacyContainerSBOMProgram"
60-
include := filterConfig.SBOMContainerInclude
61-
exclude := filterConfig.SBOMContainerExclude
62-
return createFromOldFilters(programName, include, exclude, workloadfilter.ContainerType, logger)
63-
}
64-
6516
// ContainerPausedProgram creates a program for filtering paused containers
6617
func ContainerPausedProgram(_ *FilterConfig, logger log.Component) program.FilterProgram {
67-
programName := "ContainerPausedProgram"
68-
var initErrors []error
69-
70-
exclude := containers.GetPauseContainerExcludeList()
71-
72-
excludeRegex := make([]*regexp.Regexp, 0, len(exclude))
73-
for _, pattern := range exclude {
74-
pattern = strings.TrimPrefix(pattern, "image:")
75-
regex, err := regexp.Compile(pattern)
76-
if err != nil {
77-
initErrors = append(initErrors, err)
78-
logger.Warnf("Error compiling regex pattern for %s: %v", programName, err)
79-
continue
80-
}
81-
excludeRegex = append(excludeRegex, regex)
82-
}
83-
84-
return &program.RegexProgram{
85-
Name: programName,
86-
ExcludeRegex: excludeRegex,
87-
ExtractField: func(entity workloadfilter.Filterable) string {
88-
container, ok := entity.(*workloadfilter.Container)
89-
if !ok {
90-
return ""
91-
}
92-
return container.GetImage()
93-
},
94-
InitializationErrors: initErrors,
95-
}
18+
return createLegacyContainerProgram("ContainerPausedProgram", nil, containers.GetPauseContainerExcludeList(), logger)
9619
}
9720

9821
// ContainerCELMetricsProgram creates a program for filtering container metrics via CEL rules

‎comp/core/workloadfilter/catalog/kube_endpoints.go‎

Lines changed: 0 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -12,22 +12,6 @@ import (
1212
"github.com/DataDog/datadog-agent/comp/core/workloadfilter/program"
1313
)
1414

15-
// LegacyEndpointsMetricsProgram creates a program for filtering endpoints metrics
16-
func LegacyEndpointsMetricsProgram(filterConfig *FilterConfig, logger log.Component) program.FilterProgram {
17-
programName := "LegacyEndpointsMetricsProgram"
18-
include := filterConfig.ContainerIncludeMetrics
19-
exclude := filterConfig.ContainerExcludeMetrics
20-
return createFromOldFilters(programName, include, exclude, workloadfilter.EndpointType, logger)
21-
}
22-
23-
// LegacyEndpointsGlobalProgram creates a program for filtering endpoints globally
24-
func LegacyEndpointsGlobalProgram(filterConfig *FilterConfig, logger log.Component) program.FilterProgram {
25-
programName := "LegacyEndpointsGlobalProgram"
26-
includeList := filterConfig.GetLegacyContainerInclude()
27-
excludeList := filterConfig.GetLegacyContainerExclude()
28-
return createFromOldFilters(programName, includeList, excludeList, workloadfilter.EndpointType, logger)
29-
}
30-
3115
// EndpointCELMetricsProgram creates a program for filtering endpoints metrics via CEL rules
3216
func EndpointCELMetricsProgram(filterConfig *FilterConfig, logger log.Component) program.FilterProgram {
3317
programName := "EndpointCELMetricsProgram"

‎comp/core/workloadfilter/catalog/kube_service.go‎

Lines changed: 0 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -12,22 +12,6 @@ import (
1212
"github.com/DataDog/datadog-agent/comp/core/workloadfilter/program"
1313
)
1414

15-
// LegacyServiceMetricsProgram creates a program for filtering service metrics
16-
func LegacyServiceMetricsProgram(filterConfig *FilterConfig, logger log.Component) program.FilterProgram {
17-
programName := "LegacyServiceMetricsProgram"
18-
include := filterConfig.ContainerIncludeMetrics
19-
exclude := filterConfig.ContainerExcludeMetrics
20-
return createFromOldFilters(programName, include, exclude, workloadfilter.ServiceType, logger)
21-
}
22-
23-
// LegacyServiceGlobalProgram creates a program for filtering services globally
24-
func LegacyServiceGlobalProgram(filterConfig *FilterConfig, logger log.Component) program.FilterProgram {
25-
programName := "LegacyServiceGlobalProgram"
26-
includeList := filterConfig.GetLegacyContainerInclude()
27-
excludeList := filterConfig.GetLegacyContainerExclude()
28-
return createFromOldFilters(programName, includeList, excludeList, workloadfilter.ServiceType, logger)
29-
}
30-
3115
// ServiceCELMetricsProgram creates a program for filtering services metrics via CEL rules
3216
func ServiceCELMetricsProgram(filterConfig *FilterConfig, logger log.Component) program.FilterProgram {
3317
programName := "ServiceCELMetricsProgram"

‎comp/core/workloadfilter/catalog/pod.go‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -12,14 +12,6 @@ import (
1212
"github.com/DataDog/datadog-agent/comp/core/workloadfilter/program"
1313
)
1414

15-
// LegacyPodProgram creates a program for filtering legacy pods.
16-
func LegacyPodProgram(filterConfig *FilterConfig, loggger log.Component) program.FilterProgram {
17-
programName := "LegacyPodProgram"
18-
includeList := append(filterConfig.ContainerInclude, filterConfig.ContainerIncludeMetrics...)
19-
excludeList := append(filterConfig.ContainerExclude, filterConfig.ContainerExcludeMetrics...)
20-
return createFromOldFilters(programName, includeList, excludeList, workloadfilter.PodType, loggger)
21-
}
22-
2315
// PodCELMetricsProgram creates a program for filtering pods metrics via CEL rules
2416
func PodCELMetricsProgram(filterConfig *FilterConfig, logger log.Component) program.FilterProgram {
2517
programName := "PodCELMetricsProgram"

‎comp/core/workloadfilter/catalog/utils.go‎

Lines changed: 2 additions & 147 deletions
Original file line numberDiff line numberDiff line change
@@ -9,162 +9,17 @@
99
package catalog
1010

1111
import (
12-
"fmt"
1312
"os"
14-
"strconv"
15-
"strings"
16-
17-
"github.com/google/cel-go/cel"
1813

1914
log "github.com/DataDog/datadog-agent/comp/core/log/def"
2015
"github.com/DataDog/datadog-agent/comp/core/workloadfilter/program"
16+
"github.com/DataDog/datadog-agent/comp/core/workloadfilter/util/celprogram"
2117

2218
workloadfilter "github.com/DataDog/datadog-agent/comp/core/workloadfilter/def"
23-
legacyFilter "github.com/DataDog/datadog-agent/pkg/util/containers"
2419
)
2520

26-
func createFromOldFilters(name string, oldInclude, oldExclude []string, objectType workloadfilter.ResourceType, logger log.Component) program.FilterProgram {
27-
var initErrors []error
28-
29-
includeProgram, includeErr := createProgramFromOldFilters(oldInclude, objectType)
30-
if includeErr != nil {
31-
initErrors = append(initErrors, includeErr)
32-
logger.Warnf("error creating include program for %s: %v", name, includeErr)
33-
}
34-
35-
excludeProgram, excludeErr := createProgramFromOldFilters(oldExclude, objectType)
36-
if excludeErr != nil {
37-
initErrors = append(initErrors, excludeErr)
38-
logger.Warnf("error creating exclude program for %s: %v", name, excludeErr)
39-
}
40-
41-
return program.CELProgram{
42-
Name: name,
43-
Include: includeProgram,
44-
Exclude: excludeProgram,
45-
InitializationErrors: initErrors,
46-
}
47-
}
48-
49-
// createProgramFromOldFilters handles the conversion of old filters to new filters and creates a CEL program.
50-
// Returns both the program and any errors encountered during creation.
51-
func createProgramFromOldFilters(oldFilters []string, objectType workloadfilter.ResourceType) (cel.Program, error) {
52-
filterString, err := convertOldToNewFilter(oldFilters, objectType)
53-
if err != nil {
54-
return nil, err
55-
}
56-
57-
program, err := compileCELProgram(filterString, objectType)
58-
if err != nil {
59-
return nil, err
60-
}
61-
62-
return program, nil
63-
}
64-
65-
func compileCELProgram(rules string, objectType workloadfilter.ResourceType) (cel.Program, error) {
66-
if rules == "" {
67-
return nil, nil
68-
}
69-
env, err := cel.NewEnv(
70-
cel.Types(&workloadfilter.Container{}, &workloadfilter.Pod{}, &workloadfilter.Process{}),
71-
cel.Variable(string(objectType), cel.ObjectType(convertTypeToProtoType(objectType))),
72-
)
73-
if err != nil {
74-
return nil, err
75-
}
76-
abstractSyntaxTree, issues := env.Compile(rules)
77-
if issues != nil && issues.Err() != nil {
78-
return nil, issues.Err()
79-
}
80-
prg, err := env.Program(abstractSyntaxTree, cel.EvalOptions(cel.OptOptimize))
81-
if err != nil {
82-
return nil, err
83-
}
84-
return prg, nil
85-
}
86-
87-
// getFieldMapping creates a map to associate old filter prefixes with new filter fields
88-
func getFieldMapping(objectType workloadfilter.ResourceType) map[string]string {
89-
return map[string]string{
90-
"name": fmt.Sprintf("%s.name.matches", objectType),
91-
"image": fmt.Sprintf("%s.image.matches", objectType),
92-
"kube_namespace": func() string {
93-
if objectType == workloadfilter.ContainerType {
94-
return fmt.Sprintf("%s.%s.namespace.matches", objectType, workloadfilter.PodType)
95-
}
96-
return fmt.Sprintf("%s.namespace.matches", objectType)
97-
98-
}(),
99-
}
100-
}
101-
102-
// convertOldToNewFilter converts the legacy regex ad filter format to cel-go format.
103-
//
104-
// Old Format: []string{"image:nginx.*", "name:xyz-.*"},
105-
// New Format: "container.name.matches('xyz-.*') || container.image.matches('nginx.*')"
106-
func convertOldToNewFilter(oldFilters []string, objectType workloadfilter.ResourceType) (string, error) {
107-
if oldFilters == nil {
108-
return "", nil
109-
}
110-
111-
legacyFieldMapping := getFieldMapping(objectType)
112-
113-
var newFilters []string
114-
for _, oldFilter := range oldFilters {
115-
116-
if oldFilter == "" {
117-
continue
118-
}
119-
120-
// Split the filter into key and value using the first colon
121-
key, value, ok := strings.Cut(oldFilter, ":")
122-
if !ok {
123-
return "", fmt.Errorf("invalid filter format: %s", oldFilter)
124-
}
125-
126-
// Check if the key applies for the particular workload type
127-
if objectType != workloadfilter.ContainerType && key == "image" {
128-
continue
129-
}
130-
if objectType == workloadfilter.PodType && key != "kube_namespace" {
131-
continue
132-
}
133-
134-
// Legacy support for image filtering
135-
if key == "image" {
136-
value = legacyFilter.PreprocessImageFilter(value)
137-
}
138-
139-
if newField, ok := legacyFieldMapping[key]; ok {
140-
newFilters = append(newFilters, fmt.Sprintf(`%s(%s)`, newField, strconv.Quote(value)))
141-
} else {
142-
return "", fmt.Errorf("container filter %s:%s is unknown, ignoring it. The supported filters are 'image', 'name' and 'kube_namespace'", key, value)
143-
}
144-
}
145-
return strings.Join(newFilters, " || "), nil
146-
}
147-
148-
// convertTypeToProtoType converts a filter.ResourceType to its corresponding proto type string.
149-
func convertTypeToProtoType(key workloadfilter.ResourceType) string {
150-
switch key {
151-
case workloadfilter.ContainerType:
152-
return "datadog.filter.FilterContainer"
153-
case workloadfilter.PodType:
154-
return "datadog.filter.FilterPod"
155-
case workloadfilter.ServiceType:
156-
return "datadog.filter.FilterKubeService"
157-
case workloadfilter.EndpointType:
158-
return "datadog.filter.FilterKubeEndpoint"
159-
case workloadfilter.ProcessType:
160-
return "datadog.filter.FilterProcess"
161-
default:
162-
return ""
163-
}
164-
}
165-
16621
func createCELExcludeProgram(name string, rules string, objectType workloadfilter.ResourceType, logger log.Component) program.FilterProgram {
167-
excludeProgram, excludeErr := compileCELProgram(rules, objectType)
22+
excludeProgram, excludeErr := celprogram.CreateCELProgram(rules, objectType)
16823
if excludeErr != nil {
16924
logger.Criticalf(`failed to compile '%s' from 'cel_workload_exclude' filters: %v`, name, excludeErr)
17025
logger.Flush()

0 commit comments

Comments
 (0)