Skip to content

Commit 74672ff

Browse files
lavigne958lavigne958
authored
CONTINT-4685 - Add e2e tests for host-tags presence and values (#44075)
### What does this PR do? Add a new generic function that reads host-tags from `fakeintake`, then validates the received tags against the expected tags for the tests for that env. Each env runs the test with its own expected tags and values. Some env runs with different host-type (windows and linux) and so the expected list of tags is different, we must use the optional tags for the the extra tags that are only here on some host types. Increase the fakeintake retention period as the host-tags are gathered every 30 minutes ### Motivation validate and test host-tags gathering ### Describe how you validated your changes ran the new end2end test and it passed: `dda inv new-e2e-tests.run -a ./tests/containers --run 'TestKindSuite/testHostTags'` ``` ... --- PASS: TestKindSuite (994.12s) testing: warning: no tests to run PASS ok github.com/DataDog/datadog-agent/test/new-e2e/tests/containers 996.734s [no tests to run] DONE 1 tests in 996.726s All tests passed ``` ### Additional Notes Co-authored-by: alexandre.lavigne <[email protected]>
1 parent 179ad83 commit 74672ff

File tree

10 files changed

+285
-51
lines changed

10 files changed

+285
-51
lines changed

‎test/fakeintake/aggregator/hostAggregator.go‎

Lines changed: 31 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -13,40 +13,38 @@ import (
1313
"github.com/DataDog/datadog-agent/test/fakeintake/api"
1414
)
1515

16-
// Host struct contains agents host-tags payload and attributes to fit Aggregator implementation
16+
// HostTags struct contains agents host-tags payload and attributes to fit Aggregator implementation
1717
//
1818
// Use a pointer to assign `HostTags` inner struct.
1919
// When we ParseHostPayload we receive variaous payload types.
2020
// We only want to keep those with host-tags
2121
// Using a pointer allows us to check if the pointer has been allocated
2222
// and therefore found the right payload
23-
type Host struct {
24-
HostTags *struct {
25-
System []string `json:"system"`
26-
} `json:"host-tags"`
23+
type HostTags struct {
24+
InternalHostname string
25+
HostTags []string
2726

28-
InternalHostname string `json:"internalHostname"`
29-
collectedTime time.Time
27+
collectedTime time.Time
3028
}
3129

3230
// GetCollectedTime return the time the payload was collected
33-
func (host *Host) GetCollectedTime() time.Time {
31+
func (host *HostTags) GetCollectedTime() time.Time {
3432
return host.collectedTime
3533
}
3634

3735
// GetTags returns the tags collected by the payload
3836
// currently none
39-
func (host *Host) GetTags() []string {
37+
func (host *HostTags) GetTags() []string {
4038
return nil
4139
}
4240

4341
// name return the payload name
44-
func (host *Host) name() string {
42+
func (host *HostTags) name() string {
4543
return host.InternalHostname
4644
}
4745

48-
// ParseHostPayload parses the generic payload and returns a typed struct with hostImpl data
49-
func ParseHostPayload(payload api.Payload) ([]*Host, error) {
46+
// ParseHostTagsPayload parses the generic payload and returns a typed struct with hostImpl data
47+
func ParseHostTagsPayload(payload api.Payload) ([]*HostTags, error) {
5048
if len(payload.Data) == 0 {
5149
return nil, nil
5250
}
@@ -56,7 +54,12 @@ func ParseHostPayload(payload api.Payload) ([]*Host, error) {
5654
return nil, fmt.Errorf("failed to inflate host Payload: %w", err)
5755
}
5856

59-
var data = &Host{}
57+
var data struct {
58+
HostName string `json:"internalHostname"`
59+
HostTags *struct {
60+
System []string `json:"system"`
61+
} `json:"host-tags"`
62+
}
6063

6164
if err := json.Unmarshal(inflated, &data); err != nil {
6265
return nil, fmt.Errorf("failed to unmarshall payload: %w", err)
@@ -66,23 +69,26 @@ func ParseHostPayload(payload api.Payload) ([]*Host, error) {
6669
// we only want to keep the matching payloads with host information
6770
// return an empty list with no error to skip this non-matching payload
6871
if data.HostTags == nil {
69-
return []*Host{}, nil
72+
return []*HostTags{}, nil
7073
}
7174

72-
// set hostname and collected time
73-
data.collectedTime = payload.Timestamp
74-
75-
return []*Host{data}, nil
75+
return []*HostTags{
76+
{
77+
collectedTime: payload.Timestamp,
78+
InternalHostname: data.HostName,
79+
HostTags: data.HostTags.System,
80+
},
81+
}, nil
7682
}
7783

78-
// HostAggregator structure
79-
type HostAggregator struct {
80-
Aggregator[*Host]
84+
// HostTagsAggregator structure
85+
type HostTagsAggregator struct {
86+
Aggregator[*HostTags]
8187
}
8288

83-
// NewHostAggregator returns a new Host aggregator
84-
func NewHostAggregator() HostAggregator {
85-
return HostAggregator{
86-
Aggregator: newAggregator(ParseHostPayload),
89+
// NewHostTagsAggregator returns a new Host aggregator
90+
func NewHostTagsAggregator() HostTagsAggregator {
91+
return HostTagsAggregator{
92+
Aggregator: newAggregator(ParseHostTagsPayload),
8793
}
8894
}

‎test/fakeintake/client/client.go‎

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -146,7 +146,7 @@ type Client struct {
146146
ndmflowAggregator aggregator.NDMFlowAggregator
147147
netpathAggregator aggregator.NetpathAggregator
148148
ncmAggregator aggregator.NCMAggregator
149-
hostAggregator aggregator.HostAggregator
149+
hostAggregator aggregator.HostTagsAggregator
150150
}
151151

152152
// NewClient creates a new fake intake client
@@ -178,7 +178,7 @@ func NewClient(fakeIntakeURL string, opts ...Option) *Client {
178178
ndmflowAggregator: aggregator.NewNDMFlowAggregator(),
179179
netpathAggregator: aggregator.NewNetpathAggregator(),
180180
ncmAggregator: aggregator.NewNCMAggregator(),
181-
hostAggregator: aggregator.NewHostAggregator(),
181+
hostAggregator: aggregator.NewHostTagsAggregator(),
182182
}
183183
for _, opt := range opts {
184184
opt(client)
@@ -345,7 +345,7 @@ func (c *Client) getNCMEvents() error {
345345
return c.ncmAggregator.UnmarshallPayloads(payloads)
346346
}
347347

348-
func (c *Client) getHostInfos() error {
348+
func (c *Client) getHostTags() error {
349349
payloads, err := c.getFakePayloads(intakeEndpoint)
350350
if err != nil {
351351
return err
@@ -1097,24 +1097,25 @@ func (c *Client) GetNCMPayloads() ([]*aggregator.NCMPayload, error) {
10971097
return ncmPayloads, nil
10981098
}
10991099

1100-
// GetLatestHostInfos returns the latest host information received by the fake intake
1101-
func (c *Client) GetLatestHostInfos() ([]*aggregator.Host, error) {
1102-
err := c.getHostInfos()
1103-
1100+
// GetHostTags returns the host tags received by the fake intake
1101+
func (c *Client) GetHostTags(hostname string) ([]*aggregator.HostTags, error) {
1102+
err := c.getHostTags()
11041103
if err != nil {
11051104
return nil, err
11061105
}
11071106

1108-
var hostInfos []*aggregator.Host
1109-
for _, name := range c.hostAggregator.GetNames() {
1110-
payloads := c.hostAggregator.GetPayloadsByName(name)
1107+
return c.hostAggregator.GetPayloadsByName(hostname), nil
1108+
}
11111109

1112-
if len(payloads) > 0 {
1113-
hostInfos = append(hostInfos, payloads...)
1114-
}
1110+
// GetHosts returns the list of all known hostnames that have sent some host-tags
1111+
func (c *Client) GetHosts() ([]string, error) {
1112+
err := c.getHostTags()
1113+
1114+
if err != nil {
1115+
return nil, err
11151116
}
11161117

1117-
return hostInfos, nil
1118+
return c.hostAggregator.GetNames(), nil
11181119
}
11191120

11201121
// filterPayload returns payloads matching any [MatchOpt](#MatchOpt) options

‎test/fakeintake/cmd/client/cmd/filter.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ func NewFilterCommand(cl **client.Client) (cmd *cobra.Command) {
2424
NewFilterLogsCommand(cl),
2525
NewFilterMetricsCommand(cl),
2626
NewFilterSBOMCommand(cl),
27+
NewFilterHostTagsCommand(cl),
2728
)
2829

2930
return cmd

‎test/fakeintake/cmd/client/cmd/filterhosttags.go‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
// Unless explicitly stated otherwise all files in this repository are licensed
2+
// under the Apache License Version 2.0.
3+
// This product includes software developed at Datadog (https://www.datadoghq.com/).
4+
// Copyright 2026-present Datadog, Inc.
5+
6+
package cmd
7+
8+
import (
9+
"encoding/json"
10+
"fmt"
11+
12+
"github.com/spf13/cobra"
13+
14+
"github.com/DataDog/datadog-agent/test/fakeintake/client"
15+
)
16+
17+
func NewFilterHostTagsCommand(cl **client.Client) (cmd *cobra.Command) {
18+
var hostFlagName = "host"
19+
var host string
20+
21+
cmd = &cobra.Command{
22+
Use: "host-tags",
23+
Short: "Filter host-tags",
24+
Example: "fakeintakectl --url http://internal-crayon-gcp-fakeintake.gcp.cloud filter host-tags --host my-gcp-host",
25+
RunE: func(cmd *cobra.Command, _ []string) error {
26+
hostTags, err := (*cl).GetHostTags(host)
27+
if err != nil {
28+
return fmt.Errorf("failed to get host-tags for host %s: %w", host, err)
29+
}
30+
31+
for _, hostTag := range hostTags {
32+
hostTagStr, err := json.MarshalIndent(hostTag, "", " ")
33+
if err != nil {
34+
cmd.PrintErrf("failed to format hostTag '%v' : %s", hostTag, err.Error())
35+
continue
36+
}
37+
38+
cmd.Println(string(hostTagStr))
39+
}
40+
return nil
41+
},
42+
}
43+
44+
cmd.Flags().StringVar(&host, hostFlagName, "", "hostname to get host-tags from")
45+
46+
if err := cmd.MarkFlagRequired(hostFlagName); err != nil {
47+
// only way this can fail is if the flag does not exist.
48+
panic(err)
49+
}
50+
51+
return cmd
52+
}

‎test/fakeintake/cmd/client/cmd/get.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ func NewGetCommand(cl **client.Client) (cmd *cobra.Command) {
3333
NewGetProcessesCommand(cl),
3434
NewGetSBOMCommand(cl),
3535
NewGetTracesCommand(cl),
36-
NewGetHostInfosCommand(cl),
36+
NewGetHosts(cl),
3737
)
3838

3939
return cmd

test/fakeintake/cmd/client/cmd/gethostinfos.go renamed to test/fakeintake/cmd/client/cmd/gethosts.go

Lines changed: 7 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -6,32 +6,28 @@
66
package cmd
77

88
import (
9-
"encoding/json"
109
"fmt"
1110

1211
"github.com/spf13/cobra"
1312

1413
"github.com/DataDog/datadog-agent/test/fakeintake/client"
1514
)
1615

17-
// NewGetHostInfosCommand adds a new command to get host infos
18-
func NewGetHostInfosCommand(cl **client.Client) (cmd *cobra.Command) {
16+
// NewGetHosts adds a new command to get host tags
17+
func NewGetHosts(cl **client.Client) (cmd *cobra.Command) {
1918
cmd = &cobra.Command{
20-
Use: "host-infos",
21-
Short: "Get Host infos",
19+
Use: "host",
20+
Short: "Get Hosts",
2221
RunE: func(*cobra.Command, []string) error {
23-
hostInfos, err := (*cl).GetLatestHostInfos()
22+
hosts, err := (*cl).GetHosts()
2423
if err != nil {
2524
return err
2625
}
2726

28-
output, err := json.MarshalIndent(hostInfos, "", " ")
29-
if err != nil {
30-
return err
27+
for _, host := range hosts {
28+
fmt.Println(host)
3129
}
3230

33-
fmt.Println(string(output))
34-
3531
return nil
3632
},
3733
}

‎test/new-e2e/tests/containers/base_test.go‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414

1515
"github.com/samber/lo"
1616
"github.com/stretchr/testify/assert"
17+
"github.com/stretchr/testify/require"
1718
"gopkg.in/yaml.v3"
1819
"gopkg.in/zorkian/go-datadog-api.v2"
1920

@@ -579,3 +580,100 @@ func (suite *baseSuite[Env]) testEvent(args *testEventArgs) {
579580
}, 2*time.Minute, 10*time.Second, "Failed finding `%s` with proper tags and message", prettyEventQuery)
580581
})
581582
}
583+
584+
type testHostTags struct {
585+
ExpectedTags []string
586+
OptionalTags []string
587+
}
588+
589+
func sendEvent[Env any](suite *baseSuite[Env], alertType, text string, args *testHostTags) {
590+
formattedArgs, err := yaml.Marshal(args)
591+
suite.Require().NoError(err)
592+
593+
_, err = suite.DatadogClient().PostEvent(&datadog.Event{
594+
Title: pointer.Ptr("test Host-Tags " + suite.T().Name()),
595+
AlertType: &alertType,
596+
Tags: append([]string{
597+
"app:agent-new-e2e-tests-containers",
598+
"cluster_name:" + suite.clusterName,
599+
"test:" + suite.T().Name(),
600+
}, args.ExpectedTags...),
601+
Text: pointer.Ptr(fmt.Sprintf(
602+
`%%%%%%
603+
### Result
604+
605+
`+"```"+`
606+
%s
607+
`+"```"+`
608+
609+
### Expected Tags
610+
611+
`+"```"+`
612+
%s
613+
`+"```"+`
614+
%%%%%%
615+
`, text, formattedArgs)),
616+
})
617+
618+
if err != nil {
619+
suite.T().Logf("Failed to post event: %s", err)
620+
}
621+
}
622+
623+
func (suite *baseSuite[Env]) testHostTags(args *testHostTags) {
624+
suite.EventuallyWithT(func(ct *assert.CollectT) {
625+
c := &myCollectT{
626+
CollectT: ct,
627+
errors: []error{},
628+
}
629+
630+
defer func() {
631+
if len(c.errors) == 0 {
632+
sendEvent(suite, "success", "All good!", args)
633+
} else {
634+
sendEvent(suite, "warning", errors.Join(c.errors...).Error(), args)
635+
}
636+
}()
637+
638+
hosts, err := suite.Fakeintake.GetHosts()
639+
assert.NoError(c, err, "failed to get hosts from host-tags payload")
640+
assert.GreaterOrEqual(c, len(hosts), 1, "empty host-tags payload, no hosts found")
641+
642+
regexTags := lo.Map(args.ExpectedTags, func(tag string, _ int) *regexp.Regexp {
643+
return regexp.MustCompile(tag)
644+
})
645+
646+
var optionalRegexTags []*regexp.Regexp
647+
if args.OptionalTags != nil {
648+
optionalRegexTags = lo.Map(args.OptionalTags, func(tag string, _ int) *regexp.Regexp {
649+
return regexp.MustCompile(tag)
650+
})
651+
}
652+
653+
for _, host := range hosts {
654+
suite.T().Logf("%s - validate host tags on host %s", time.Now().Format(time.TimeOnly), host)
655+
656+
hostInfos, err := suite.Fakeintake.GetHostTags(host)
657+
require.NoError(c, err, "failed to get host-tags for host %s", host)
658+
require.NotEmpty(c, hostInfos, "missing host tags in payload, should be len >= 1")
659+
660+
// we only want the latest known hostInfos
661+
hostInfo := hostInfos[len(hostInfos)-1]
662+
663+
// we don't know how to handle payloads with no host-name.
664+
// these are fargate host that runs side containers for the test.
665+
if hostInfo.InternalHostname == "" {
666+
continue
667+
}
668+
669+
hostTags := hostInfo.HostTags
670+
671+
suite.T().Logf("Found host tags:\n- %s\n", strings.ReplaceAll(strings.Join(hostTags, "\n"), "\n", "\n- "))
672+
673+
require.NotNil(c, hostTags, "wrong payload, could not find 'host-tags' object in payload")
674+
err = assertTags(hostTags, regexTags, optionalRegexTags, false)
675+
assert.NoError(c, err)
676+
}
677+
678+
}, 33*time.Minute, 1*time.Minute, "Failed to validate all host-tags")
679+
}

0 commit comments

Comments
 (0)