Skip to content

Commit 600879c

Browse files
hermanschaafhermanschaaf
committed
Use lists in config
1 parent 9a231fe commit 600879c

File tree

10 files changed

+139
-95
lines changed

10 files changed

+139
-95
lines changed

‎plugins/source/aws/client/tableoptions/accessanalyzer.go‎

Lines changed: 8 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,10 @@ import (
77
"github.com/aws/aws-sdk-go-v2/aws"
88
"github.com/aws/aws-sdk-go-v2/service/accessanalyzer"
99
"github.com/cloudquery/plugin-sdk/v3/caser"
10-
"github.com/jinzhu/copier"
1110
)
1211

1312
type AccessanalyzerFindings struct {
14-
ListFindingOpts CustomAccessAnalyzerListFindingsInput `json:"list_findings,omitempty"`
13+
ListFindingOpts []CustomAccessAnalyzerListFindingsInput `json:"list_findings,omitempty"`
1514
}
1615

1716
type CustomAccessAnalyzerListFindingsInput struct {
@@ -34,27 +33,17 @@ func (c *CustomAccessAnalyzerListFindingsInput) UnmarshalJSON(data []byte) error
3433
}
3534

3635
func (c *AccessanalyzerFindings) validateListFindings() error {
37-
if aws.ToString(c.ListFindingOpts.NextToken) != "" {
38-
return errors.New("invalid input: cannot set NextToken in ListFindings")
39-
}
40-
if aws.ToString(c.ListFindingOpts.AnalyzerArn) != "" {
41-
return errors.New("invalid input: cannot set AnalyzerARN in ListFindings")
36+
for _, opt := range c.ListFindingOpts {
37+
if aws.ToString(opt.NextToken) != "" {
38+
return errors.New("invalid input: cannot set NextToken in ListFindings")
39+
}
40+
if aws.ToString(opt.AnalyzerArn) != "" {
41+
return errors.New("invalid input: cannot set AnalyzerARN in ListFindings")
42+
}
4243
}
4344
return nil
4445
}
4546

4647
func (c *AccessanalyzerFindings) Validate() error {
4748
return c.validateListFindings()
4849
}
49-
50-
func (c *AccessanalyzerFindings) ListFindings() (*accessanalyzer.ListFindingsInput, error) {
51-
var aaLFI accessanalyzer.ListFindingsInput
52-
if c == nil {
53-
return &aaLFI, nil
54-
}
55-
if err := c.validateListFindings(); err != nil {
56-
return &aaLFI, err
57-
}
58-
59-
return &aaLFI, copier.Copy(&aaLFI, &c.ListFindingOpts)
60-
}

‎plugins/source/aws/client/tableoptions/accessanalyzer_test.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,16 +15,16 @@ func TestAAListFindings(t *testing.T) {
1515
}
1616

1717
api := AccessanalyzerFindings{
18-
ListFindingOpts: u,
18+
ListFindingOpts: []CustomAccessAnalyzerListFindingsInput{u},
1919
}
2020
// Ensure that the validation works as expected
2121
err := api.Validate()
2222
assert.EqualError(t, err, "invalid input: cannot set NextToken in ListFindings")
23-
api.ListFindingOpts.NextToken = nil
23+
api.ListFindingOpts[0].NextToken = nil
2424

2525
err = api.Validate()
2626
assert.EqualError(t, err, "invalid input: cannot set AnalyzerARN in ListFindings")
27-
api.ListFindingOpts.AnalyzerArn = nil
27+
api.ListFindingOpts[0].AnalyzerArn = nil
2828

2929
// Ensure that as soon as the validation passes that there are no unexpected empty or nil fields
3030
err = api.Validate()

‎plugins/source/aws/client/tableoptions/cloudtrail.go‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import (
1010
)
1111

1212
type CloudtrailAPIs struct {
13-
LookupEventsOpts CustomLookupEventsOpts `json:"lookup_events,omitempty"`
13+
LookupEventsOpts []CustomLookupEventsOpts `json:"lookup_events,omitempty"`
1414
}
1515

1616
type CustomLookupEventsOpts struct {
@@ -32,8 +32,10 @@ func (c *CustomLookupEventsOpts) UnmarshalJSON(data []byte) error {
3232
}
3333

3434
func (c *CloudtrailAPIs) validateLookupEvents() error {
35-
if aws.ToString(c.LookupEventsOpts.NextToken) != "" {
36-
return errors.New("invalid input: cannot set NextToken in LookupEvents")
35+
for _, opt := range c.LookupEventsOpts {
36+
if aws.ToString(opt.NextToken) != "" {
37+
return errors.New("invalid input: cannot set NextToken in LookupEvents")
38+
}
3739
}
3840
return nil
3941
}

‎plugins/source/aws/client/tableoptions/cloudtrail_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,14 @@ func TestLookupEvents(t *testing.T) {
1313
t.Fatal(err)
1414
}
1515
api := CloudtrailAPIs{
16-
LookupEventsOpts: u,
16+
LookupEventsOpts: []CustomLookupEventsOpts{u},
1717
}
1818
// Ensure that the validation works as expected
1919
err := api.Validate()
2020
assert.EqualError(t, err, "invalid input: cannot set NextToken in LookupEvents")
2121

2222
// Ensure that as soon as the validation passes that there are no unexpected empty or nil fields
23-
api.LookupEventsOpts.NextToken = nil
23+
api.LookupEventsOpts[0].NextToken = nil
2424
err = api.Validate()
2525
assert.Nil(t, err)
2626
}

‎plugins/source/aws/client/tableoptions/inspector2.go‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import (
1010
)
1111

1212
type Inspector2APIs struct {
13-
ListFindingsOpts CustomInspector2ListFindingsInput `json:"list_findings,omitempty"`
13+
ListFindingsOpts []CustomInspector2ListFindingsInput `json:"list_findings,omitempty"`
1414
}
1515

1616
type CustomInspector2ListFindingsInput struct {
@@ -32,8 +32,10 @@ func (c *CustomInspector2ListFindingsInput) UnmarshalJSON(data []byte) error {
3232
}
3333

3434
func (c *Inspector2APIs) validateListFindings() error {
35-
if aws.ToString(c.ListFindingsOpts.NextToken) != "" {
36-
return errors.New("invalid input: cannot set NextToken in ListFindings")
35+
for _, opt := range c.ListFindingsOpts {
36+
if aws.ToString(opt.NextToken) != "" {
37+
return errors.New("invalid input: cannot set NextToken in ListFindings")
38+
}
3739
}
3840
return nil
3941
}

‎plugins/source/aws/client/tableoptions/inspector2_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,14 +15,14 @@ func TestInspector2ListFindings(t *testing.T) {
1515
}
1616

1717
api := Inspector2APIs{
18-
ListFindingsOpts: u,
18+
ListFindingsOpts: []CustomInspector2ListFindingsInput{u},
1919
}
2020
// Ensure that the validation works as expected
2121
err := api.Validate()
2222
assert.EqualError(t, err, "invalid input: cannot set NextToken in ListFindings")
2323

2424
// Ensure that as soon as the validation passes that there are no unexpected empty or nil fields
25-
api.ListFindingsOpts.NextToken = nil
25+
api.ListFindingsOpts[0].NextToken = nil
2626
err = api.Validate()
2727

2828
assert.Nil(t, err)

‎plugins/source/aws/client/tableoptions/table_options_test.go‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"encoding/json"
55
"testing"
66

7+
"github.com/aws/aws-sdk-go-v2/aws"
78
"github.com/aws/aws-sdk-go-v2/service/accessanalyzer"
89
accessanalyzertypes "github.com/aws/aws-sdk-go-v2/service/accessanalyzer/types"
910
"github.com/aws/aws-sdk-go-v2/service/cloudtrail"
@@ -17,6 +18,33 @@ import (
1718
"github.com/google/go-cmp/cmp/cmpopts"
1819
)
1920

21+
func TestTableOptionsValidate(t *testing.T) {
22+
tOpts := TableOptions{}
23+
err := tOpts.Validate()
24+
if err != nil {
25+
t.Fatalf("unexpected error: %v", err)
26+
}
27+
28+
tOpts.CloudTrailEvents = &CloudtrailAPIs{
29+
LookupEventsOpts: []CustomLookupEventsOpts{
30+
{
31+
LookupEventsInput: cloudtrail.LookupEventsInput{
32+
EndTime: nil,
33+
EventCategory: "",
34+
LookupAttributes: nil,
35+
MaxResults: nil,
36+
NextToken: aws.String("123"),
37+
StartTime: nil,
38+
},
39+
},
40+
},
41+
}
42+
err = tOpts.Validate()
43+
if err == nil {
44+
t.Fatal("expected error validating cloud_trail_events, got nil")
45+
}
46+
}
47+
2048
// TestTableOptionsUnmarshal tests that the TableOptions struct can be unmarshaled from JSON using
2149
// snake_case keys.
2250
func TestTableOptionsUnmarshal(t *testing.T) {

‎plugins/source/aws/resources/services/accessanalyzer/analyzer_findings.go‎

Lines changed: 14 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"github.com/aws/aws-sdk-go-v2/service/accessanalyzer"
99
"github.com/aws/aws-sdk-go-v2/service/accessanalyzer/types"
1010
"github.com/cloudquery/cloudquery/plugins/source/aws/client"
11+
"github.com/cloudquery/cloudquery/plugins/source/aws/client/tableoptions"
1112
"github.com/cloudquery/plugin-sdk/v2/schema"
1213
"github.com/cloudquery/plugin-sdk/v2/transformers"
1314
)
@@ -42,24 +43,22 @@ func fetchAccessanalyzerAnalyzerFindings(ctx context.Context, meta schema.Client
4243
analyzer := parent.Item.(types.AnalyzerSummary)
4344
cl := meta.(*client.Client)
4445
svc := cl.Services().Accessanalyzer
45-
var err error
46-
lfi := &accessanalyzer.ListFindingsInput{}
46+
allConfigs := []tableoptions.CustomAccessAnalyzerListFindingsInput{{}}
4747
if cl.Spec.TableOptions.AccessAnalyzerFindings != nil {
48-
lfi, err = cl.Spec.TableOptions.AccessAnalyzerFindings.ListFindings()
49-
if err != nil {
50-
return err
51-
}
48+
allConfigs = cl.Spec.TableOptions.AccessAnalyzerFindings.ListFindingOpts
5249
}
53-
lfi.AnalyzerArn = analyzer.Arn
54-
paginator := accessanalyzer.NewListFindingsPaginator(svc, lfi)
55-
for paginator.HasMorePages() {
56-
page, err := paginator.NextPage(ctx, func(options *accessanalyzer.Options) {
57-
options.Region = cl.Region
58-
})
59-
if err != nil {
60-
return err
50+
for _, cfg := range allConfigs {
51+
cfg.AnalyzerArn = analyzer.Arn
52+
paginator := accessanalyzer.NewListFindingsPaginator(svc, &cfg.ListFindingsInput)
53+
for paginator.HasMorePages() {
54+
page, err := paginator.NextPage(ctx, func(options *accessanalyzer.Options) {
55+
options.Region = cl.Region
56+
})
57+
if err != nil {
58+
return err
59+
}
60+
res <- page.Findings
6161
}
62-
res <- page.Findings
6362
}
6463
return nil
6564
}

‎plugins/source/aws/resources/services/cloudtrail/events.go‎

Lines changed: 56 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,10 @@ import (
88
"github.com/aws/aws-sdk-go-v2/service/cloudtrail"
99
"github.com/aws/aws-sdk-go-v2/service/cloudtrail/types"
1010
"github.com/cloudquery/cloudquery/plugins/source/aws/client"
11+
"github.com/cloudquery/cloudquery/plugins/source/aws/client/tableoptions"
1112
"github.com/cloudquery/plugin-sdk/v2/schema"
1213
"github.com/cloudquery/plugin-sdk/v2/transformers"
14+
"github.com/mitchellh/hashstructure/v2"
1315
)
1416

1517
const tableName = "aws_cloudtrail_events"
@@ -45,51 +47,71 @@ func Events() *schema.Table {
4547
func fetchCloudtrailEvents(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error {
4648
cl := meta.(*client.Client)
4749
svc := cl.Services().Cloudtrail
48-
le := &cloudtrail.LookupEventsInput{}
50+
51+
allConfigs := []tableoptions.CustomLookupEventsOpts{{}}
52+
noTableConfig := true
4953
if cl.Spec.TableOptions.CloudTrailEvents != nil {
50-
le = &cl.Spec.TableOptions.CloudTrailEvents.LookupEventsOpts.LookupEventsInput
54+
allConfigs = cl.Spec.TableOptions.CloudTrailEvents.LookupEventsOpts
55+
noTableConfig = false
5156
}
57+
for _, w := range allConfigs {
58+
le := w.LookupEventsInput
5259

53-
if cl.Backend != nil {
54-
value, err := cl.Backend.Get(ctx, tableName, cl.ID())
55-
if err != nil {
56-
return fmt.Errorf("failed to retrieve state from backend: %w", err)
57-
}
58-
59-
if value != "" {
60-
date, err := time.Parse(time.RFC3339Nano, value)
60+
var backendKey string
61+
if cl.Backend != nil {
62+
// Retrieve the last event time from the backend for this table option config.
63+
// We use a hash of the config as the key, so changing the config will cause a full refresh.
64+
hash, err := hashstructure.Hash(le, hashstructure.FormatV2, nil)
6165
if err != nil {
62-
return fmt.Errorf("retrieved invalid state value: %q %w", value, err)
66+
return err
67+
}
68+
backendKey = fmt.Sprintf("%s-%d", cl.ID(), hash)
69+
if noTableConfig {
70+
// for backwards-compatibility, default to client id if there is no table config
71+
backendKey = cl.ID()
72+
}
73+
value, err := cl.Backend.Get(ctx, tableName, backendKey)
74+
if err != nil {
75+
return fmt.Errorf("failed to retrieve state from backend: %w", err)
76+
}
77+
78+
if value != "" {
79+
date, err := time.Parse(time.RFC3339Nano, value)
80+
if err != nil {
81+
return fmt.Errorf("retrieved invalid state value: %q %w", value, err)
82+
}
83+
le.StartTime = &date
6384
}
64-
le.StartTime = &date
65-
}
66-
}
67-
var lastEventTime *time.Time
68-
// var err error
69-
paginator := cloudtrail.NewLookupEventsPaginator(svc, le)
70-
for paginator.HasMorePages() {
71-
page, err := paginator.NextPage(ctx, func(options *cloudtrail.Options) {
72-
options.Region = cl.Region
73-
})
74-
if err != nil {
75-
return err
7685
}
77-
res <- page.Events
7886

79-
// Retrieve the timestamp from the latest event
80-
for _, event := range page.Events {
81-
if lastEventTime == nil {
82-
lastEventTime = event.EventTime
83-
continue
87+
var lastEventTime *time.Time
88+
// var err error
89+
paginator := cloudtrail.NewLookupEventsPaginator(svc, &le)
90+
for paginator.HasMorePages() {
91+
page, err := paginator.NextPage(ctx, func(options *cloudtrail.Options) {
92+
options.Region = cl.Region
93+
})
94+
if err != nil {
95+
return err
8496
}
85-
if event.EventTime.After(*lastEventTime) {
86-
lastEventTime = event.EventTime
97+
res <- page.Events
98+
99+
// Retrieve the timestamp from the latest event
100+
for _, event := range page.Events {
101+
if lastEventTime == nil {
102+
lastEventTime = event.EventTime
103+
continue
104+
}
105+
if event.EventTime.After(*lastEventTime) {
106+
lastEventTime = event.EventTime
107+
}
87108
}
88109
}
89-
}
90110

91-
if cl.Backend != nil && lastEventTime != nil {
92-
return cl.Backend.Set(ctx, tableName, cl.ID(), lastEventTime.Format(time.RFC3339Nano))
111+
if cl.Backend != nil && lastEventTime != nil {
112+
return cl.Backend.Set(ctx, tableName, backendKey, lastEventTime.Format(time.RFC3339Nano))
113+
}
93114
}
115+
94116
return nil
95117
}

‎plugins/source/aws/resources/services/inspector2/findings.go‎

Lines changed: 16 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"github.com/aws/aws-sdk-go-v2/service/inspector2"
88
"github.com/aws/aws-sdk-go-v2/service/inspector2/types"
99
"github.com/cloudquery/cloudquery/plugins/source/aws/client"
10+
"github.com/cloudquery/cloudquery/plugins/source/aws/client/tableoptions"
1011
"github.com/cloudquery/plugin-sdk/v2/schema"
1112
"github.com/cloudquery/plugin-sdk/v2/transformers"
1213
)
@@ -49,24 +50,25 @@ func fetchInspector2Findings(ctx context.Context, meta schema.ClientMeta, parent
4950
c := meta.(*client.Client)
5051
svc := c.Services().Inspector2
5152

52-
input := &inspector2.ListFindingsInput{}
53+
allConfigs := []tableoptions.CustomInspector2ListFindingsInput{{}}
5354
if c.Spec.TableOptions.Inspector2Findings != nil {
54-
input = &c.Spec.TableOptions.Inspector2Findings.ListFindingsOpts.ListFindingsInput
55-
}
56-
57-
if input.MaxResults == nil {
58-
input.MaxResults = aws.Int32(100)
55+
allConfigs = c.Spec.TableOptions.Inspector2Findings.ListFindingsOpts
5956
}
57+
for _, input := range allConfigs {
58+
if input.MaxResults == nil {
59+
input.MaxResults = aws.Int32(100)
60+
}
6061

61-
paginator := inspector2.NewListFindingsPaginator(svc, input)
62-
for paginator.HasMorePages() {
63-
page, err := paginator.NextPage(ctx, func(options *inspector2.Options) {
64-
options.Region = c.Region
65-
})
66-
if err != nil {
67-
return err
62+
paginator := inspector2.NewListFindingsPaginator(svc, &input.ListFindingsInput)
63+
for paginator.HasMorePages() {
64+
page, err := paginator.NextPage(ctx, func(options *inspector2.Options) {
65+
options.Region = c.Region
66+
})
67+
if err != nil {
68+
return err
69+
}
70+
res <- page.Findings
6871
}
69-
res <- page.Findings
7072
}
7173
return nil
7274
}

0 commit comments

Comments
 (0)