Skip to content

Commit bf1bfb5

Browse files
hermanschaafhermanschaaf
authored
feat: Add snyk_reporting_issues table (#9762)
Closes #9755
1 parent 76e0d29 commit bf1bfb5

File tree

15 files changed

+356
-46
lines changed

15 files changed

+356
-46
lines changed

‎plugins/source/snyk/client/client.go‎

Lines changed: 60 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@ package client
33
import (
44
"context"
55
"fmt"
6+
"net/http"
7+
"time"
68

79
"github.com/cloudquery/plugin-sdk/plugins/source"
810
"github.com/cloudquery/plugin-sdk/schema"
@@ -11,13 +13,21 @@ import (
1113
"github.com/rs/zerolog"
1214
)
1315

16+
const (
17+
defaultMaxRetries = 5
18+
defaultBackoff = 60 * time.Second
19+
)
20+
1421
type Client struct {
1522
*snyk.Client
1623

1724
OrganizationID string
1825
organizations []string
1926

2027
logger zerolog.Logger
28+
29+
maxRetries int
30+
backoff time.Duration // backoff duration between retries (jitter will be added)
2131
}
2232

2333
var _ schema.ClientMeta = (*Client)(nil)
@@ -30,14 +40,61 @@ func (c *Client) Logger() *zerolog.Logger {
3040
return &c.logger
3141
}
3242

43+
type SnykLogger struct {
44+
logger zerolog.Logger
45+
}
46+
47+
func (l *SnykLogger) Log(args ...any) {
48+
if len(args) == 1 {
49+
l.logger.Debug().Interface("msg", args[0]).Msgf("Log from Snyk SDK")
50+
return
51+
}
52+
if len(args)%2 != 0 {
53+
l.logger.Debug().Interface("args", args).Msgf("Log from Snyk SDK")
54+
return
55+
}
56+
m := l.logger.Debug()
57+
for i := 0; i < len(args); i += 2 {
58+
k, ok := args[i].(string)
59+
if !ok {
60+
m = m.Interface(fmt.Sprintf("arg%02d", i), args[i])
61+
m = m.Interface(fmt.Sprintf("arg%02d", i+1), args[i+1])
62+
continue
63+
}
64+
if i+1 < len(args) {
65+
m = m.Interface(k, args[i+1])
66+
}
67+
}
68+
m.Msg("Log from Snyk SDK")
69+
}
70+
3371
func Configure(ctx context.Context, logger zerolog.Logger, spec specs.Source, _ source.Options) (schema.ClientMeta, error) {
3472
snykSpec := new(Spec)
3573
err := spec.UnmarshalSpec(snykSpec)
3674
if err != nil {
3775
return nil, fmt.Errorf("failed to unmarshal spec: %w", err)
3876
}
77+
err = snykSpec.Validate()
78+
if err != nil {
79+
return nil, fmt.Errorf("failed to validate spec: %w", err)
80+
}
81+
82+
snykLogger := SnykLogger{
83+
logger: logger,
84+
}
85+
httpClient := http.DefaultClient
86+
httpClient.Timeout = 1 * time.Minute
87+
options := []snyk.ClientOption{
88+
snyk.WithHTTPClient(httpClient),
89+
snyk.WithUserAgent("cloudquery/snyk/" + spec.Version),
90+
snyk.WithLogger(&snykLogger),
91+
snyk.WithLogRequests(true), // these will be filtered out by the logger if not in debug mode
92+
}
93+
if len(snykSpec.EndpointURL) > 0 {
94+
options = append(options, snyk.WithBaseURL(snykSpec.EndpointURL))
95+
}
3996

40-
client, err := snykSpec.getClient(spec.Version)
97+
client := snyk.NewClient(snykSpec.APIKey, options...)
4198
if err != nil {
4299
return nil, fmt.Errorf("failed to create Snyk client: %w", err)
43100
}
@@ -46,6 +103,8 @@ func Configure(ctx context.Context, logger zerolog.Logger, spec specs.Source, _
46103
Client: client,
47104
logger: logger,
48105
organizations: snykSpec.Organizations,
106+
maxRetries: defaultMaxRetries,
107+
backoff: defaultBackoff,
49108
}
50109

51110
return c, c.initOrganizations(ctx)

‎plugins/source/snyk/client/errors.go‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
package client
2+
3+
import (
4+
"context"
5+
"math/rand"
6+
"net/http"
7+
"time"
8+
9+
"github.com/pavel-snyk/snyk-sdk-go/snyk"
10+
)
11+
12+
// RetryOnError will run the given resolver function and retry on rate limit exceeded errors
13+
// or other retryable errors (like internal server errors) after waiting some amount of time.
14+
func (c *Client) RetryOnError(ctx context.Context, tableName string, f func() error) error {
15+
retries := 0
16+
var err error
17+
for err = f(); retries < c.maxRetries; err = f() {
18+
if shouldRetry(err) {
19+
retryAfter := time.Duration(rand.Float64() * float64(c.backoff))
20+
retries++
21+
c.logger.Info().Str("table", tableName).Msgf("Got retryable error (%v), retrying in %.2fs (%d/%d)", err, retryAfter.Seconds(), retries, c.maxRetries)
22+
select {
23+
case <-ctx.Done():
24+
return ctx.Err()
25+
case <-time.After(retryAfter):
26+
continue
27+
}
28+
}
29+
return err
30+
}
31+
return err
32+
}
33+
34+
func shouldRetry(err error) bool {
35+
if err == nil {
36+
return false
37+
}
38+
if resp, ok := err.(*snyk.ErrorResponse); ok {
39+
return resp.Response.StatusCode >= http.StatusInternalServerError || resp.Response.StatusCode == http.StatusTooManyRequests
40+
}
41+
return false
42+
}

‎plugins/source/snyk/client/errors_test.go‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
package client
2+
3+
import (
4+
"context"
5+
"errors"
6+
"net/http"
7+
"os"
8+
"testing"
9+
"time"
10+
11+
"github.com/pavel-snyk/snyk-sdk-go/snyk"
12+
"github.com/rs/zerolog"
13+
)
14+
15+
func TestRetryOnRateLimitError(t *testing.T) {
16+
logger := zerolog.New(zerolog.NewTestWriter(t)).Output(
17+
zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: time.StampMicro},
18+
).Level(zerolog.DebugLevel).With().Timestamp().Logger()
19+
c := Client{
20+
logger: logger,
21+
maxRetries: 1,
22+
backoff: 1 * time.Microsecond,
23+
}
24+
ctx := context.Background()
25+
t.Run("no_error", func(t *testing.T) {
26+
got := c.RetryOnError(ctx, "table_name", func() error {
27+
return nil
28+
})
29+
if got != nil {
30+
t.Errorf("RetryOnError returned error: %v, want nil", got)
31+
}
32+
})
33+
34+
t.Run("with_error", func(t *testing.T) {
35+
got := c.RetryOnError(ctx, "table_name", func() error {
36+
return errors.New("test error")
37+
})
38+
if got == nil || got.Error() != "test error" {
39+
t.Errorf("RetryOnError returned error: %v, want %v", got, "test error")
40+
}
41+
})
42+
43+
t.Run("retryable_error_that_never_succeeds", func(t *testing.T) {
44+
err := errors.New("server error: 500 Internal Server Error")
45+
got := c.RetryOnError(ctx, "table_name", func() error {
46+
return err
47+
})
48+
if got != err {
49+
t.Errorf("RetryOnError returned error: %v, want %v", got, err)
50+
}
51+
})
52+
53+
t.Run("temporary_error", func(t *testing.T) {
54+
i := 0
55+
got := c.RetryOnError(ctx, "table_name", func() error {
56+
if i == 0 {
57+
i++
58+
return &snyk.ErrorResponse{
59+
Response: &snyk.Response{
60+
Response: &http.Response{StatusCode: http.StatusInternalServerError},
61+
SnykRequestID: "",
62+
},
63+
ErrorElement: snyk.ErrorElement{},
64+
}
65+
}
66+
return nil
67+
})
68+
if got != nil {
69+
t.Errorf("RetryOnError returned error: %v, want %v", got, nil)
70+
}
71+
})
72+
}

‎plugins/source/snyk/client/multiplexer.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,5 +24,7 @@ func (c *Client) WithOrganization(organizationID string) schema.ClientMeta {
2424
OrganizationID: organizationID,
2525
organizations: c.organizations,
2626
logger: c.logger.With().Str("organization", organizationID).Logger(),
27+
maxRetries: c.maxRetries,
28+
backoff: c.backoff,
2729
}
2830
}

‎plugins/source/snyk/client/spec.go‎

Lines changed: 5 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,6 @@ package client
22

33
import (
44
"fmt"
5-
6-
"github.com/pavel-snyk/snyk-sdk-go/snyk"
75
)
86

97
type Spec struct {
@@ -14,19 +12,14 @@ type Spec struct {
1412
// By default, will fetch from all organizations available for user.
1513
Organizations []string `json:"organizations,omitempty"`
1614

17-
// EndpointURL is optional parameter to override the API URL for snyk.Client.
15+
// EndpointURL is an optional parameter to override the API URL for snyk.Client.
16+
// It defaults to https://api.snyk.io/api/
1817
EndpointURL string `json:"endpoint_url,omitempty"`
1918
}
2019

21-
func (s *Spec) getClient(version string) (*snyk.Client, error) {
20+
func (s *Spec) Validate() error {
2221
if len(s.APIKey) == 0 {
23-
return nil, fmt.Errorf("missing API key")
24-
}
25-
26-
options := []snyk.ClientOption{snyk.WithUserAgent("cloudquery/snyk/" + version)}
27-
if len(s.EndpointURL) > 0 {
28-
options = append(options, snyk.WithBaseURL(s.EndpointURL))
22+
return fmt.Errorf("missing API key")
2923
}
30-
31-
return snyk.NewClient(s.APIKey, options...), nil
24+
return nil
3225
}

‎plugins/source/snyk/client/testing.go‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
"github.com/cloudquery/plugin-sdk/schema"
1313
"github.com/cloudquery/plugin-sdk/specs"
1414
"github.com/julienschmidt/httprouter"
15+
"github.com/pavel-snyk/snyk-sdk-go/snyk"
1516
"github.com/rs/zerolog"
1617
)
1718

@@ -31,11 +32,7 @@ func MockTestHelper(t *testing.T, table *schema.Table, createService func(*httpr
3132
}
3233
ts.Start()
3334

34-
snykClient, err := (&Spec{
35-
APIKey: "test-key",
36-
Organizations: []string{"test-org"},
37-
EndpointURL: ts.URL + "/", // requires trailing slash
38-
}).getClient(version)
35+
snykClient := snyk.NewClient("test-key", snyk.WithBaseURL(ts.URL+"/"))
3936
if err != nil {
4037
return nil, fmt.Errorf("failed to create client: %w", err)
4138
}
@@ -45,7 +42,6 @@ func MockTestHelper(t *testing.T, table *schema.Table, createService func(*httpr
4542
logger: logger,
4643
organizations: []string{"test-org"},
4744
}
48-
4945
return c, nil
5046
}
5147

‎plugins/source/snyk/docs/tables/README.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,4 +5,5 @@
55
- [snyk_dependencies](../../../../../website/tables/snyk/snyk_dependencies.md)
66
- [snyk_integrations](../../../../../website/tables/snyk/snyk_integrations.md)
77
- [snyk_organizations](../../../../../website/tables/snyk/snyk_organizations.md)
8-
- [snyk_projects](../../../../../website/tables/snyk/snyk_projects.md)
8+
- [snyk_projects](../../../../../website/tables/snyk/snyk_projects.md)
9+
- [snyk_reporting_issues](../../../../../website/tables/snyk/snyk_reporting_issues.md)

‎plugins/source/snyk/go.mod‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ require (
1818
gopkg.in/yaml.v3 v3.0.1 // indirect
1919
)
2020

21-
replace github.com/pavel-snyk/snyk-sdk-go => github.com/cloudquery/snyk-sdk-go v0.3.0
21+
replace github.com/pavel-snyk/snyk-sdk-go => github.com/cloudquery/snyk-sdk-go v0.3.1
2222

2323
require (
2424
github.com/getsentry/sentry-go v0.20.0 // indirect
@@ -34,7 +34,7 @@ require (
3434
github.com/spf13/pflag v1.0.5 // indirect
3535
github.com/thoas/go-funk v0.9.3 // indirect
3636
golang.org/x/net v0.8.0 // indirect; indirect // indirect
37-
golang.org/x/sync v0.1.0 // indirect
37+
golang.org/x/sync v0.1.0
3838
golang.org/x/sys v0.6.0 // indirect
3939
golang.org/x/text v0.8.0 // indirect
4040
google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633 // indirect; indirect // indirect

‎plugins/source/snyk/go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,8 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn
4242
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
4343
github.com/cloudquery/plugin-sdk v1.45.0 h1:5vrfQZtaO1dp6ebKt8ouXDmPC7eeLuOB3JMd+FTRSYk=
4444
github.com/cloudquery/plugin-sdk v1.45.0/go.mod h1:9KGuuTGjTCKgh9amKwS+7Zrrqq7/M6lormteOyqoKwg=
45-
github.com/cloudquery/snyk-sdk-go v0.3.0 h1:Nu7bUk27zf/3/VqVwJOa9P9vhYVDpZHbSbbPrpCMa58=
46-
github.com/cloudquery/snyk-sdk-go v0.3.0/go.mod h1:LRL1TRuuM925gnyGp54WtS9p8S4yJMd0oS4JpLg+n7Y=
45+
github.com/cloudquery/snyk-sdk-go v0.3.1 h1:cJxsotIONWM6iIQX92Q7/7k6AYeo2H8W2wxQ8KoChQU=
46+
github.com/cloudquery/snyk-sdk-go v0.3.1/go.mod h1:LRL1TRuuM925gnyGp54WtS9p8S4yJMd0oS4JpLg+n7Y=
4747
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
4848
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
4949
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=

‎plugins/source/snyk/resources/plugin/plugin.go‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,13 @@ package plugin
22

33
import (
44
"github.com/cloudquery/cloudquery/plugins/source/snyk/client"
5+
"github.com/cloudquery/cloudquery/plugins/source/snyk/resources/services/dependency"
6+
"github.com/cloudquery/cloudquery/plugins/source/snyk/resources/services/integration"
7+
"github.com/cloudquery/cloudquery/plugins/source/snyk/resources/services/organization"
8+
"github.com/cloudquery/cloudquery/plugins/source/snyk/resources/services/project"
9+
"github.com/cloudquery/cloudquery/plugins/source/snyk/resources/services/reporting"
510
"github.com/cloudquery/plugin-sdk/plugins/source"
11+
"github.com/cloudquery/plugin-sdk/schema"
612
)
713

814
var Version = "Development"
@@ -11,7 +17,13 @@ func Snyk() *source.Plugin {
1117
return source.NewPlugin(
1218
"snyk",
1319
Version,
14-
tables(),
20+
[]*schema.Table{
21+
dependency.Dependencies(),
22+
integration.Integrations(),
23+
organization.Organizations(),
24+
project.Projects(),
25+
reporting.Issues(),
26+
},
1527
client.Configure,
1628
)
1729
}

0 commit comments

Comments
 (0)