Skip to content

Commit a133a2e

Browse files
yevgenypatsdisq
andauthored
feat(gitlab): Update to SDK V4 (#11969)
Closes #11966 --------- Co-authored-by: Kemal <[email protected]>
1 parent c9eff12 commit a133a2e

32 files changed

+187
-145
lines changed

plugins/source/gitlab/client/client.go

Lines changed: 9 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,8 @@ package client
22

33
import (
44
"context"
5-
"fmt"
65

7-
"github.com/cloudquery/plugin-pb-go/specs"
8-
"github.com/cloudquery/plugin-sdk/v3/plugins/source"
9-
"github.com/cloudquery/plugin-sdk/v3/schema"
6+
"github.com/cloudquery/plugin-sdk/v4/schema"
107
"github.com/rs/zerolog"
118
"github.com/xanzy/go-gitlab"
129
)
@@ -16,7 +13,7 @@ type Client struct {
1613
// It will be passed for each resource fetcher.
1714
logger zerolog.Logger
1815
Gitlab *gitlab.Client
19-
spec specs.Source
16+
spec Spec
2017
BaseURL string
2118
MinAccessLevel *gitlab.AccessLevelValue
2219
}
@@ -25,30 +22,26 @@ func (c *Client) Logger() *zerolog.Logger {
2522
return &c.logger
2623
}
2724

28-
func (c *Client) ID() string {
29-
return c.spec.Name
25+
func (*Client) ID() string {
26+
return "gitlab"
3027
}
3128

32-
func Configure(ctx context.Context, logger zerolog.Logger, s specs.Source, _ source.Options) (schema.ClientMeta, error) {
33-
gitlabSpec := &Spec{}
34-
if err := s.UnmarshalSpec(gitlabSpec); err != nil {
35-
return nil, fmt.Errorf("failed to unmarshal gitlab spec: %w", err)
36-
}
37-
if err := gitlabSpec.Validate(); err != nil {
29+
func Configure(ctx context.Context, logger zerolog.Logger, s Spec) (schema.ClientMeta, error) {
30+
if err := s.Validate(); err != nil {
3831
return nil, err
3932
}
4033

4134
var minAccessLevel *gitlab.AccessLevelValue
4235
opts := []gitlab.ClientOptionFunc{}
43-
if gitlabSpec.BaseURL != "" {
44-
opts = append(opts, gitlab.WithBaseURL(gitlabSpec.BaseURL))
36+
if s.BaseURL != "" {
37+
opts = append(opts, gitlab.WithBaseURL(s.BaseURL))
4538
} else {
4639
// on GitLab SaaS we don't want to sync the whole of GitLab, so we sync based on access level
4740
// TODO: use gitlab.MinimalAccessPermissions once supported. Related https://gitlab.com/gitlab-org/gitlab/-/issues/296089#note_496386453
4841
minAccessLevel = gitlab.AccessLevel(gitlab.GuestPermissions)
4942
}
5043

51-
c, err := gitlab.NewClient(gitlabSpec.Token, opts...)
44+
c, err := gitlab.NewClient(s.Token, opts...)
5245
if err != nil {
5346
return nil, err
5447
}

plugins/source/gitlab/client/columns.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ package client
22

33
import (
44
"github.com/apache/arrow/go/v13/arrow"
5-
"github.com/cloudquery/plugin-sdk/v3/schema"
5+
"github.com/cloudquery/plugin-sdk/v4/schema"
66
)
77

88
var BaseURLColumn = schema.Column{

plugins/source/gitlab/client/resolvers.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ package client
33
import (
44
"context"
55

6-
"github.com/cloudquery/plugin-sdk/v3/schema"
6+
"github.com/cloudquery/plugin-sdk/v4/schema"
77
)
88

99
func ResolveURL(_ context.Context, meta schema.ClientMeta, r *schema.Resource, col schema.Column) error {

plugins/source/gitlab/client/spec.go

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,26 @@ package client
22

33
import "errors"
44

5+
const (
6+
defaultConcurrency = 10000
7+
)
8+
59
type Spec struct {
6-
Token string `json:"access_token,omitempty"`
7-
BaseURL string `json:"base_url,omitempty"`
10+
Token string `json:"access_token,omitempty"`
11+
BaseURL string `json:"base_url,omitempty"`
12+
Concurrency int `json:"concurrency,omitempty"`
813
}
914

10-
func (s Spec) Validate() error {
15+
func (s *Spec) Validate() error {
1116
gitlabToken := s.Token
1217
if gitlabToken == "" {
1318
return errors.New("missing GitLab API token in configuration file")
1419
}
1520
return nil
1621
}
22+
23+
func (s *Spec) SetDefaults() {
24+
if s.Concurrency == 0 {
25+
s.Concurrency = defaultConcurrency
26+
}
27+
}

plugins/source/gitlab/client/testing.go

Lines changed: 31 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,14 @@ package client
22

33
import (
44
"context"
5-
"fmt"
65
"net/http"
76
"net/http/httptest"
87
"os"
98
"testing"
109
"time"
1110

12-
"github.com/cloudquery/plugin-pb-go/specs"
13-
"github.com/cloudquery/plugin-sdk/v3/plugins/source"
14-
"github.com/cloudquery/plugin-sdk/v3/schema"
11+
"github.com/cloudquery/plugin-sdk/v4/scheduler"
12+
"github.com/cloudquery/plugin-sdk/v4/schema"
1513
"github.com/julienschmidt/httprouter"
1614
"github.com/rs/zerolog"
1715
"github.com/xanzy/go-gitlab"
@@ -20,65 +18,43 @@ import (
2018
type TestOptions struct{}
2119

2220
func GitlabMockTestHelper(t *testing.T, table *schema.Table, createService func(*httprouter.Router) error, options TestOptions) {
23-
version := "vDev"
2421
t.Helper()
25-
2622
table.IgnoreInTests = false
2723
mux := httprouter.New()
2824
ts := httptest.NewUnstartedServer(mux)
2925
defer ts.Close()
3026
l := zerolog.New(zerolog.NewTestWriter(t)).Output(
3127
zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: time.StampMicro},
3228
).Level(zerolog.DebugLevel).With().Timestamp().Logger()
33-
newTestExecutionClient := func(ctx context.Context, logger zerolog.Logger, spec specs.Source, _ source.Options) (schema.ClientMeta, error) {
34-
err := createService(mux)
35-
if err != nil {
36-
return nil, fmt.Errorf("failed to createService: %w", err)
37-
}
38-
ts.Start()
39-
client, err := gitlab.NewClient("",
40-
gitlab.WithBaseURL(ts.URL),
41-
// Disable backoff to speed up tests that expect errors.
42-
gitlab.WithCustomBackoff(func(_, _ time.Duration, _ int, _ *http.Response) time.Duration {
43-
return 0
44-
}),
45-
)
46-
if err != nil {
47-
t.Fatalf("Failed to create client: %v", err)
48-
}
49-
c := &Client{
50-
logger: l,
51-
Gitlab: client,
52-
BaseURL: ts.URL,
53-
}
54-
55-
return c, nil
29+
err := createService(mux)
30+
if err != nil {
31+
t.Fatal(err)
32+
}
33+
ts.Start()
34+
client, err := gitlab.NewClient("",
35+
gitlab.WithBaseURL(ts.URL),
36+
// Disable backoff to speed up tests that expect errors.
37+
gitlab.WithCustomBackoff(func(_, _ time.Duration, _ int, _ *http.Response) time.Duration {
38+
return 0
39+
}),
40+
)
41+
if err != nil {
42+
t.Fatalf("Failed to create client: %v", err)
43+
}
44+
c := &Client{
45+
logger: l,
46+
Gitlab: client,
47+
BaseURL: ts.URL,
5648
}
5749

58-
p := source.NewPlugin(
59-
table.Name,
60-
version,
61-
schema.Tables{table},
62-
newTestExecutionClient)
63-
p.SetLogger(l)
64-
source.TestPluginSync(t, p, specs.Source{
65-
Name: "dev",
66-
Path: "cloudquery/dev",
67-
Version: version,
68-
Tables: []string{table.Name},
69-
Destinations: []string{"mock-destination"},
70-
})
50+
sched := scheduler.NewScheduler(scheduler.WithLogger(l))
51+
messages, err := sched.SyncAll(context.Background(), c, schema.Tables{table})
52+
if err != nil {
53+
t.Fatalf("failed to sync: %v", err)
54+
}
55+
records := messages.GetInserts().GetRecordsForTable(table)
56+
emptyColumns := schema.FindEmptyColumns(table, records)
57+
if len(emptyColumns) > 0 {
58+
t.Fatalf("empty columns: %v", emptyColumns)
59+
}
7160
}
72-
73-
// func setup(t *testing.T) (*http.ServeMux, *gitlab.Client) {
74-
// // mux is the HTTP request multiplexer used with the test server.
75-
// mux := http.NewServeMux()
76-
77-
// // server is a test HTTP server used to provide mock API responses.
78-
// server := httptest.NewServer(mux)
79-
// t.Cleanup(server.Close)
80-
81-
// // client is the Gitlab client being tested.
82-
83-
// return mux, client
84-
// }

plugins/source/gitlab/client/transformers.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,9 @@ import (
55
"reflect"
66

77
"github.com/apache/arrow/go/v13/arrow"
8-
"github.com/cloudquery/plugin-sdk/v3/schema"
9-
"github.com/cloudquery/plugin-sdk/v3/transformers"
10-
"github.com/cloudquery/plugin-sdk/v3/types"
8+
"github.com/cloudquery/plugin-sdk/v4/schema"
9+
"github.com/cloudquery/plugin-sdk/v4/transformers"
10+
"github.com/cloudquery/plugin-sdk/v4/types"
1111
"github.com/xanzy/go-gitlab"
1212
)
1313

plugins/source/gitlab/go.mod

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,7 @@ go 1.19
44

55
require (
66
github.com/apache/arrow/go/v13 v13.0.0-20230630125530-5a06b2ec2a8e
7-
github.com/cloudquery/plugin-pb-go v1.6.0
8-
github.com/cloudquery/plugin-sdk/v3 v3.10.6
7+
github.com/cloudquery/plugin-sdk/v4 v4.8.1-rc1
98
github.com/julienschmidt/httprouter v1.3.0
109
github.com/rs/zerolog v1.29.1
1110
github.com/xanzy/go-gitlab v0.83.0
@@ -15,6 +14,7 @@ require (
1514
replace github.com/apache/arrow/go/v13 => github.com/cloudquery/arrow/go/v13 v13.0.0-20230703001435-df3b664a289d
1615

1716
require (
17+
github.com/cloudquery/plugin-pb-go v1.6.0 // indirect
1818
github.com/cloudquery/plugin-sdk/v2 v2.7.0 // indirect
1919
github.com/davecgh/go-spew v1.1.1 // indirect
2020
github.com/getsentry/sentry-go v0.20.0 // indirect
@@ -30,13 +30,13 @@ require (
3030
github.com/hashicorp/go-hclog v1.2.2 // indirect
3131
github.com/hashicorp/go-retryablehttp v0.7.2 // indirect
3232
github.com/inconshreveable/mousetrap v1.1.0 // indirect
33-
github.com/klauspost/compress v1.16.0 // indirect
33+
github.com/klauspost/compress v1.16.6 // indirect
3434
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
35+
github.com/kr/pretty v0.3.0 // indirect
3536
github.com/mattn/go-colorable v0.1.13 // indirect
3637
github.com/mattn/go-isatty v0.0.19 // indirect
3738
github.com/pierrec/lz4/v4 v4.1.17 // indirect
3839
github.com/pmezard/go-difflib v1.0.0 // indirect
39-
github.com/spf13/cast v1.5.0 // indirect
4040
github.com/spf13/cobra v1.6.1 // indirect
4141
github.com/spf13/pflag v1.0.5 // indirect
4242
github.com/stretchr/testify v1.8.4 // indirect

plugins/source/gitlab/go.sum

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -45,13 +45,14 @@ github.com/cloudquery/plugin-pb-go v1.6.0 h1:4g+tPhRAQUWpGyQBMcj6D3yVwKqwEN3nynv
4545
github.com/cloudquery/plugin-pb-go v1.6.0/go.mod h1:R0Wse6NbJDZIHcRQjJ1sZGYDo3mrIDm4k3El1YUrvGA=
4646
github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U=
4747
github.com/cloudquery/plugin-sdk/v2 v2.7.0/go.mod h1:pAX6ojIW99b/Vg4CkhnsGkRIzNaVEceYMR+Bdit73ug=
48-
github.com/cloudquery/plugin-sdk/v3 v3.10.6 h1:KqTsLZ6OA1h8BUMeMcU6BAD6TBW6ojgQaC4zDZMgvu0=
49-
github.com/cloudquery/plugin-sdk/v3 v3.10.6/go.mod h1:QhBaVgiNyQ3P6uAzJWOYpYykHXL+WDZffwg1riTwv60=
48+
github.com/cloudquery/plugin-sdk/v4 v4.8.1-rc1 h1:23MfOXbCXzoHuXbEGPTomneiPWAAucq8M+BaAaSi/6Q=
49+
github.com/cloudquery/plugin-sdk/v4 v4.8.1-rc1/go.mod h1:Y5HzxesZrpmSTUrbvR8EmGCK5wXDAhqSqzNEXC2ouvI=
5050
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
5151
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
5252
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
5353
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
5454
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
55+
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
5556
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
5657
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
5758
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -62,7 +63,6 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.m
6263
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
6364
github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
6465
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
65-
github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE=
6666
github.com/getsentry/sentry-go v0.20.0 h1:bwXW98iMRIWxn+4FgPW7vMrjmbym6HblXALmhjHmQaQ=
6767
github.com/getsentry/sentry-go v0.20.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
6868
github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
@@ -158,15 +158,17 @@ github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4d
158158
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
159159
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
160160
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
161-
github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4=
162-
github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
161+
github.com/klauspost/compress v1.16.6 h1:91SKEy4K37vkp255cJ8QesJhjyRO0hn9i9G0GoUwLsk=
162+
github.com/klauspost/compress v1.16.6/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
163163
github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg=
164164
github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
165165
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
166166
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
167+
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
167168
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
168169
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
169170
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
171+
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
170172
github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
171173
github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
172174
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
@@ -187,15 +189,14 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
187189
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
188190
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
189191
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
192+
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
190193
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
191194
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
192195
github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
193196
github.com/rs/zerolog v1.19.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJRjo=
194197
github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc=
195198
github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU=
196199
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
197-
github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w=
198-
github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU=
199200
github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA=
200201
github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY=
201202
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=

plugins/source/gitlab/main.go

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,19 @@
11
package main
22

33
import (
4+
"context"
5+
"log"
6+
47
"github.com/cloudquery/cloudquery/plugins/source/gitlab/resources/plugin"
5-
"github.com/cloudquery/plugin-sdk/v3/serve"
8+
"github.com/cloudquery/plugin-sdk/v4/serve"
69
)
710

811
const sentryDSN = "https://[email protected]/4504797528981504"
912

1013
func main() {
11-
serve.Source(plugin.Plugin(),
12-
serve.WithSourceSentryDSN(sentryDSN),
13-
)
14+
if err := serve.Plugin(plugin.Plugin(),
15+
serve.WithPluginSentryDSN(sentryDSN),
16+
).Serve(context.Background()); err != nil {
17+
log.Fatal(err)
18+
}
1419
}

0 commit comments

Comments
 (0)