Skip to content

Commit bc8ef17

Browse files
authored
feat(cloudflare): Update to SDK V4 (#11953)
Closes #11952
1 parent 10a4891 commit bc8ef17

File tree

63 files changed

+198
-164
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

63 files changed

+198
-164
lines changed

plugins/source/cloudflare/client/client.go

Lines changed: 5 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,11 @@ package client
33
import (
44
"context"
55
"errors"
6-
"fmt"
76
"os"
87
"strings"
98

109
"github.com/cloudflare/cloudflare-go"
11-
"github.com/cloudquery/plugin-pb-go/specs"
12-
"github.com/cloudquery/plugin-sdk/v3/plugins/source"
13-
"github.com/cloudquery/plugin-sdk/v3/schema"
10+
"github.com/cloudquery/plugin-sdk/v4/schema"
1411
"github.com/rs/zerolog"
1512
"github.com/thoas/go-funk"
1613
)
@@ -78,13 +75,8 @@ func (c *Client) withZoneID(accountId, zoneId string) *Client {
7875
}
7976
}
8077

81-
func Configure(ctx context.Context, logger zerolog.Logger, s specs.Source, _ source.Options) (schema.ClientMeta, error) {
82-
cfSpec := &Spec{}
83-
if err := s.UnmarshalSpec(cfSpec); err != nil {
84-
return nil, fmt.Errorf("failed to unmarshal cloudflare spec: %w", err)
85-
}
86-
87-
clientApi, err := getCloudflareClient(cfSpec)
78+
func Configure(ctx context.Context, logger zerolog.Logger, spec *Spec) (schema.ClientMeta, error) {
79+
clientApi, err := getCloudflareClient(spec)
8880
if err != nil {
8981
return nil, err
9082
}
@@ -98,7 +90,7 @@ func Configure(ctx context.Context, logger zerolog.Logger, s specs.Source, _ sou
9890
}
9991

10092
for _, account := range accounts {
101-
if len(cfSpec.Accounts) > 0 && !funk.ContainsString(cfSpec.Accounts, account.ID) {
93+
if len(spec.Accounts) > 0 && !funk.ContainsString(spec.Accounts, account.ID) {
10294
continue
10395
}
10496

@@ -128,7 +120,7 @@ func Configure(ctx context.Context, logger zerolog.Logger, s specs.Source, _ sou
128120

129121
clients := make(Clients)
130122
for _, account := range accountsZones {
131-
c, err := getCloudflareClient(cfSpec)
123+
c, err := getCloudflareClient(spec)
132124
if err != nil {
133125
return nil, err
134126
}

plugins/source/cloudflare/client/filter.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
package client
22

3-
import "github.com/cloudquery/plugin-sdk/v3/schema"
3+
import "github.com/cloudquery/plugin-sdk/v4/schema"
44

55
func DeleteAccountFilter(meta schema.ClientMeta, _ *schema.Resource) []any {
66
client := meta.(*Client)

plugins/source/cloudflare/client/multiplexers.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
package client
22

3-
import "github.com/cloudquery/plugin-sdk/v3/schema"
3+
import "github.com/cloudquery/plugin-sdk/v4/schema"
44

55
func AccountMultiplex(meta schema.ClientMeta) []schema.ClientMeta {
66
var l = make([]schema.ClientMeta, 0)

plugins/source/cloudflare/client/resolvers.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ package client
33
import (
44
"context"
55

6-
"github.com/cloudquery/plugin-sdk/v3/schema"
6+
"github.com/cloudquery/plugin-sdk/v4/schema"
77
)
88

99
func ResolveAccountID(_ context.Context, meta schema.ClientMeta, r *schema.Resource, col schema.Column) error {
Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,20 @@
11
package client
22

3+
const (
4+
defaultConccurency = 10000
5+
)
6+
37
type Spec struct {
4-
Token string `json:"api_token,omitempty"`
5-
ApiKey string `json:"api_key,omitempty"`
6-
ApiEmail string `json:"api_email,omitempty"`
7-
Accounts []string `json:"accounts,omitempty"`
8-
Zones []string `json:"zones,omitempty"`
8+
Token string `json:"api_token,omitempty"`
9+
ApiKey string `json:"api_key,omitempty"`
10+
ApiEmail string `json:"api_email,omitempty"`
11+
Accounts []string `json:"accounts,omitempty"`
12+
Zones []string `json:"zones,omitempty"`
13+
Concurrency int `json:"concurrency,omitempty"`
14+
}
15+
16+
func (s *Spec) SetDefaults() {
17+
if s.Concurrency == 0 {
18+
s.Concurrency = defaultConccurency
19+
}
920
}

plugins/source/cloudflare/client/testing.go

Lines changed: 30 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,13 @@ package client
22

33
import (
44
"context"
5-
"fmt"
65
"os"
76
"testing"
87
"time"
98

10-
"github.com/cloudquery/plugin-pb-go/specs"
11-
"github.com/cloudquery/plugin-sdk/v3/plugins/source"
12-
"github.com/cloudquery/plugin-sdk/v3/schema"
9+
"github.com/cloudquery/plugin-sdk/v4/message"
10+
"github.com/cloudquery/plugin-sdk/v4/scheduler"
11+
"github.com/cloudquery/plugin-sdk/v4/schema"
1312
"github.com/golang/mock/gomock"
1413
"github.com/rs/zerolog"
1514
)
@@ -19,49 +18,41 @@ const (
1918
TestZoneID = "test_zone"
2019
)
2120

22-
func MockTestHelper(t *testing.T, table *schema.Table, builder func(*testing.T, *gomock.Controller) Clients) {
23-
version := "vDev"
21+
func filterInserts(msgs message.SyncMessages) message.SyncInserts {
22+
inserts := []*message.SyncInsert{}
23+
for _, msg := range msgs {
24+
if m, ok := msg.(*message.SyncInsert); ok {
25+
inserts = append(inserts, m)
26+
}
27+
}
28+
return inserts
29+
}
2430

31+
func MockTestHelper(t *testing.T, table *schema.Table, builder func(*testing.T, *gomock.Controller) Clients) {
2532
t.Helper()
2633
table.IgnoreInTests = false
2734

2835
ctrl := gomock.NewController(t)
2936
defer ctrl.Finish()
30-
logger := zerolog.New(zerolog.NewTestWriter(t)).Output(
37+
l := zerolog.New(zerolog.NewTestWriter(t)).Output(
3138
zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: time.StampMicro},
3239
).Level(zerolog.DebugLevel).With().Timestamp().Logger()
33-
34-
newTestExecutionClient := func(ctx context.Context, _ zerolog.Logger, spec specs.Source, _ source.Options) (schema.ClientMeta, error) {
35-
var cfSpec Spec
36-
if err := spec.UnmarshalSpec(&cfSpec); err != nil {
37-
return nil, fmt.Errorf("failed to unmarshal cloudflare spec: %w", err)
38-
}
39-
40-
clients := builder(t, ctrl)
41-
c := New(logger, clients, clients[TestAccountID], AccountZones{
42-
TestAccountID: {
43-
AccountId: TestAccountID,
44-
Zones: []string{TestZoneID},
45-
},
46-
})
47-
48-
return c.withZoneID(TestAccountID, TestZoneID), nil
49-
}
50-
51-
p := source.NewPlugin(
52-
table.Name,
53-
version,
54-
[]*schema.Table{
55-
table,
40+
sched := scheduler.NewScheduler(scheduler.WithLogger(l))
41+
clients := builder(t, ctrl)
42+
c := New(l, clients, clients[TestAccountID], AccountZones{
43+
TestAccountID: {
44+
AccountId: TestAccountID,
45+
Zones: []string{TestZoneID},
5646
},
57-
newTestExecutionClient,
58-
)
59-
p.SetLogger(logger)
60-
source.TestPluginSync(t, p, specs.Source{
61-
Name: "dev",
62-
Path: "cloudquery/dev",
63-
Version: version,
64-
Tables: []string{table.Name},
65-
Destinations: []string{"mock-destination"},
6647
})
48+
49+
messages, err := sched.SyncAll(context.Background(), &c, schema.Tables{table})
50+
if err != nil {
51+
t.Fatalf("failed to sync: %v", err)
52+
}
53+
records := filterInserts(messages).GetRecordsForTable(table)
54+
emptyColumns := schema.FindEmptyColumns(table, records)
55+
if len(emptyColumns) > 0 {
56+
t.Fatalf("empty columns: %v", emptyColumns)
57+
}
6758
}

plugins/source/cloudflare/client/transformers.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,9 @@ import (
44
"reflect"
55

66
"github.com/apache/arrow/go/v13/arrow"
7-
"github.com/cloudquery/plugin-sdk/v3/schema"
8-
"github.com/cloudquery/plugin-sdk/v3/transformers"
9-
"github.com/cloudquery/plugin-sdk/v3/types"
7+
"github.com/cloudquery/plugin-sdk/v4/schema"
8+
"github.com/cloudquery/plugin-sdk/v4/transformers"
9+
"github.com/cloudquery/plugin-sdk/v4/types"
1010
)
1111

1212
func typeTransformer(field reflect.StructField) (arrow.DataType, error) {

plugins/source/cloudflare/go.mod

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,7 @@ go 1.19
55
require (
66
github.com/apache/arrow/go/v13 v13.0.0-20230630125530-5a06b2ec2a8e
77
github.com/cloudflare/cloudflare-go v0.57.1
8-
github.com/cloudquery/plugin-pb-go v1.6.0
9-
github.com/cloudquery/plugin-sdk/v3 v3.10.6
8+
github.com/cloudquery/plugin-sdk/v4 v4.7.1-rc1
109
github.com/golang/mock v1.6.0
1110
github.com/rs/zerolog v1.29.1
1211
github.com/thoas/go-funk v0.9.3
@@ -15,6 +14,7 @@ require (
1514
replace github.com/apache/arrow/go/v13 => github.com/cloudquery/arrow/go/v13 v13.0.0-20230703001435-df3b664a289d
1615

1716
require (
17+
github.com/cloudquery/plugin-pb-go v1.6.0 // indirect
1818
github.com/cloudquery/plugin-sdk/v2 v2.7.0 // indirect
1919
github.com/davecgh/go-spew v1.1.1 // indirect
2020
github.com/getsentry/sentry-go v0.20.0 // indirect
@@ -29,13 +29,12 @@ require (
2929
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
3030
github.com/hashicorp/go-retryablehttp v0.7.2 // indirect
3131
github.com/inconshreveable/mousetrap v1.1.0 // indirect
32-
github.com/klauspost/compress v1.16.0 // indirect
32+
github.com/klauspost/compress v1.16.6 // indirect
3333
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
3434
github.com/mattn/go-colorable v0.1.13 // indirect
3535
github.com/mattn/go-isatty v0.0.19 // indirect
3636
github.com/pierrec/lz4/v4 v4.1.17 // indirect
3737
github.com/pmezard/go-difflib v1.0.0 // indirect
38-
github.com/spf13/cast v1.5.0 // indirect
3938
github.com/spf13/cobra v1.6.1 // indirect
4039
github.com/spf13/pflag v1.0.5 // indirect
4140
github.com/stretchr/testify v1.8.4 // indirect

plugins/source/cloudflare/go.sum

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,8 @@ github.com/cloudquery/plugin-pb-go v1.6.0 h1:4g+tPhRAQUWpGyQBMcj6D3yVwKqwEN3nynv
4747
github.com/cloudquery/plugin-pb-go v1.6.0/go.mod h1:R0Wse6NbJDZIHcRQjJ1sZGYDo3mrIDm4k3El1YUrvGA=
4848
github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U=
4949
github.com/cloudquery/plugin-sdk/v2 v2.7.0/go.mod h1:pAX6ojIW99b/Vg4CkhnsGkRIzNaVEceYMR+Bdit73ug=
50-
github.com/cloudquery/plugin-sdk/v3 v3.10.6 h1:KqTsLZ6OA1h8BUMeMcU6BAD6TBW6ojgQaC4zDZMgvu0=
51-
github.com/cloudquery/plugin-sdk/v3 v3.10.6/go.mod h1:QhBaVgiNyQ3P6uAzJWOYpYykHXL+WDZffwg1riTwv60=
50+
github.com/cloudquery/plugin-sdk/v4 v4.7.1-rc1 h1:de3TwdcKEO+bjfPjOvmNfaOBfrIkEOiK8TeS54yj86I=
51+
github.com/cloudquery/plugin-sdk/v4 v4.7.1-rc1/go.mod h1:Y5HzxesZrpmSTUrbvR8EmGCK5wXDAhqSqzNEXC2ouvI=
5252
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
5353
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
5454
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
@@ -63,7 +63,6 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
6363
github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
6464
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
6565
github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
66-
github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE=
6766
github.com/getsentry/sentry-go v0.20.0 h1:bwXW98iMRIWxn+4FgPW7vMrjmbym6HblXALmhjHmQaQ=
6867
github.com/getsentry/sentry-go v0.20.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
6968
github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
@@ -158,8 +157,8 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1
158157
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
159158
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
160159
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
161-
github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4=
162-
github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
160+
github.com/klauspost/compress v1.16.6 h1:91SKEy4K37vkp255cJ8QesJhjyRO0hn9i9G0GoUwLsk=
161+
github.com/klauspost/compress v1.16.6/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
163162
github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg=
164163
github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
165164
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@@ -192,8 +191,6 @@ github.com/rs/zerolog v1.19.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJ
192191
github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc=
193192
github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU=
194193
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
195-
github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w=
196-
github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU=
197194
github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA=
198195
github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY=
199196
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=

plugins/source/cloudflare/main.go

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,17 @@
11
package main
22

33
import (
4-
"github.com/cloudquery/cloudquery/plugins/source/cloudflare/resources/plugin"
5-
"github.com/cloudquery/plugin-sdk/v3/serve"
4+
"context"
5+
"log"
6+
7+
internalPlugin "github.com/cloudquery/cloudquery/plugins/source/cloudflare/resources/plugin"
8+
"github.com/cloudquery/plugin-sdk/v4/serve"
69
)
710

811
const sentryDSN = "https://[email protected]/6747634"
912

1013
func main() {
11-
serve.Source(plugin.Plugin(), serve.WithSourceSentryDSN(sentryDSN))
14+
if err := serve.Plugin(internalPlugin.Plugin(), serve.WithPluginSentryDSN(sentryDSN)).Serve(context.Background()); err != nil {
15+
log.Fatal(err)
16+
}
1217
}

0 commit comments

Comments
 (0)