Skip to content

Commit 2a302b0

Browse files
disqdisq
authored
feat(gandi)!: Migrate to Arrow native SDK (#10981)
Closes #10743 BEGIN_COMMIT_OVERRIDE feat: Update to use [Apache Arrow](https://arrow.apache.org/) type system (#10981) BREAKING-CHANGE: This release introduces an internal change to our type system to use [Apache Arrow](https://arrow.apache.org/). This should not have any visible breaking changes, however due to the size of the change we are introducing it under a major version bump to communicate that it might have some bugs that we weren't able to catch during our internal tests. If you encounter an issue during the upgrade, please submit a [bug report](https://github.com/cloudquery/cloudquery/issues/new/choose). You will also need to update destinations depending on which one you use: - Azure Blob Storage >= v3.2.0 - BigQuery >= v3.0.0 - ClickHouse >= v3.1.1 - DuckDB >= v1.1.6 - Elasticsearch >= v2.0.0 - File >= v3.2.0 - Firehose >= v2.0.2 - GCS >= v3.2.0 - Gremlin >= v2.1.10 - Kafka >= v3.0.1 - Meilisearch >= v2.0.1 - Microsoft SQL Server >= v4.2.0 - MongoDB >= v2.0.1 - MySQL >= v2.0.2 - Neo4j >= v3.0.0 - PostgreSQL >= v4.2.0 - S3 >= v4.4.0 - Snowflake >= v2.1.1 - SQLite >= v2.2.0 END_COMMIT_OVERRIDE
1 parent 46f5e24 commit 2a302b0

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+295
-313
lines changed

‎plugins/source/gandi/client/client.go‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@ import (
77
"time"
88

99
"github.com/cloudquery/plugin-pb-go/specs"
10-
"github.com/cloudquery/plugin-sdk/v2/plugins/source"
11-
"github.com/cloudquery/plugin-sdk/v2/schema"
10+
"github.com/cloudquery/plugin-sdk/v3/plugins/source"
11+
"github.com/cloudquery/plugin-sdk/v3/schema"
1212
"github.com/go-gandi/go-gandi"
1313
"github.com/go-gandi/go-gandi/config"
1414
"github.com/rs/zerolog"
@@ -28,8 +28,6 @@ type Client struct {
2828
Services Services
2929
}
3030

31-
const MaxItemsPerPage = 200
32-
3331
func New(logger zerolog.Logger, services Services, sharingId string) Client {
3432
return Client{
3533
logger: logger,

‎plugins/source/gandi/client/resolvers.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ package client
33
import (
44
"context"
55

6-
"github.com/cloudquery/plugin-sdk/v2/schema"
6+
"github.com/cloudquery/plugin-sdk/v3/schema"
77
)
88

99
func ResolveSharingID(_ context.Context, meta schema.ClientMeta, r *schema.Resource, col schema.Column) error {

‎plugins/source/gandi/client/testing.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,8 @@ import (
88
"time"
99

1010
"github.com/cloudquery/plugin-pb-go/specs"
11-
"github.com/cloudquery/plugin-sdk/v2/plugins/source"
12-
"github.com/cloudquery/plugin-sdk/v2/schema"
11+
"github.com/cloudquery/plugin-sdk/v3/plugins/source"
12+
"github.com/cloudquery/plugin-sdk/v3/schema"
1313
"github.com/golang/mock/gomock"
1414
"github.com/rs/zerolog"
1515
)

‎plugins/source/gandi/go.mod‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,20 +3,20 @@ module github.com/cloudquery/cloudquery/plugins/source/gandi
33
go 1.19
44

55
require (
6+
github.com/apache/arrow/go/v13 v13.0.0-20230509040948-de6c3cd2b604
67
github.com/cloudquery/plugin-pb-go v1.0.8
7-
github.com/cloudquery/plugin-sdk/v2 v2.7.0
8+
github.com/cloudquery/plugin-sdk/v3 v3.6.4
89
github.com/go-gandi/go-gandi v0.5.1-0.20221118201059-f69b292fa399
910
github.com/golang/mock v1.6.0
1011
github.com/rs/zerolog v1.29.0
1112
)
1213

13-
// TODO: remove once all updates are merged
14-
replace github.com/apache/arrow/go/v13 => github.com/cloudquery/arrow/go/v13 v13.0.0-20230509053643-898a79b1d3c8
14+
replace github.com/apache/arrow/go/v13 => github.com/cloudquery/arrow/go/v13 v13.0.0-20230525142029-2d32efeedad8
1515

1616
require (
1717
github.com/andybalholm/brotli v1.0.5 // indirect
18-
github.com/apache/arrow/go/v13 v13.0.0-20230509040948-de6c3cd2b604 // indirect
1918
github.com/apache/thrift v0.16.0 // indirect
19+
github.com/cloudquery/plugin-sdk/v2 v2.7.0 // indirect
2020
github.com/davecgh/go-spew v1.1.1 // indirect
2121
github.com/getsentry/sentry-go v0.20.0 // indirect
2222
github.com/ghodss/yaml v1.0.0 // indirect
@@ -38,6 +38,7 @@ require (
3838
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
3939
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
4040
github.com/peterhellberg/link v1.2.0 // indirect
41+
github.com/pierrec/lz4/v4 v4.1.15 // indirect
4142
github.com/pmezard/go-difflib v1.0.0 // indirect
4243
github.com/smartystreets/assertions v1.13.0 // indirect
4344
github.com/spf13/cast v1.5.0 // indirect

‎plugins/source/gandi/go.sum‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,12 +44,14 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR
4444
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
4545
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
4646
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
47-
github.com/cloudquery/arrow/go/v13 v13.0.0-20230509053643-898a79b1d3c8 h1:CmgLSEGQNLHpUQ5cU4L4aF7cuJZRnc1toIIWqC1gmPg=
48-
github.com/cloudquery/arrow/go/v13 v13.0.0-20230509053643-898a79b1d3c8/go.mod h1:/XatdE3kDIBqZKhZ7OBUHwP2jaASDFZHqF4puOWM8po=
47+
github.com/cloudquery/arrow/go/v13 v13.0.0-20230525142029-2d32efeedad8 h1:/mgK+To5HNUzuZDWdVygbfn8oaPG5c7I/8qCxwwpWO8=
48+
github.com/cloudquery/arrow/go/v13 v13.0.0-20230525142029-2d32efeedad8/go.mod h1:/XatdE3kDIBqZKhZ7OBUHwP2jaASDFZHqF4puOWM8po=
4949
github.com/cloudquery/plugin-pb-go v1.0.8 h1:wn3GXhcNItcP+6wUUZuzUFbvdL59liKBO37/izMi+FQ=
5050
github.com/cloudquery/plugin-pb-go v1.0.8/go.mod h1:vAGA27psem7ZZNAY4a3S9TKuA/JDQWstjKcHPJX91Mc=
5151
github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U=
5252
github.com/cloudquery/plugin-sdk/v2 v2.7.0/go.mod h1:pAX6ojIW99b/Vg4CkhnsGkRIzNaVEceYMR+Bdit73ug=
53+
github.com/cloudquery/plugin-sdk/v3 v3.6.4 h1:P4OkS5tJYkv3OqeL60DAVqXXbFQUyPKJ5YDtAgjl9b4=
54+
github.com/cloudquery/plugin-sdk/v3 v3.6.4/go.mod h1:3JrZXEULmGXpkOukVaRIzaA63d7TJr9Ukp6hemTjbtc=
5355
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
5456
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
5557
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
@@ -185,6 +187,7 @@ github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFSt
185187
github.com/peterhellberg/link v1.2.0 h1:UA5pg3Gp/E0F2WdX7GERiNrPQrM1K6CVJUUWfHa4t6c=
186188
github.com/peterhellberg/link v1.2.0/go.mod h1:gYfAh+oJgQu2SrZHg5hROVRQe1ICoK0/HHJTcE0edxc=
187189
github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
190+
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
188191
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
189192
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
190193
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=

‎plugins/source/gandi/main.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ package main
22

33
import (
44
"github.com/cloudquery/cloudquery/plugins/source/gandi/resources/plugin"
5-
"github.com/cloudquery/plugin-sdk/v2/serve"
5+
"github.com/cloudquery/plugin-sdk/v3/serve"
66
)
77

88
const sentryDSN = "https://[email protected]/4504214828875776"

‎plugins/source/gandi/resources/plugin/plugin.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,9 @@ import (
66
"github.com/cloudquery/cloudquery/plugins/source/gandi/resources/services/domains"
77
"github.com/cloudquery/cloudquery/plugins/source/gandi/resources/services/livedns"
88
"github.com/cloudquery/cloudquery/plugins/source/gandi/resources/services/simplehosting"
9-
"github.com/cloudquery/plugin-sdk/v2/caser"
10-
"github.com/cloudquery/plugin-sdk/v2/plugins/source"
11-
"github.com/cloudquery/plugin-sdk/v2/schema"
9+
"github.com/cloudquery/plugin-sdk/v3/caser"
10+
"github.com/cloudquery/plugin-sdk/v3/plugins/source"
11+
"github.com/cloudquery/plugin-sdk/v3/schema"
1212
)
1313

1414
var (

‎plugins/source/gandi/resources/services/certificates/certificate_packages.go‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
package certificates
22

33
import (
4+
"github.com/apache/arrow/go/v13/arrow"
45
"github.com/cloudquery/cloudquery/plugins/source/gandi/client"
5-
"github.com/cloudquery/plugin-sdk/v2/schema"
6-
"github.com/cloudquery/plugin-sdk/v2/transformers"
6+
"github.com/cloudquery/plugin-sdk/v3/schema"
7+
"github.com/cloudquery/plugin-sdk/v3/transformers"
78
"github.com/go-gandi/go-gandi/certificate"
89
)
910

@@ -15,17 +16,15 @@ func CertificatePackages() *schema.Table {
1516
Columns: []schema.Column{
1617
{
1718
Name: "sharing_id",
18-
Type: schema.TypeString,
19+
Type: arrow.BinaryTypes.String,
1920
Resolver: client.ResolveSharingID,
2021
Description: `The Sharing ID of the resource.`,
2122
},
2223
{
23-
Name: "name",
24-
Type: schema.TypeString,
25-
Resolver: schema.PathResolver("Name"),
26-
CreationOptions: schema.ColumnCreationOptions{
27-
PrimaryKey: true,
28-
},
24+
Name: "name",
25+
Type: arrow.BinaryTypes.String,
26+
Resolver: schema.PathResolver("Name"),
27+
PrimaryKey: true,
2928
},
3029
},
3130
}

‎plugins/source/gandi/resources/services/certificates/certificate_packages_fetch.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import (
44
"context"
55

66
"github.com/cloudquery/cloudquery/plugins/source/gandi/client"
7-
"github.com/cloudquery/plugin-sdk/v2/schema"
7+
"github.com/cloudquery/plugin-sdk/v3/schema"
88
)
99

1010
func fetchCertificatePackages(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error {

‎plugins/source/gandi/resources/services/certificates/certificate_packages_mock_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import (
55

66
"github.com/cloudquery/cloudquery/plugins/source/gandi/client"
77
"github.com/cloudquery/cloudquery/plugins/source/gandi/client/mocks"
8-
"github.com/cloudquery/plugin-sdk/v2/faker"
8+
"github.com/cloudquery/plugin-sdk/v3/faker"
99
"github.com/go-gandi/go-gandi/certificate"
1010
"github.com/golang/mock/gomock"
1111
)

0 commit comments

Comments
 (0)