Skip to content

Commit 4a75335

Browse files
disqdisq
authored
feat(vercel)!: Migrate to Arrow native SDK (#10996)
Closes #10772 BEGIN_COMMIT_OVERRIDE feat: Update to use [Apache Arrow](https://arrow.apache.org/) type system (#10983) BREAKING-CHANGE: This release introduces an internal change to our type system to use [Apache Arrow](https://arrow.apache.org/). This should not have any visible breaking changes, however due to the size of the change we are introducing it under a major version bump to communicate that it might have some bugs that we weren't able to catch during our internal tests. If you encounter an issue during the upgrade, please submit a [bug report](https://github.com/cloudquery/cloudquery/issues/new/choose). You will also need to update destinations depending on which one you use: - Azure Blob Storage >= v3.2.0 - BigQuery >= v3.0.0 - ClickHouse >= v3.1.1 - DuckDB >= v1.1.6 - Elasticsearch >= v2.0.0 - File >= v3.2.0 - Firehose >= v2.0.2 - GCS >= v3.2.0 - Gremlin >= v2.1.10 - Kafka >= v3.0.1 - Meilisearch >= v2.0.1 - Microsoft SQL Server >= v4.2.0 - MongoDB >= v2.0.1 - MySQL >= v2.0.2 - Neo4j >= v3.0.0 - PostgreSQL >= v4.2.0 - S3 >= v4.4.0 - Snowflake >= v2.1.1 - SQLite >= v2.2.0 END_COMMIT_OVERRIDE
1 parent 7b4c5e2 commit 4a75335

36 files changed

+259
-267
lines changed

‎plugins/source/vercel/client/client.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,9 @@ import (
1010

1111
"github.com/cloudquery/cloudquery/plugins/source/vercel/internal/vercel"
1212
"github.com/cloudquery/plugin-pb-go/specs"
13-
"github.com/cloudquery/plugin-sdk/v2/backend"
14-
"github.com/cloudquery/plugin-sdk/v2/plugins/source"
15-
"github.com/cloudquery/plugin-sdk/v2/schema"
13+
"github.com/cloudquery/plugin-sdk/v3/backend"
14+
"github.com/cloudquery/plugin-sdk/v3/plugins/source"
15+
"github.com/cloudquery/plugin-sdk/v3/schema"
1616
"github.com/rs/zerolog"
1717
)
1818

‎plugins/source/vercel/client/multiplexer.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
package client
22

33
import (
4-
"github.com/cloudquery/plugin-sdk/v2/schema"
4+
"github.com/cloudquery/plugin-sdk/v3/schema"
55
)
66

77
func TeamMultiplex(meta schema.ClientMeta) []schema.ClientMeta {

‎plugins/source/vercel/client/testing.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,9 @@ import (
1414

1515
"github.com/cloudquery/cloudquery/plugins/source/vercel/internal/vercel"
1616
"github.com/cloudquery/plugin-pb-go/specs"
17-
"github.com/cloudquery/plugin-sdk/v2/backend"
18-
"github.com/cloudquery/plugin-sdk/v2/plugins/source"
19-
"github.com/cloudquery/plugin-sdk/v2/schema"
17+
"github.com/cloudquery/plugin-sdk/v3/backend"
18+
"github.com/cloudquery/plugin-sdk/v3/plugins/source"
19+
"github.com/cloudquery/plugin-sdk/v3/schema"
2020
"github.com/gorilla/mux"
2121
"github.com/rs/zerolog"
2222
)

‎plugins/source/vercel/client/transformers.go‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,11 @@ package client
33
import (
44
"reflect"
55

6+
"github.com/apache/arrow/go/v13/arrow"
67
"github.com/cloudquery/cloudquery/plugins/source/vercel/internal/vercel"
7-
"github.com/cloudquery/plugin-sdk/v2/schema"
8-
"github.com/cloudquery/plugin-sdk/v2/transformers"
8+
"github.com/cloudquery/plugin-sdk/v3/schema"
9+
"github.com/cloudquery/plugin-sdk/v3/transformers"
10+
"github.com/cloudquery/plugin-sdk/v3/types"
911
)
1012

1113
var options = []transformers.StructTransformerOption{
@@ -16,11 +18,13 @@ func TransformWithStruct(t any, opts ...transformers.StructTransformerOption) sc
1618
return transformers.TransformWithStruct(t, append(options, opts...)...)
1719
}
1820

19-
func typeTransformer(field reflect.StructField) (schema.ValueType, error) {
21+
func typeTransformer(field reflect.StructField) (arrow.DataType, error) {
2022
switch field.Type {
2123
case reflect.TypeOf(vercel.MilliTime{}), reflect.TypeOf(&vercel.MilliTime{}):
22-
return schema.TypeTimestamp, nil
24+
return arrow.FixedWidthTypes.Timestamp_us, nil
25+
case reflect.TypeOf([]any{}):
26+
return types.ExtensionTypes.JSON, nil
2327
default:
24-
return schema.TypeInvalid, nil
28+
return nil, nil
2529
}
2630
}

‎plugins/source/vercel/go.mod‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,19 +3,19 @@ module github.com/cloudquery/cloudquery/plugins/source/vercel
33
go 1.19
44

55
require (
6+
github.com/apache/arrow/go/v13 v13.0.0-20230509040948-de6c3cd2b604
67
github.com/cloudquery/plugin-pb-go v1.0.8
7-
github.com/cloudquery/plugin-sdk/v2 v2.7.0
8+
github.com/cloudquery/plugin-sdk/v3 v3.6.4
89
github.com/gorilla/mux v1.8.0
910
github.com/rs/zerolog v1.29.0
1011
)
1112

12-
// TODO: remove once all updates are merged
13-
replace github.com/apache/arrow/go/v13 => github.com/cloudquery/arrow/go/v13 v13.0.0-20230509053643-898a79b1d3c8
13+
replace github.com/apache/arrow/go/v13 => github.com/cloudquery/arrow/go/v13 v13.0.0-20230525142029-2d32efeedad8
1414

1515
require (
1616
github.com/andybalholm/brotli v1.0.5 // indirect
17-
github.com/apache/arrow/go/v13 v13.0.0-20230509040948-de6c3cd2b604 // indirect
1817
github.com/apache/thrift v0.16.0 // indirect
18+
github.com/cloudquery/plugin-sdk/v2 v2.7.0 // indirect
1919
github.com/davecgh/go-spew v1.1.1 // indirect
2020
github.com/getsentry/sentry-go v0.20.0 // indirect
2121
github.com/ghodss/yaml v1.0.0 // indirect
@@ -34,6 +34,7 @@ require (
3434
github.com/mattn/go-isatty v0.0.18 // indirect
3535
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
3636
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
37+
github.com/pierrec/lz4/v4 v4.1.15 // indirect
3738
github.com/pmezard/go-difflib v1.0.0 // indirect
3839
github.com/spf13/cast v1.5.0 // indirect
3940
github.com/spf13/cobra v1.6.1 // indirect

‎plugins/source/vercel/go.sum‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,12 +44,14 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR
4444
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
4545
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
4646
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
47-
github.com/cloudquery/arrow/go/v13 v13.0.0-20230509053643-898a79b1d3c8 h1:CmgLSEGQNLHpUQ5cU4L4aF7cuJZRnc1toIIWqC1gmPg=
48-
github.com/cloudquery/arrow/go/v13 v13.0.0-20230509053643-898a79b1d3c8/go.mod h1:/XatdE3kDIBqZKhZ7OBUHwP2jaASDFZHqF4puOWM8po=
47+
github.com/cloudquery/arrow/go/v13 v13.0.0-20230525142029-2d32efeedad8 h1:/mgK+To5HNUzuZDWdVygbfn8oaPG5c7I/8qCxwwpWO8=
48+
github.com/cloudquery/arrow/go/v13 v13.0.0-20230525142029-2d32efeedad8/go.mod h1:/XatdE3kDIBqZKhZ7OBUHwP2jaASDFZHqF4puOWM8po=
4949
github.com/cloudquery/plugin-pb-go v1.0.8 h1:wn3GXhcNItcP+6wUUZuzUFbvdL59liKBO37/izMi+FQ=
5050
github.com/cloudquery/plugin-pb-go v1.0.8/go.mod h1:vAGA27psem7ZZNAY4a3S9TKuA/JDQWstjKcHPJX91Mc=
5151
github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U=
5252
github.com/cloudquery/plugin-sdk/v2 v2.7.0/go.mod h1:pAX6ojIW99b/Vg4CkhnsGkRIzNaVEceYMR+Bdit73ug=
53+
github.com/cloudquery/plugin-sdk/v3 v3.6.4 h1:P4OkS5tJYkv3OqeL60DAVqXXbFQUyPKJ5YDtAgjl9b4=
54+
github.com/cloudquery/plugin-sdk/v3 v3.6.4/go.mod h1:3JrZXEULmGXpkOukVaRIzaA63d7TJr9Ukp6hemTjbtc=
5355
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
5456
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
5557
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
@@ -176,6 +178,7 @@ github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8D
176178
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE=
177179
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
178180
github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
181+
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
179182
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
180183
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
181184
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=

‎plugins/source/vercel/main.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ package main
22

33
import (
44
"github.com/cloudquery/cloudquery/plugins/source/vercel/resources/plugin"
5-
"github.com/cloudquery/plugin-sdk/v2/serve"
5+
"github.com/cloudquery/plugin-sdk/v3/serve"
66
)
77

88
const sentryDSN = "https://[email protected]/4504316330901504"

‎plugins/source/vercel/resources/plugin/plugin.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@ import (
66
"github.com/cloudquery/cloudquery/plugins/source/vercel/resources/services/domain"
77
"github.com/cloudquery/cloudquery/plugins/source/vercel/resources/services/project"
88
"github.com/cloudquery/cloudquery/plugins/source/vercel/resources/services/team"
9-
"github.com/cloudquery/plugin-sdk/v2/plugins/source"
10-
"github.com/cloudquery/plugin-sdk/v2/schema"
9+
"github.com/cloudquery/plugin-sdk/v3/plugins/source"
10+
"github.com/cloudquery/plugin-sdk/v3/schema"
1111
)
1212

1313
var (

‎plugins/source/vercel/resources/services/deployment/deployment_checks.go‎

Lines changed: 10 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
package deployment
22

33
import (
4+
"github.com/apache/arrow/go/v13/arrow"
45
"github.com/cloudquery/cloudquery/plugins/source/vercel/client"
56
"github.com/cloudquery/cloudquery/plugins/source/vercel/internal/vercel"
6-
"github.com/cloudquery/plugin-sdk/v2/schema"
7+
"github.com/cloudquery/plugin-sdk/v3/schema"
78
)
89

910
func DeploymentChecks() *schema.Table {
@@ -15,20 +16,16 @@ func DeploymentChecks() *schema.Table {
1516
IsIncremental: true,
1617
Columns: []schema.Column{
1718
{
18-
Name: "deployment_id",
19-
Type: schema.TypeString,
20-
Resolver: schema.ParentColumnResolver("uid"),
21-
CreationOptions: schema.ColumnCreationOptions{
22-
PrimaryKey: true,
23-
},
19+
Name: "deployment_id",
20+
Type: arrow.BinaryTypes.String,
21+
Resolver: schema.ParentColumnResolver("uid"),
22+
PrimaryKey: true,
2423
},
2524
{
26-
Name: "id",
27-
Type: schema.TypeString,
28-
Resolver: schema.PathResolver("ID"),
29-
CreationOptions: schema.ColumnCreationOptions{
30-
PrimaryKey: true,
31-
},
25+
Name: "id",
26+
Type: arrow.BinaryTypes.String,
27+
Resolver: schema.PathResolver("ID"),
28+
PrimaryKey: true,
3229
},
3330
},
3431
}

‎plugins/source/vercel/resources/services/deployment/deployment_checks_fetch.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import (
55

66
"github.com/cloudquery/cloudquery/plugins/source/vercel/client"
77
"github.com/cloudquery/cloudquery/plugins/source/vercel/internal/vercel"
8-
"github.com/cloudquery/plugin-sdk/v2/schema"
8+
"github.com/cloudquery/plugin-sdk/v3/schema"
99
)
1010

1111
func fetchDeploymentChecks(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error {

0 commit comments

Comments
 (0)