Skip to content

Commit f558615

Browse files
hermanschaafhermanschaaf
authored
feat(fastly)!: Migrate to Arrow native SDK (#11018)
Closes [#10751](#10741) BEGIN_COMMIT_OVERRIDE feat: Update to use [Apache Arrow](https://arrow.apache.org/) type system (#11018) BREAKING-CHANGE: This release introduces an internal change to our type system to use [Apache Arrow](https://arrow.apache.org/). This should not have any visible breaking changes, however due to the size of the change we are introducing it under a major version bump to communicate that it might have some bugs that we weren't able to catch during our internal tests. If you encounter an issue during the upgrade, please submit a [bug report](https://github.com/cloudquery/cloudquery/issues/new/choose). You will also need to update destinations depending on which one you use: - Azure Blob Storage >= v3.2.0 - BigQuery >= v3.0.0 - ClickHouse >= v3.1.1 - DuckDB >= v1.1.6 - Elasticsearch >= v2.0.0 - File >= v3.2.0 - Firehose >= v2.0.2 - GCS >= v3.2.0 - Gremlin >= v2.1.10 - Kafka >= v3.0.1 - Meilisearch >= v2.0.1 - Microsoft SQL Server >= v4.2.0 - MongoDB >= v2.0.1 - MySQL >= v2.0.2 - Neo4j >= v3.0.0 - PostgreSQL >= v4.2.0 - S3 >= v4.4.0 - Snowflake >= v2.1.1 - SQLite >= v2.2.0 END_COMMIT_OVERRIDE
1 parent 13df940 commit f558615

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

48 files changed

+429
-454
lines changed

‎plugins/source/fastly/client/client.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@ import (
77

88
"github.com/cloudquery/cloudquery/plugins/source/fastly/client/services"
99
"github.com/cloudquery/plugin-pb-go/specs"
10-
"github.com/cloudquery/plugin-sdk/v2/plugins/source"
11-
"github.com/cloudquery/plugin-sdk/v2/schema"
10+
"github.com/cloudquery/plugin-sdk/v3/plugins/source"
11+
"github.com/cloudquery/plugin-sdk/v3/schema"
1212
"github.com/fastly/go-fastly/v7/fastly"
1313
"github.com/rs/zerolog"
1414
"github.com/thoas/go-funk"

‎plugins/source/fastly/client/multiplexers.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
package client
22

33
import (
4-
"github.com/cloudquery/plugin-sdk/v2/schema"
4+
"github.com/cloudquery/plugin-sdk/v3/schema"
55
)
66

77
func ServiceRegionMultiplex(meta schema.ClientMeta) []schema.ClientMeta {

‎plugins/source/fastly/client/resolvers.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import (
44
"context"
55
"time"
66

7-
"github.com/cloudquery/plugin-sdk/v2/schema"
7+
"github.com/cloudquery/plugin-sdk/v3/schema"
88
"github.com/thoas/go-funk"
99
)
1010

‎plugins/source/fastly/client/testing.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,8 @@ import (
88

99
"github.com/cloudquery/cloudquery/plugins/source/fastly/client/services"
1010
"github.com/cloudquery/plugin-pb-go/specs"
11-
"github.com/cloudquery/plugin-sdk/v2/plugins/source"
12-
"github.com/cloudquery/plugin-sdk/v2/schema"
11+
"github.com/cloudquery/plugin-sdk/v3/plugins/source"
12+
"github.com/cloudquery/plugin-sdk/v3/schema"
1313
"github.com/fastly/go-fastly/v7/fastly"
1414
"github.com/golang/mock/gomock"
1515
"github.com/rs/zerolog"

‎plugins/source/fastly/go.mod‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,10 @@ module github.com/cloudquery/cloudquery/plugins/source/fastly
33
go 1.19
44

55
require (
6+
github.com/apache/arrow/go/v13 v13.0.0-20230526080908-77a71305090f
67
github.com/cloudquery/codegen v0.2.1
78
github.com/cloudquery/plugin-pb-go v1.0.8
8-
github.com/cloudquery/plugin-sdk/v2 v2.7.0
9+
github.com/cloudquery/plugin-sdk/v3 v3.6.4
910
github.com/fastly/go-fastly/v7 v7.0.0
1011
github.com/golang/mock v1.6.0
1112
github.com/rs/zerolog v1.29.0
@@ -17,9 +18,9 @@ replace github.com/apache/arrow/go/v13 => github.com/cloudquery/arrow/go/v13 v13
1718

1819
require (
1920
github.com/andybalholm/brotli v1.0.5 // indirect
20-
github.com/apache/arrow/go/v13 v13.0.0-20230509040948-de6c3cd2b604 // indirect
2121
github.com/apache/thrift v0.16.0 // indirect
2222
github.com/cloudquery/plugin-sdk v1.45.0 // indirect
23+
github.com/cloudquery/plugin-sdk/v2 v2.7.0 // indirect
2324
github.com/davecgh/go-spew v1.1.1 // indirect
2425
github.com/getsentry/sentry-go v0.20.0 // indirect
2526
github.com/ghodss/yaml v1.0.0 // indirect
@@ -43,6 +44,7 @@ require (
4344
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
4445
github.com/mitchellh/mapstructure v1.4.3 // indirect
4546
github.com/peterhellberg/link v1.1.0 // indirect
47+
github.com/pierrec/lz4/v4 v4.1.15 // indirect
4648
github.com/pmezard/go-difflib v1.0.0 // indirect
4749
github.com/spf13/cast v1.5.0 // indirect
4850
github.com/spf13/cobra v1.6.1 // indirect

‎plugins/source/fastly/go.sum‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,8 @@ github.com/cloudquery/plugin-sdk v1.45.0 h1:5vrfQZtaO1dp6ebKt8ouXDmPC7eeLuOB3JMd
5454
github.com/cloudquery/plugin-sdk v1.45.0/go.mod h1:9KGuuTGjTCKgh9amKwS+7Zrrqq7/M6lormteOyqoKwg=
5555
github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U=
5656
github.com/cloudquery/plugin-sdk/v2 v2.7.0/go.mod h1:pAX6ojIW99b/Vg4CkhnsGkRIzNaVEceYMR+Bdit73ug=
57+
github.com/cloudquery/plugin-sdk/v3 v3.6.4 h1:P4OkS5tJYkv3OqeL60DAVqXXbFQUyPKJ5YDtAgjl9b4=
58+
github.com/cloudquery/plugin-sdk/v3 v3.6.4/go.mod h1:3JrZXEULmGXpkOukVaRIzaA63d7TJr9Ukp6hemTjbtc=
5759
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
5860
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
5961
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
@@ -196,6 +198,7 @@ github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFSt
196198
github.com/peterhellberg/link v1.1.0 h1:s2+RH8EGuI/mI4QwrWGSYQCRz7uNgip9BaM04HKu5kc=
197199
github.com/peterhellberg/link v1.1.0/go.mod h1:gtSlOT4jmkY8P47hbTc8PTgiDDWpdPbFYl75keYyBB8=
198200
github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
201+
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
199202
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
200203
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
201204
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=

‎plugins/source/fastly/main.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ package main
22

33
import (
44
"github.com/cloudquery/cloudquery/plugins/source/fastly/resources/plugin"
5-
"github.com/cloudquery/plugin-sdk/v2/serve"
5+
"github.com/cloudquery/plugin-sdk/v3/serve"
66
)
77

88
const sentryDSN = "https://[email protected]/4504379395604480"

‎plugins/source/fastly/resources/plugin/plugin.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ package plugin
22

33
import (
44
"github.com/cloudquery/cloudquery/plugins/source/fastly/client"
5-
"github.com/cloudquery/plugin-sdk/v2/plugins/source"
5+
"github.com/cloudquery/plugin-sdk/v3/plugins/source"
66
)
77

88
var (

‎plugins/source/fastly/resources/plugin/tables.go‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎plugins/source/fastly/resources/services/account/account_events.go‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
package account
22

33
import (
4-
"github.com/cloudquery/plugin-sdk/v2/schema"
5-
"github.com/cloudquery/plugin-sdk/v2/transformers"
4+
"github.com/apache/arrow/go/v13/arrow"
5+
"github.com/cloudquery/plugin-sdk/v3/schema"
6+
"github.com/cloudquery/plugin-sdk/v3/transformers"
67
"github.com/fastly/go-fastly/v7/fastly"
78
)
89

@@ -14,16 +15,14 @@ func AccountEvents() *schema.Table {
1415
Transform: transformers.TransformWithStruct(&fastly.Event{}),
1516
Columns: []schema.Column{
1617
{
17-
Name: "id",
18-
Type: schema.TypeString,
19-
Resolver: schema.PathResolver("ID"),
20-
CreationOptions: schema.ColumnCreationOptions{
21-
PrimaryKey: true,
22-
},
18+
Name: "id",
19+
Type: arrow.BinaryTypes.String,
20+
Resolver: schema.PathResolver("ID"),
21+
PrimaryKey: true,
2322
},
2423
{
2524
Name: "ip",
26-
Type: schema.TypeString,
25+
Type: arrow.BinaryTypes.String,
2726
Resolver: schema.PathResolver("IP"),
2827
},
2928
},

0 commit comments

Comments
 (0)