Skip to content

Commit ee3465f

Browse files
brucehsubrucehsu
authored
feat(hackernews)!: Migrate to SDKv3 (#10831)
#### Summary This PR addresses #10749. - Migrate dependencies to plugin-sdk/v3 - Deprecate `ColumnCreationOptions` in favour of inline fields - Migrate to Arrow types (`TypeInt`->`Int64`, `TypeTimestamp`->`Timestamp_us`) BEGIN_COMMIT_OVERRIDE feat: Update to use [Apache Arrow](https://arrow.apache.org/) type system (#10831) BREAKING-CHANGE: This release introduces an internal change to our type system to use [Apache Arrow](https://arrow.apache.org/). This should not have any visible breaking changes, however due to the size of the change we are introducing it under a major version bump to communicate that it might have some bugs that we weren't able to catch during our internal tests. If you encounter an issue during the upgrade, please submit a [bug report](https://github.com/cloudquery/cloudquery/issues/new/choose). You will also need to update destinations depending on which one you use: - Azure Blob Storage >= v3.2.0 - BigQuery >= v3.0.0 - ClickHouse >= v3.1.1 - DuckDB >= v1.1.6 - Elasticsearch >= v2.0.0 - File >= v3.2.0 - Firehose >= v2.0.2 - GCS >= v3.2.0 - Gremlin >= v2.1.10 - Kafka >= v3.0.1 - Meilisearch >= v2.0.1 - Microsoft SQL Server >= v4.2.0 - MongoDB >= v2.0.1 - MySQL >= v2.0.2 - Neo4j >= v3.0.0 - PostgreSQL >= v4.2.0 - S3 >= v4.4.0 - Snowflake >= v2.1.1 - SQLite >= v2.2.0 END_COMMIT_OVERRIDE
1 parent df8dc8e commit ee3465f

File tree

12 files changed

+48
-44
lines changed

12 files changed

+48
-44
lines changed

‎plugins/source/hackernews/client/client.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,9 @@ import (
77

88
"github.com/cloudquery/cloudquery/plugins/source/hackernews/client/services"
99
"github.com/cloudquery/plugin-pb-go/specs"
10-
"github.com/cloudquery/plugin-sdk/v2/backend"
11-
"github.com/cloudquery/plugin-sdk/v2/plugins/source"
12-
"github.com/cloudquery/plugin-sdk/v2/schema"
10+
"github.com/cloudquery/plugin-sdk/v3/backend"
11+
"github.com/cloudquery/plugin-sdk/v3/plugins/source"
12+
"github.com/cloudquery/plugin-sdk/v3/schema"
1313
"github.com/hermanschaaf/hackernews"
1414
"github.com/rs/zerolog"
1515
)

‎plugins/source/hackernews/client/resolvers.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import (
55
"fmt"
66
"time"
77

8-
"github.com/cloudquery/plugin-sdk/v2/schema"
8+
"github.com/cloudquery/plugin-sdk/v3/schema"
99
"github.com/thoas/go-funk"
1010
)
1111

‎plugins/source/hackernews/client/services/backend.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
package services
22

3-
import "github.com/cloudquery/plugin-sdk/v2/backend"
3+
import "github.com/cloudquery/plugin-sdk/v3/backend"
44

55
//go:generate mockgen -package=mocks -destination=../mocks/backend.go -source=backend.go BackendClient
66
type Backend interface {

‎plugins/source/hackernews/client/testing.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,9 @@ import (
88

99
"github.com/cloudquery/cloudquery/plugins/source/hackernews/client/services"
1010
"github.com/cloudquery/plugin-pb-go/specs"
11-
"github.com/cloudquery/plugin-sdk/v2/backend"
12-
"github.com/cloudquery/plugin-sdk/v2/plugins/source"
13-
"github.com/cloudquery/plugin-sdk/v2/schema"
11+
"github.com/cloudquery/plugin-sdk/v3/backend"
12+
"github.com/cloudquery/plugin-sdk/v3/plugins/source"
13+
"github.com/cloudquery/plugin-sdk/v3/schema"
1414
"github.com/golang/mock/gomock"
1515
"github.com/rs/zerolog"
1616
)

‎plugins/source/hackernews/go.mod‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,9 @@ module github.com/cloudquery/cloudquery/plugins/source/hackernews
33
go 1.19
44

55
require (
6+
github.com/apache/arrow/go/v13 v13.0.0-20230509040948-de6c3cd2b604
67
github.com/cloudquery/plugin-pb-go v1.0.8
7-
github.com/cloudquery/plugin-sdk/v2 v2.7.0
8+
github.com/cloudquery/plugin-sdk/v3 v3.6.4
89
github.com/golang/mock v1.6.0
910
github.com/hermanschaaf/hackernews v1.0.1
1011
github.com/rs/zerolog v1.29.0
@@ -17,8 +18,8 @@ replace github.com/apache/arrow/go/v13 => github.com/cloudquery/arrow/go/v13 v13
1718

1819
require (
1920
github.com/andybalholm/brotli v1.0.5 // indirect
20-
github.com/apache/arrow/go/v13 v13.0.0-20230509040948-de6c3cd2b604 // indirect
2121
github.com/apache/thrift v0.16.0 // indirect
22+
github.com/cloudquery/plugin-sdk/v2 v2.7.0 // indirect
2223
github.com/davecgh/go-spew v1.1.1 // indirect
2324
github.com/getsentry/sentry-go v0.20.0 // indirect
2425
github.com/ghodss/yaml v1.0.0 // indirect
@@ -37,6 +38,7 @@ require (
3738
github.com/mattn/go-isatty v0.0.18 // indirect
3839
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
3940
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
41+
github.com/pierrec/lz4/v4 v4.1.15 // indirect
4042
github.com/pmezard/go-difflib v1.0.0 // indirect
4143
github.com/spf13/cast v1.5.0 // indirect
4244
github.com/spf13/cobra v1.6.1 // indirect

‎plugins/source/hackernews/go.sum‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,8 @@ github.com/cloudquery/plugin-pb-go v1.0.8 h1:wn3GXhcNItcP+6wUUZuzUFbvdL59liKBO37
5050
github.com/cloudquery/plugin-pb-go v1.0.8/go.mod h1:vAGA27psem7ZZNAY4a3S9TKuA/JDQWstjKcHPJX91Mc=
5151
github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U=
5252
github.com/cloudquery/plugin-sdk/v2 v2.7.0/go.mod h1:pAX6ojIW99b/Vg4CkhnsGkRIzNaVEceYMR+Bdit73ug=
53+
github.com/cloudquery/plugin-sdk/v3 v3.6.4 h1:P4OkS5tJYkv3OqeL60DAVqXXbFQUyPKJ5YDtAgjl9b4=
54+
github.com/cloudquery/plugin-sdk/v3 v3.6.4/go.mod h1:3JrZXEULmGXpkOukVaRIzaA63d7TJr9Ukp6hemTjbtc=
5355
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
5456
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
5557
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
@@ -178,6 +180,7 @@ github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8D
178180
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE=
179181
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
180182
github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
183+
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
181184
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
182185
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
183186
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=

‎plugins/source/hackernews/main.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ package main
22

33
import (
44
"github.com/cloudquery/cloudquery/plugins/source/hackernews/resources/plugin"
5-
"github.com/cloudquery/plugin-sdk/v2/serve"
5+
"github.com/cloudquery/plugin-sdk/v3/serve"
66
)
77

88
const sentryDSN = "https://[email protected]/4504446079860736"

‎plugins/source/hackernews/resources/plugin/plugin.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@ package plugin
33
import (
44
"github.com/cloudquery/cloudquery/plugins/source/hackernews/client"
55
"github.com/cloudquery/cloudquery/plugins/source/hackernews/resources/services/items"
6-
"github.com/cloudquery/plugin-sdk/v2/plugins/source"
7-
"github.com/cloudquery/plugin-sdk/v2/schema"
6+
"github.com/cloudquery/plugin-sdk/v3/plugins/source"
7+
"github.com/cloudquery/plugin-sdk/v3/schema"
88
)
99

1010
var (

‎plugins/source/hackernews/resources/services/items/items.go‎

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,10 @@ package items
33
import (
44
"reflect"
55

6+
"github.com/apache/arrow/go/v13/arrow"
67
"github.com/cloudquery/cloudquery/plugins/source/hackernews/client"
7-
"github.com/cloudquery/plugin-sdk/v2/schema"
8-
"github.com/cloudquery/plugin-sdk/v2/transformers"
8+
"github.com/cloudquery/plugin-sdk/v3/schema"
9+
"github.com/cloudquery/plugin-sdk/v3/transformers"
910
"github.com/hermanschaaf/hackernews"
1011
)
1112

@@ -23,21 +24,19 @@ func Items() *schema.Table {
2324
),
2425
Columns: []schema.Column{
2526
{
26-
Name: "id",
27-
Type: schema.TypeInt,
28-
Resolver: schema.PathResolver("ID"),
29-
CreationOptions: schema.ColumnCreationOptions{
30-
PrimaryKey: true,
31-
IncrementalKey: true,
32-
},
27+
Name: "id",
28+
Type: arrow.PrimitiveTypes.Int64,
29+
Resolver: schema.PathResolver("ID"),
30+
PrimaryKey: true,
31+
IncrementalKey: true,
3332
},
3433
},
3534
}
3635
}
3736

38-
func typeTransformer(f reflect.StructField) (schema.ValueType, error) {
37+
func typeTransformer(f reflect.StructField) (arrow.DataType, error) {
3938
if f.Name == "Time" {
40-
return schema.TypeTimestamp, nil
39+
return arrow.FixedWidthTypes.Timestamp_us, nil
4140
}
4241
return transformers.DefaultTypeTransformer(f)
4342
}

‎plugins/source/hackernews/resources/services/items/items_fetch.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import (
88
"time"
99

1010
"github.com/cloudquery/cloudquery/plugins/source/hackernews/client"
11-
"github.com/cloudquery/plugin-sdk/v2/schema"
11+
"github.com/cloudquery/plugin-sdk/v3/schema"
1212
"github.com/hermanschaaf/hackernews"
1313
"golang.org/x/sync/errgroup"
1414
)

0 commit comments

Comments
 (0)