Skip to content

Commit 4429b31

Browse files
yevgenypatsyevgenypats
authored
feat(k8s)!: Update to SDK V3 native Apache Arrow Support (#10978)
Closes #10754 BEGIN_COMMIT_OVERRIDE feat: Update to use [Apache Arrow](https://arrow.apache.org/) type system (#10978) BREAKING-CHANGE: This release introduces an internal change to our type system to use [Apache Arrow](https://arrow.apache.org/). This should not have any visible breaking changes, however due to the size of the change we are introducing it under a major version bump to communicate that it might have some bugs that we weren't able to catch during our internal tests. If you encounter an issue during the upgrade, please submit a [bug report](https://github.com/cloudquery/cloudquery/issues/new/choose). You will also need to update destinations depending on which one you use: - Azure Blob Storage >= v3.2.0 - BigQuery >= v3.0.0 - ClickHouse >= v3.1.1 - DuckDB >= v1.1.6 - Elasticsearch >= v2.0.0 - File >= v3.2.0 - Firehose >= v2.0.2 - GCS >= v3.2.0 - Gremlin >= v2.1.10 - Kafka >= v3.0.1 - Meilisearch >= v2.0.1 - Microsoft SQL Server >= v4.2.0 - MongoDB >= v2.0.1 - MySQL >= v2.0.2 - Neo4j >= v3.0.0 - PostgreSQL >= v4.2.0 - S3 >= v4.4.0 - Snowflake >= v2.1.1 - SQLite >= v2.2.0 END_COMMIT_OVERRIDE
1 parent eb380c2 commit 4429b31

File tree

137 files changed

+1230
-1216
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

137 files changed

+1230
-1216
lines changed

‎.github/workflows/source_k8s.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,7 @@ jobs:
144144
- name: Setup CloudQuery
145145
uses: cloudquery/setup-cloudquery@v3
146146
with:
147-
version: 'v2.5.3'
147+
version: 'v3.5.0'
148148
- name: Migrate DB
149149
run: cloudquery migrate test/policy_cq_config.yml
150150
env:

‎plugins/source/k8s/client/client.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@ import (
66
"strings"
77

88
"github.com/cloudquery/plugin-pb-go/specs"
9-
"github.com/cloudquery/plugin-sdk/v2/plugins/source"
10-
"github.com/cloudquery/plugin-sdk/v2/schema"
9+
"github.com/cloudquery/plugin-sdk/v3/plugins/source"
10+
"github.com/cloudquery/plugin-sdk/v3/schema"
1111
"github.com/rs/zerolog"
1212
apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
1313
"k8s.io/client-go/kubernetes"

‎plugins/source/k8s/client/columns.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
package client
22

33
import (
4-
"github.com/cloudquery/plugin-sdk/v2/schema"
4+
"github.com/apache/arrow/go/v13/arrow"
5+
"github.com/cloudquery/plugin-sdk/v3/schema"
56
)
67

7-
var ContextColumn = schema.Column{Name: "context", Type: schema.TypeString, Resolver: ResolveContext}
8+
var ContextColumn = schema.Column{Name: "context", Type: arrow.BinaryTypes.String, Resolver: ResolveContext}

‎plugins/source/k8s/client/helpers.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import (
44
"context"
55
"reflect"
66

7-
"github.com/cloudquery/plugin-sdk/v2/schema"
7+
"github.com/cloudquery/plugin-sdk/v3/schema"
88
"github.com/thoas/go-funk"
99
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1010
"k8s.io/apimachinery/pkg/types"

‎plugins/source/k8s/client/testing.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,8 @@ import (
88
"time"
99

1010
"github.com/cloudquery/plugin-pb-go/specs"
11-
"github.com/cloudquery/plugin-sdk/v2/plugins/source"
12-
"github.com/cloudquery/plugin-sdk/v2/schema"
11+
"github.com/cloudquery/plugin-sdk/v3/plugins/source"
12+
"github.com/cloudquery/plugin-sdk/v3/schema"
1313
"github.com/golang/mock/gomock"
1414
"github.com/rs/zerolog"
1515
v1 "k8s.io/api/core/v1"

‎plugins/source/k8s/client/transformers.go‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,9 @@ package client
33
import (
44
"reflect"
55

6-
"github.com/cloudquery/plugin-sdk/v2/schema"
7-
"github.com/cloudquery/plugin-sdk/v2/transformers"
6+
"github.com/apache/arrow/go/v13/arrow"
7+
"github.com/cloudquery/plugin-sdk/v3/schema"
8+
"github.com/cloudquery/plugin-sdk/v3/transformers"
89
)
910

1011
var (
@@ -32,9 +33,9 @@ func WithMoreSkipFields(extra ...string) transformers.StructTransformerOption {
3233
return transformers.WithSkipFields(append(skipFields, extra...)...)
3334
}
3435

35-
func typeTransformer(field reflect.StructField) (schema.ValueType, error) {
36+
func typeTransformer(field reflect.StructField) (arrow.DataType, error) {
3637
if isK8sTimeStruct(field.Type) {
37-
return schema.TypeTimestamp, nil
38+
return arrow.FixedWidthTypes.Timestamp_us, nil
3839
}
3940

4041
return transformers.DefaultTypeTransformer(field)

‎plugins/source/k8s/go.mod‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,9 @@ module github.com/cloudquery/cloudquery/plugins/source/k8s
33
go 1.19
44

55
require (
6+
github.com/apache/arrow/go/v13 v13.0.0-20230509040948-de6c3cd2b604
67
github.com/cloudquery/plugin-pb-go v1.0.8
7-
github.com/cloudquery/plugin-sdk/v2 v2.7.0
8+
github.com/cloudquery/plugin-sdk/v3 v3.6.4
89
github.com/golang/mock v1.6.0
910
github.com/google/gnostic v0.6.9
1011
github.com/rs/zerolog v1.29.0
@@ -21,8 +22,8 @@ replace github.com/apache/arrow/go/v13 => github.com/cloudquery/arrow/go/v13 v13
2122

2223
require (
2324
github.com/andybalholm/brotli v1.0.5 // indirect
24-
github.com/apache/arrow/go/v13 v13.0.0-20230509040948-de6c3cd2b604 // indirect
2525
github.com/apache/thrift v0.16.0 // indirect
26+
github.com/cloudquery/plugin-sdk/v2 v2.7.0 // indirect
2627
github.com/davecgh/go-spew v1.1.1 // indirect
2728
github.com/emicklei/go-restful/v3 v3.9.0 // indirect
2829
github.com/getsentry/sentry-go v0.20.0 // indirect
@@ -56,6 +57,7 @@ require (
5657
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
5758
github.com/modern-go/reflect2 v1.0.2 // indirect
5859
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
60+
github.com/pierrec/lz4/v4 v4.1.15 // indirect
5961
github.com/pmezard/go-difflib v1.0.0 // indirect
6062
github.com/spf13/cast v1.5.0 // indirect
6163
github.com/spf13/cobra v1.6.1 // indirect

‎plugins/source/k8s/go.sum‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,8 @@ github.com/cloudquery/plugin-pb-go v1.0.8 h1:wn3GXhcNItcP+6wUUZuzUFbvdL59liKBO37
5454
github.com/cloudquery/plugin-pb-go v1.0.8/go.mod h1:vAGA27psem7ZZNAY4a3S9TKuA/JDQWstjKcHPJX91Mc=
5555
github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U=
5656
github.com/cloudquery/plugin-sdk/v2 v2.7.0/go.mod h1:pAX6ojIW99b/Vg4CkhnsGkRIzNaVEceYMR+Bdit73ug=
57+
github.com/cloudquery/plugin-sdk/v3 v3.6.4 h1:P4OkS5tJYkv3OqeL60DAVqXXbFQUyPKJ5YDtAgjl9b4=
58+
github.com/cloudquery/plugin-sdk/v3 v3.6.4/go.mod h1:3JrZXEULmGXpkOukVaRIzaA63d7TJr9Ukp6hemTjbtc=
5759
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
5860
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
5961
github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
@@ -228,6 +230,7 @@ github.com/onsi/ginkgo/v2 v2.4.0 h1:+Ig9nvqgS5OBSACXNk15PLdp0U9XPYROt9CFzVdFGIs=
228230
github.com/onsi/gomega v1.23.0 h1:/oxKu9c2HVap+F3PfKort2Hw5DEU+HGlW8n+tguWsys=
229231
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
230232
github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
233+
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
231234
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
232235
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
233236
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=

‎plugins/source/k8s/main.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ package main
33
import (
44
"github.com/cloudquery/cloudquery/plugins/source/k8s/resources/plugin"
55

6-
"github.com/cloudquery/plugin-sdk/v2/serve"
6+
"github.com/cloudquery/plugin-sdk/v3/serve"
77
)
88

99
var sentryDsn = `https://[email protected]/6771378`

‎plugins/source/k8s/resources/plugin/plugin.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,9 @@ import (
1818
"github.com/cloudquery/cloudquery/plugins/source/k8s/resources/services/policy"
1919
"github.com/cloudquery/cloudquery/plugins/source/k8s/resources/services/rbac"
2020
"github.com/cloudquery/cloudquery/plugins/source/k8s/resources/services/storage"
21-
"github.com/cloudquery/plugin-sdk/v2/caser"
22-
"github.com/cloudquery/plugin-sdk/v2/plugins/source"
23-
"github.com/cloudquery/plugin-sdk/v2/schema"
21+
"github.com/cloudquery/plugin-sdk/v3/caser"
22+
"github.com/cloudquery/plugin-sdk/v3/plugins/source"
23+
"github.com/cloudquery/plugin-sdk/v3/schema"
2424
"golang.org/x/exp/maps"
2525
)
2626

0 commit comments

Comments
 (0)