Skip to content

Commit 03473dd

Browse files
disqdisq
authored
feat(okta)!: Migrate to Arrow native SDK (#10983)
Closes #10758 BEGIN_COMMIT_OVERRIDE feat: Update to use [Apache Arrow](https://arrow.apache.org/) type system (#10983) BREAKING-CHANGE: This release introduces an internal change to our type system to use [Apache Arrow](https://arrow.apache.org/). This should not have any visible breaking changes, however due to the size of the change we are introducing it under a major version bump to communicate that it might have some bugs that we weren't able to catch during our internal tests. If you encounter an issue during the upgrade, please submit a [bug report](https://github.com/cloudquery/cloudquery/issues/new/choose). You will also need to update destinations depending on which one you use: - Azure Blob Storage >= v3.2.0 - BigQuery >= v3.0.0 - ClickHouse >= v3.1.1 - DuckDB >= v1.1.6 - Elasticsearch >= v2.0.0 - File >= v3.2.0 - Firehose >= v2.0.2 - GCS >= v3.2.0 - Gremlin >= v2.1.10 - Kafka >= v3.0.1 - Meilisearch >= v2.0.1 - Microsoft SQL Server >= v4.2.0 - MongoDB >= v2.0.1 - MySQL >= v2.0.2 - Neo4j >= v3.0.0 - PostgreSQL >= v4.2.0 - S3 >= v4.4.0 - Snowflake >= v2.1.1 - SQLite >= v2.2.0 END_COMMIT_OVERRIDE
1 parent 2a302b0 commit 03473dd

File tree

22 files changed

+131
-128
lines changed

22 files changed

+131
-128
lines changed

‎plugins/source/okta/client/client.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@ import (
66
"time"
77

88
"github.com/cloudquery/plugin-pb-go/specs"
9-
"github.com/cloudquery/plugin-sdk/v2/plugins/source"
10-
"github.com/cloudquery/plugin-sdk/v2/schema"
9+
"github.com/cloudquery/plugin-sdk/v3/plugins/source"
10+
"github.com/cloudquery/plugin-sdk/v3/schema"
1111
"github.com/okta/okta-sdk-golang/v3/okta"
1212
"github.com/rs/zerolog"
1313
)

‎plugins/source/okta/client/testing.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,8 @@ import (
1313
"time"
1414

1515
"github.com/cloudquery/plugin-pb-go/specs"
16-
"github.com/cloudquery/plugin-sdk/v2/plugins/source"
17-
"github.com/cloudquery/plugin-sdk/v2/schema"
16+
"github.com/cloudquery/plugin-sdk/v3/plugins/source"
17+
"github.com/cloudquery/plugin-sdk/v3/schema"
1818
"github.com/gorilla/mux"
1919
"github.com/okta/okta-sdk-golang/v3/okta"
2020
"github.com/rs/zerolog"

‎plugins/source/okta/client/transformers.go‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,9 @@ import (
55
"fmt"
66
"reflect"
77

8-
"github.com/cloudquery/plugin-sdk/v2/schema"
9-
"github.com/cloudquery/plugin-sdk/v2/transformers"
8+
"github.com/apache/arrow/go/v13/arrow"
9+
"github.com/cloudquery/plugin-sdk/v3/schema"
10+
"github.com/cloudquery/plugin-sdk/v3/transformers"
1011
"github.com/okta/okta-sdk-golang/v3/okta"
1112
"github.com/thoas/go-funk"
1213
)
@@ -21,12 +22,12 @@ func TransformWithStruct(t any, opts ...transformers.StructTransformerOption) sc
2122
return transformers.TransformWithStruct(t, append(options, opts...)...)
2223
}
2324

24-
func typeTransformer(field reflect.StructField) (schema.ValueType, error) {
25+
func typeTransformer(field reflect.StructField) (arrow.DataType, error) {
2526
if field.Type == reflect.TypeOf(okta.NullableTime{}) {
26-
return schema.TypeTimestamp, nil
27+
return arrow.FixedWidthTypes.Timestamp_us, nil
2728
}
2829

29-
return schema.TypeInvalid, nil
30+
return nil, nil
3031
}
3132

3233
func resolverTransformer(field reflect.StructField, path string) schema.ColumnResolver {

‎plugins/source/okta/go.mod‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,22 +3,22 @@ module github.com/cloudquery/cloudquery/plugins/source/okta
33
go 1.19
44

55
require (
6+
github.com/apache/arrow/go/v13 v13.0.0-20230509040948-de6c3cd2b604
67
github.com/cloudquery/plugin-pb-go v1.0.8
7-
github.com/cloudquery/plugin-sdk/v2 v2.7.0
8+
github.com/cloudquery/plugin-sdk/v3 v3.6.4
89
github.com/gorilla/mux v1.8.0
910
github.com/okta/okta-sdk-golang/v3 v3.0.2
1011
github.com/rs/zerolog v1.29.0
1112
github.com/thoas/go-funk v0.9.3
1213
)
1314

14-
// TODO: remove once all updates are merged
15-
replace github.com/apache/arrow/go/v13 => github.com/cloudquery/arrow/go/v13 v13.0.0-20230509053643-898a79b1d3c8
15+
replace github.com/apache/arrow/go/v13 => github.com/cloudquery/arrow/go/v13 v13.0.0-20230525142029-2d32efeedad8
1616

1717
require (
1818
github.com/andybalholm/brotli v1.0.5 // indirect
19-
github.com/apache/arrow/go/v13 v13.0.0-20230509040948-de6c3cd2b604 // indirect
2019
github.com/apache/thrift v0.16.0 // indirect
2120
github.com/cenkalti/backoff/v4 v4.1.3 // indirect
21+
github.com/cloudquery/plugin-sdk/v2 v2.7.0 // indirect
2222
github.com/davecgh/go-spew v1.1.1 // indirect
2323
github.com/getsentry/sentry-go v0.20.0 // indirect
2424
github.com/ghodss/yaml v1.0.0 // indirect
@@ -39,6 +39,7 @@ require (
3939
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
4040
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
4141
github.com/patrickmn/go-cache v0.0.0-20180815053127-5633e0862627 // indirect
42+
github.com/pierrec/lz4/v4 v4.1.15 // indirect
4243
github.com/pmezard/go-difflib v1.0.0 // indirect
4344
github.com/spf13/cast v1.5.0 // indirect
4445
github.com/spf13/cobra v1.6.1 // indirect

‎plugins/source/okta/go.sum‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,12 +46,14 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR
4646
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
4747
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
4848
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
49-
github.com/cloudquery/arrow/go/v13 v13.0.0-20230509053643-898a79b1d3c8 h1:CmgLSEGQNLHpUQ5cU4L4aF7cuJZRnc1toIIWqC1gmPg=
50-
github.com/cloudquery/arrow/go/v13 v13.0.0-20230509053643-898a79b1d3c8/go.mod h1:/XatdE3kDIBqZKhZ7OBUHwP2jaASDFZHqF4puOWM8po=
49+
github.com/cloudquery/arrow/go/v13 v13.0.0-20230525142029-2d32efeedad8 h1:/mgK+To5HNUzuZDWdVygbfn8oaPG5c7I/8qCxwwpWO8=
50+
github.com/cloudquery/arrow/go/v13 v13.0.0-20230525142029-2d32efeedad8/go.mod h1:/XatdE3kDIBqZKhZ7OBUHwP2jaASDFZHqF4puOWM8po=
5151
github.com/cloudquery/plugin-pb-go v1.0.8 h1:wn3GXhcNItcP+6wUUZuzUFbvdL59liKBO37/izMi+FQ=
5252
github.com/cloudquery/plugin-pb-go v1.0.8/go.mod h1:vAGA27psem7ZZNAY4a3S9TKuA/JDQWstjKcHPJX91Mc=
5353
github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U=
5454
github.com/cloudquery/plugin-sdk/v2 v2.7.0/go.mod h1:pAX6ojIW99b/Vg4CkhnsGkRIzNaVEceYMR+Bdit73ug=
55+
github.com/cloudquery/plugin-sdk/v3 v3.6.4 h1:P4OkS5tJYkv3OqeL60DAVqXXbFQUyPKJ5YDtAgjl9b4=
56+
github.com/cloudquery/plugin-sdk/v3 v3.6.4/go.mod h1:3JrZXEULmGXpkOukVaRIzaA63d7TJr9Ukp6hemTjbtc=
5557
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
5658
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
5759
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
@@ -187,6 +189,7 @@ github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFSt
187189
github.com/patrickmn/go-cache v0.0.0-20180815053127-5633e0862627 h1:pSCLCl6joCFRnjpeojzOpEYs4q7Vditq8fySFG5ap3Y=
188190
github.com/patrickmn/go-cache v0.0.0-20180815053127-5633e0862627/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
189191
github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
192+
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
190193
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
191194
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
192195
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=

‎plugins/source/okta/main.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ package main
22

33
import (
44
"github.com/cloudquery/cloudquery/plugins/source/okta/resources/plugin"
5-
"github.com/cloudquery/plugin-sdk/v2/serve"
5+
"github.com/cloudquery/plugin-sdk/v3/serve"
66
)
77

88
const sentryDSN = "https://[email protected]/6747629"

‎plugins/source/okta/resources/plugin/plugin.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@ import (
55
"github.com/cloudquery/cloudquery/plugins/source/okta/resources/services/applications"
66
"github.com/cloudquery/cloudquery/plugins/source/okta/resources/services/groups"
77
"github.com/cloudquery/cloudquery/plugins/source/okta/resources/services/users"
8-
"github.com/cloudquery/plugin-sdk/v2/plugins/source"
9-
"github.com/cloudquery/plugin-sdk/v2/schema"
8+
"github.com/cloudquery/plugin-sdk/v3/plugins/source"
9+
"github.com/cloudquery/plugin-sdk/v3/schema"
1010
)
1111

1212
var Version = "Development"

‎plugins/source/okta/resources/services/applications/applications.go‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,9 @@ package applications
33
import (
44
"context"
55

6+
"github.com/apache/arrow/go/v13/arrow"
67
"github.com/cloudquery/cloudquery/plugins/source/okta/client"
7-
"github.com/cloudquery/plugin-sdk/v2/schema"
8+
"github.com/cloudquery/plugin-sdk/v3/schema"
89
"github.com/okta/okta-sdk-golang/v3/okta"
910
)
1011

@@ -94,10 +95,8 @@ func appToApp(obj *okta.ListApplications200ResponseInner) *okta.Application {
9495
}
9596

9697
var appIDColumn = schema.Column{
97-
Name: "app_id",
98-
Type: schema.TypeString,
99-
Resolver: schema.ParentColumnResolver("id"),
100-
CreationOptions: schema.ColumnCreationOptions{
101-
PrimaryKey: true,
102-
},
98+
Name: "app_id",
99+
Type: arrow.BinaryTypes.String,
100+
Resolver: schema.ParentColumnResolver("id"),
101+
PrimaryKey: true,
103102
}

‎plugins/source/okta/resources/services/applications/applications_mock_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import (
66
"testing"
77

88
"github.com/cloudquery/cloudquery/plugins/source/okta/client"
9-
"github.com/cloudquery/plugin-sdk/v2/faker"
9+
"github.com/cloudquery/plugin-sdk/v3/faker"
1010
"github.com/gorilla/mux"
1111
"github.com/okta/okta-sdk-golang/v3/okta"
1212
)

‎plugins/source/okta/resources/services/applications/group_assignments.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import (
44
"context"
55

66
"github.com/cloudquery/cloudquery/plugins/source/okta/client"
7-
"github.com/cloudquery/plugin-sdk/v2/schema"
7+
"github.com/cloudquery/plugin-sdk/v3/schema"
88
"github.com/okta/okta-sdk-golang/v3/okta"
99
)
1010

0 commit comments

Comments
 (0)