Skip to content

Commit 133d82e

Browse files
yevgenypatsyevgenypats
authored
feat(oracle)!: Update to SDK V3 Arrow native (#11037)
Closes #10759 BEGIN_COMMIT_OVERRIDE feat: Update to use [Apache Arrow](https://arrow.apache.org/) type system (#11037) BREAKING-CHANGE: This release introduces an internal change to our type system to use [Apache Arrow](https://arrow.apache.org/). This should not have any visible breaking changes, however due to the size of the change we are introducing it under a major version bump to communicate that it might have some bugs that we weren't able to catch during our internal tests. If you encounter an issue during the upgrade, please submit a [bug report](https://github.com/cloudquery/cloudquery/issues/new/choose). You will also need to update destinations depending on which one you use: Azure Blob Storage >= v3.2.0 BigQuery >= v3.0.0 ClickHouse >= v3.1.1 DuckDB >= v1.1.6 Elasticsearch >= v2.0.0 File >= v3.2.0 Firehose >= v2.0.2 GCS >= v3.2.0 Gremlin >= v2.1.10 Kafka >= v3.0.1 Meilisearch >= v2.0.1 Microsoft SQL Server >= v4.2.0 MongoDB >= v2.0.1 MySQL >= v2.0.2 Neo4j >= v3.0.0 PostgreSQL >= v4.2.0 S3 >= v4.4.0 Snowflake >= v2.1.1 SQLite >= v2.2.0 END_COMMIT_OVERRIDE
1 parent 463c535 commit 133d82e

File tree

244 files changed

+2895
-3264
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

244 files changed

+2895
-3264
lines changed

‎plugins/source/oracle/client/client.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@ import (
66
"strings"
77

88
"github.com/cloudquery/plugin-pb-go/specs"
9-
"github.com/cloudquery/plugin-sdk/v2/plugins/source"
10-
"github.com/cloudquery/plugin-sdk/v2/schema"
9+
"github.com/cloudquery/plugin-sdk/v3/plugins/source"
10+
"github.com/cloudquery/plugin-sdk/v3/schema"
1111
"github.com/oracle/oci-go-sdk/v65/common"
1212
"github.com/oracle/oci-go-sdk/v65/identity"
1313
"github.com/oracle/oci-go-sdk/v65/objectstorage"

‎plugins/source/oracle/client/multiplex.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
package client
22

3-
import "github.com/cloudquery/plugin-sdk/v2/schema"
3+
import "github.com/cloudquery/plugin-sdk/v3/schema"
44

55
func RegionCompartmentMultiplex(meta schema.ClientMeta) []schema.ClientMeta {
66
cqClient := meta.(*Client)

‎plugins/source/oracle/client/resovers.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ package client
33
import (
44
"context"
55

6-
"github.com/cloudquery/plugin-sdk/v2/schema"
6+
"github.com/cloudquery/plugin-sdk/v3/schema"
77
)
88

99
// grabs the region from the cq-client, not from the resource.

‎plugins/source/oracle/client/transform.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,20 +3,20 @@ package client
33
import (
44
"reflect"
55

6-
"github.com/cloudquery/plugin-sdk/v2/schema"
7-
"github.com/cloudquery/plugin-sdk/v2/transformers"
6+
"github.com/apache/arrow/go/v13/arrow"
7+
"github.com/cloudquery/plugin-sdk/v3/transformers"
88
"github.com/oracle/oci-go-sdk/v65/common"
99
)
1010

11-
func OracleTypeTransformer(field reflect.StructField) (schema.ValueType, error) {
11+
func OracleTypeTransformer(field reflect.StructField) (arrow.DataType, error) {
1212
fieldType := field.Type
1313

1414
if fieldType.Kind() == reflect.Ptr {
1515
fieldType = fieldType.Elem()
1616
}
1717

1818
if fieldType.Kind() == reflect.Struct && fieldType == reflect.TypeOf(common.SDKTime{}) {
19-
return schema.TypeTimestamp, nil
19+
return arrow.FixedWidthTypes.Timestamp_us, nil
2020
}
2121

2222
return transformers.DefaultTypeTransformer(field)

‎plugins/source/oracle/codegen/templates/table.go.tpl‎

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -17,27 +17,21 @@ func {{.SubService | ToCamel}}() *schema.Table {
1717
Columns: []schema.Column{
1818
{
1919
Name: "region",
20-
Type: schema.TypeString,
20+
Type: arrow.BinaryTypes.String,
2121
Resolver: client.ResolveOracleRegion,
22-
CreationOptions: schema.ColumnCreationOptions{
23-
PrimaryKey: true,
24-
},
22+
PrimaryKey: true,
2523
},
2624
{
2725
Name: "compartment_id",
28-
Type: schema.TypeString,
26+
Type: arrow.BinaryTypes.String,
2927
Resolver: client.ResolveCompartmentId,
30-
CreationOptions: schema.ColumnCreationOptions{
31-
PrimaryKey: true,
32-
},
28+
PrimaryKey: true,
3329
},
3430
{
3531
Name: "id",
36-
Type: schema.TypeString,
32+
Type: arrow.BinaryTypes.String,
3733
Resolver: schema.PathResolver("Id"),
38-
CreationOptions: schema.ColumnCreationOptions{
39-
PrimaryKey: true,
40-
},
34+
PrimaryKey: true,
4135
},
4236
},
4337
}

‎plugins/source/oracle/go.mod‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,9 @@ module github.com/cloudquery/cloudquery/plugins/source/oracle
33
go 1.19
44

55
require (
6+
github.com/apache/arrow/go/v13 v13.0.0-20230509040948-de6c3cd2b604
67
github.com/cloudquery/plugin-pb-go v1.0.8
7-
github.com/cloudquery/plugin-sdk/v2 v2.7.0
8+
github.com/cloudquery/plugin-sdk/v3 v3.6.7
89
github.com/iancoleman/strcase v0.2.0
910
github.com/oracle/oci-go-sdk/v65 v65.28.3
1011
github.com/rs/zerolog v1.29.0
@@ -16,8 +17,8 @@ replace github.com/apache/arrow/go/v13 => github.com/cloudquery/arrow/go/v13 v13
1617

1718
require (
1819
github.com/andybalholm/brotli v1.0.5 // indirect
19-
github.com/apache/arrow/go/v13 v13.0.0-20230509040948-de6c3cd2b604 // indirect
2020
github.com/apache/thrift v0.16.0 // indirect
21+
github.com/cloudquery/plugin-sdk/v2 v2.7.0 // indirect
2122
github.com/davecgh/go-spew v1.1.1 // indirect
2223
github.com/getsentry/sentry-go v0.20.0 // indirect
2324
github.com/ghodss/yaml v1.0.0 // indirect
@@ -37,6 +38,7 @@ require (
3738
github.com/mattn/go-isatty v0.0.18 // indirect
3839
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
3940
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
41+
github.com/pierrec/lz4/v4 v4.1.15 // indirect
4042
github.com/pmezard/go-difflib v1.0.0 // indirect
4143
github.com/sony/gobreaker v0.5.0 // indirect
4244
github.com/spf13/cast v1.5.0 // indirect

‎plugins/source/oracle/go.sum‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,8 @@ github.com/cloudquery/plugin-pb-go v1.0.8 h1:wn3GXhcNItcP+6wUUZuzUFbvdL59liKBO37
5050
github.com/cloudquery/plugin-pb-go v1.0.8/go.mod h1:vAGA27psem7ZZNAY4a3S9TKuA/JDQWstjKcHPJX91Mc=
5151
github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U=
5252
github.com/cloudquery/plugin-sdk/v2 v2.7.0/go.mod h1:pAX6ojIW99b/Vg4CkhnsGkRIzNaVEceYMR+Bdit73ug=
53+
github.com/cloudquery/plugin-sdk/v3 v3.6.7 h1:QJqZGHs+3uN+CE3y9oBOjkchH/v5XisRnVP2X4aj9Wo=
54+
github.com/cloudquery/plugin-sdk/v3 v3.6.7/go.mod h1:+ta6OETfGfzh6nCpCyZi5Er1rj+zvn7m2QR2wokEvA8=
5355
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
5456
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
5557
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
@@ -180,6 +182,7 @@ github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFSt
180182
github.com/oracle/oci-go-sdk/v65 v65.28.3 h1:LOuUI2gf0Q2ygrWFBjE14dWYExvZu/Wq5IuXE2vx56s=
181183
github.com/oracle/oci-go-sdk/v65 v65.28.3/go.mod h1:oyMrMa1vOzzKTmPN+kqrTR9y9kPA2tU1igN3NUSNTIE=
182184
github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
185+
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
183186
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
184187
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
185188
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=

‎plugins/source/oracle/main.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ package main
22

33
import (
44
"github.com/cloudquery/cloudquery/plugins/source/oracle/resources/plugin"
5-
"github.com/cloudquery/plugin-sdk/v2/serve"
5+
"github.com/cloudquery/plugin-sdk/v3/serve"
66
)
77

88
const sentryDSN = "https://[email protected]/4504474234650624"

‎plugins/source/oracle/resources/plugin/autogen_tables.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import (
66
"github.com/cloudquery/cloudquery/plugins/source/oracle/resources/services/networkfirewall"
77
"github.com/cloudquery/cloudquery/plugins/source/oracle/resources/services/networkloadbalancer"
88
"github.com/cloudquery/cloudquery/plugins/source/oracle/resources/services/virtualnetwork"
9-
"github.com/cloudquery/plugin-sdk/v2/schema"
9+
"github.com/cloudquery/plugin-sdk/v3/schema"
1010
)
1111

1212
func AutogenTables() []*schema.Table {

‎plugins/source/oracle/resources/plugin/plugin.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,9 @@ import (
99
"github.com/cloudquery/cloudquery/plugins/source/oracle/resources/services/filestorage"
1010
"github.com/cloudquery/cloudquery/plugins/source/oracle/resources/services/identity"
1111
"github.com/cloudquery/cloudquery/plugins/source/oracle/resources/services/objectstorage"
12-
"github.com/cloudquery/plugin-sdk/v2/caser"
13-
"github.com/cloudquery/plugin-sdk/v2/plugins/source"
14-
"github.com/cloudquery/plugin-sdk/v2/schema"
12+
"github.com/cloudquery/plugin-sdk/v3/caser"
13+
"github.com/cloudquery/plugin-sdk/v3/plugins/source"
14+
"github.com/cloudquery/plugin-sdk/v3/schema"
1515
)
1616

1717
var (

0 commit comments

Comments
 (0)