Skip to content

Commit 4b9971e

Browse files
disqdisq
authored
feat(terraform)!: Migrate to Arrow native SDK (#10995)
Closes #10770 BEGIN_COMMIT_OVERRIDE feat: Update to use [Apache Arrow](https://arrow.apache.org/) type system (#10984) BREAKING-CHANGE: This release introduces an internal change to our type system to use [Apache Arrow](https://arrow.apache.org/). This should not have any visible breaking changes, however due to the size of the change we are introducing it under a major version bump to communicate that it might have some bugs that we weren't able to catch during our internal tests. If you encounter an issue during the upgrade, please submit a [bug report](https://github.com/cloudquery/cloudquery/issues/new/choose). You will also need to update destinations depending on which one you use: - Azure Blob Storage >= v3.2.0 - BigQuery >= v3.0.0 - ClickHouse >= v3.1.1 - DuckDB >= v1.1.6 - Elasticsearch >= v2.0.0 - File >= v3.2.0 - Firehose >= v2.0.2 - GCS >= v3.2.0 - Gremlin >= v2.1.10 - Kafka >= v3.0.1 - Meilisearch >= v2.0.1 - Microsoft SQL Server >= v4.2.0 - MongoDB >= v2.0.1 - MySQL >= v2.0.2 - Neo4j >= v3.0.0 - PostgreSQL >= v4.2.0 - S3 >= v4.4.0 - Snowflake >= v2.1.1 - SQLite >= v2.2.0 END_COMMIT_OVERRIDE
1 parent f07c4da commit 4b9971e

File tree

14 files changed

+77
-69
lines changed

14 files changed

+77
-69
lines changed

‎plugins/source/terraform/client/client.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@ import (
66
"fmt"
77

88
"github.com/cloudquery/plugin-pb-go/specs"
9-
"github.com/cloudquery/plugin-sdk/v2/plugins/source"
10-
"github.com/cloudquery/plugin-sdk/v2/schema"
9+
"github.com/cloudquery/plugin-sdk/v3/plugins/source"
10+
"github.com/cloudquery/plugin-sdk/v3/schema"
1111
"github.com/rs/zerolog"
1212
)
1313

‎plugins/source/terraform/client/multiplexers.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
package client
22

3-
import "github.com/cloudquery/plugin-sdk/v2/schema"
3+
import "github.com/cloudquery/plugin-sdk/v3/schema"
44

55
// BackendMultiplex add the ability to handle multiple backends in the same fetch
66
func BackendMultiplex(meta schema.ClientMeta) []schema.ClientMeta {

‎plugins/source/terraform/client/testing.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,8 @@ import (
88
"time"
99

1010
"github.com/cloudquery/plugin-pb-go/specs"
11-
"github.com/cloudquery/plugin-sdk/v2/plugins/source"
12-
"github.com/cloudquery/plugin-sdk/v2/schema"
11+
"github.com/cloudquery/plugin-sdk/v3/plugins/source"
12+
"github.com/cloudquery/plugin-sdk/v3/schema"
1313
"github.com/golang/mock/gomock"
1414
"github.com/rs/zerolog"
1515
)

‎plugins/source/terraform/go.mod‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,24 +3,23 @@ module github.com/cloudquery/cloudquery/plugins/source/terraform
33
go 1.19
44

55
require (
6+
github.com/apache/arrow/go/v13 v13.0.0-20230509040948-de6c3cd2b604
67
github.com/aws/aws-sdk-go-v2 v1.18.0
78
github.com/aws/aws-sdk-go-v2/config v1.18.25
89
github.com/aws/aws-sdk-go-v2/credentials v1.13.24
910
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.67
1011
github.com/aws/aws-sdk-go-v2/service/s3 v1.33.1
1112
github.com/aws/aws-sdk-go-v2/service/sts v1.19.0
1213
github.com/cloudquery/plugin-pb-go v1.0.8
13-
github.com/cloudquery/plugin-sdk/v2 v2.7.0
14+
github.com/cloudquery/plugin-sdk/v3 v3.6.4
1415
github.com/golang/mock v1.6.0
1516
github.com/rs/zerolog v1.29.0
1617
)
1718

18-
// TODO: remove once all updates are merged
19-
replace github.com/apache/arrow/go/v13 => github.com/cloudquery/arrow/go/v13 v13.0.0-20230509053643-898a79b1d3c8
19+
replace github.com/apache/arrow/go/v13 => github.com/cloudquery/arrow/go/v13 v13.0.0-20230525142029-2d32efeedad8
2020

2121
require (
2222
github.com/andybalholm/brotli v1.0.5 // indirect
23-
github.com/apache/arrow/go/v13 v13.0.0-20230509040948-de6c3cd2b604 // indirect
2423
github.com/apache/thrift v0.16.0 // indirect
2524
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
2625
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3 // indirect
@@ -35,6 +34,7 @@ require (
3534
github.com/aws/aws-sdk-go-v2/service/sso v1.12.10 // indirect
3635
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.10 // indirect
3736
github.com/aws/smithy-go v1.13.5 // indirect
37+
github.com/cloudquery/plugin-sdk/v2 v2.7.0 // indirect
3838
github.com/davecgh/go-spew v1.1.1 // indirect
3939
github.com/getsentry/sentry-go v0.20.0 // indirect
4040
github.com/ghodss/yaml v1.0.0 // indirect
@@ -54,6 +54,7 @@ require (
5454
github.com/mattn/go-isatty v0.0.18 // indirect
5555
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
5656
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
57+
github.com/pierrec/lz4/v4 v4.1.15 // indirect
5758
github.com/pmezard/go-difflib v1.0.0 // indirect
5859
github.com/spf13/cast v1.5.0 // indirect
5960
github.com/spf13/cobra v1.6.1 // indirect

‎plugins/source/terraform/go.sum‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -82,12 +82,14 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR
8282
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
8383
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
8484
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
85-
github.com/cloudquery/arrow/go/v13 v13.0.0-20230509053643-898a79b1d3c8 h1:CmgLSEGQNLHpUQ5cU4L4aF7cuJZRnc1toIIWqC1gmPg=
86-
github.com/cloudquery/arrow/go/v13 v13.0.0-20230509053643-898a79b1d3c8/go.mod h1:/XatdE3kDIBqZKhZ7OBUHwP2jaASDFZHqF4puOWM8po=
85+
github.com/cloudquery/arrow/go/v13 v13.0.0-20230525142029-2d32efeedad8 h1:/mgK+To5HNUzuZDWdVygbfn8oaPG5c7I/8qCxwwpWO8=
86+
github.com/cloudquery/arrow/go/v13 v13.0.0-20230525142029-2d32efeedad8/go.mod h1:/XatdE3kDIBqZKhZ7OBUHwP2jaASDFZHqF4puOWM8po=
8787
github.com/cloudquery/plugin-pb-go v1.0.8 h1:wn3GXhcNItcP+6wUUZuzUFbvdL59liKBO37/izMi+FQ=
8888
github.com/cloudquery/plugin-pb-go v1.0.8/go.mod h1:vAGA27psem7ZZNAY4a3S9TKuA/JDQWstjKcHPJX91Mc=
8989
github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U=
9090
github.com/cloudquery/plugin-sdk/v2 v2.7.0/go.mod h1:pAX6ojIW99b/Vg4CkhnsGkRIzNaVEceYMR+Bdit73ug=
91+
github.com/cloudquery/plugin-sdk/v3 v3.6.4 h1:P4OkS5tJYkv3OqeL60DAVqXXbFQUyPKJ5YDtAgjl9b4=
92+
github.com/cloudquery/plugin-sdk/v3 v3.6.4/go.mod h1:3JrZXEULmGXpkOukVaRIzaA63d7TJr9Ukp6hemTjbtc=
9193
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
9294
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
9395
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
@@ -219,6 +221,7 @@ github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8D
219221
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE=
220222
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
221223
github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
224+
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
222225
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
223226
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
224227
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=

‎plugins/source/terraform/main.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ package main
22

33
import (
44
"github.com/cloudquery/cloudquery/plugins/source/terraform/resources/plugin"
5-
"github.com/cloudquery/plugin-sdk/v2/serve"
5+
"github.com/cloudquery/plugin-sdk/v3/serve"
66
)
77

88
const sentryDSN = "https://[email protected]/6747630"

‎plugins/source/terraform/resources/plugin/plugin.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@ package plugin
33
import (
44
"github.com/cloudquery/cloudquery/plugins/source/terraform/client"
55
"github.com/cloudquery/cloudquery/plugins/source/terraform/resources/services"
6-
"github.com/cloudquery/plugin-sdk/v2/plugins/source"
7-
"github.com/cloudquery/plugin-sdk/v2/schema"
6+
"github.com/cloudquery/plugin-sdk/v3/plugins/source"
7+
"github.com/cloudquery/plugin-sdk/v3/schema"
88
)
99

1010
var (

‎plugins/source/terraform/resources/services/tf_resource_instances.go‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,10 @@ import (
55
"encoding/json"
66
"fmt"
77

8+
"github.com/apache/arrow/go/v13/arrow"
89
"github.com/cloudquery/cloudquery/plugins/source/terraform/client"
9-
"github.com/cloudquery/plugin-sdk/v2/schema"
10+
"github.com/cloudquery/plugin-sdk/v3/schema"
11+
"github.com/cloudquery/plugin-sdk/v3/types"
1012
)
1113

1214
func TFResourceInstances() *schema.Table {
@@ -17,35 +19,35 @@ func TFResourceInstances() *schema.Table {
1719
Columns: []schema.Column{
1820
{
1921
Name: "resource_name",
20-
Type: schema.TypeString,
22+
Type: arrow.BinaryTypes.String,
2123
Resolver: schema.ParentColumnResolver("name"),
2224
},
2325
{
2426
Name: "instance_id",
2527
Description: "Instance id",
26-
Type: schema.TypeString,
28+
Type: arrow.BinaryTypes.String,
2729
Resolver: resolveInstanceInternalId,
2830
},
2931
{
3032
Name: "schema_version",
3133
Description: "Terraform schema version",
32-
Type: schema.TypeInt,
34+
Type: arrow.PrimitiveTypes.Int64,
3335
},
3436
{
3537
Name: "attributes",
3638
Description: "Instance attributes",
37-
Type: schema.TypeJSON,
39+
Type: types.ExtensionTypes.JSON,
3840
Resolver: resolveInstanceAttributes,
3941
},
4042
{
4143
Name: "dependencies",
4244
Description: "Instance dependencies array",
43-
Type: schema.TypeStringArray,
45+
Type: arrow.ListOf(arrow.BinaryTypes.String),
4446
},
4547
{
4648
Name: "create_before_destroy",
4749
Description: "Should resource should be created before destroying",
48-
Type: schema.TypeBool,
50+
Type: arrow.FixedWidthTypes.Boolean,
4951
},
5052
},
5153
}

‎plugins/source/terraform/resources/services/tf_resources.go‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,9 @@ package services
33
import (
44
"context"
55

6+
"github.com/apache/arrow/go/v13/arrow"
67
"github.com/cloudquery/cloudquery/plugins/source/terraform/client"
7-
"github.com/cloudquery/plugin-sdk/v2/schema"
8+
"github.com/cloudquery/plugin-sdk/v3/schema"
89
)
910

1011
func TFResources() *schema.Table {
@@ -15,39 +16,39 @@ func TFResources() *schema.Table {
1516
Columns: []schema.Column{
1617
{
1718
Name: "data_backend_name",
18-
Type: schema.TypeString,
19+
Type: arrow.BinaryTypes.String,
1920
Resolver: schema.ParentColumnResolver("backend_name"),
2021
},
2122
{
2223
Name: "module",
2324
Description: "Resource module if exists",
24-
Type: schema.TypeString,
25+
Type: arrow.BinaryTypes.String,
2526
},
2627
{
2728
Name: "mode",
2829
Description: "Resource mode, for example: data, managed, etc",
29-
Type: schema.TypeString,
30+
Type: arrow.BinaryTypes.String,
3031
},
3132
{
3233
Name: "type",
3334
Description: "Resource type",
34-
Type: schema.TypeString,
35+
Type: arrow.BinaryTypes.String,
3536
},
3637
{
3738
Name: "name",
3839
Description: "Resource name",
39-
Type: schema.TypeString,
40+
Type: arrow.BinaryTypes.String,
4041
},
4142
{
4243
Name: "provider_path",
4344
Description: "Resource provider full path, for example: provider[\"registry.terraform.io/hashicorp/aws\"]",
44-
Type: schema.TypeString,
45+
Type: arrow.BinaryTypes.String,
4546
Resolver: schema.PathResolver("ProviderConfig"),
4647
},
4748
{
4849
Name: "provider",
4950
Description: "Resource provider name, for example: aws, gcp, etc",
50-
Type: schema.TypeString,
51+
Type: arrow.BinaryTypes.String,
5152
Resolver: resolveProviderName,
5253
},
5354
},

‎plugins/source/terraform/resources/services/tfdata.go‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,9 @@ import (
44
"context"
55
"regexp"
66

7+
"github.com/apache/arrow/go/v13/arrow"
78
"github.com/cloudquery/cloudquery/plugins/source/terraform/client"
8-
"github.com/cloudquery/plugin-sdk/v2/schema"
9+
"github.com/cloudquery/plugin-sdk/v3/schema"
910
)
1011

1112
var providerNameRegex = regexp.MustCompile(`^.*\["(?P<Hostname>.*)/(?P<Namespace>.*)/(?P<Type>.*)"\].*?$`)
@@ -20,33 +21,33 @@ func TFData() *schema.Table {
2021
{
2122
Name: "backend_type",
2223
Description: "Terraform backend type",
23-
Type: schema.TypeString,
24+
Type: arrow.BinaryTypes.String,
2425
Resolver: resolveBackendType,
2526
},
2627
{
2728
Name: "backend_name",
28-
Type: schema.TypeString,
29+
Type: arrow.BinaryTypes.String,
2930
Description: "Terraform backend name",
3031
Resolver: resolveBackendName,
3132
},
3233
{
3334
Name: "version",
34-
Type: schema.TypeInt,
35+
Type: arrow.PrimitiveTypes.Int64,
3536
Description: "Terraform backend version",
3637
},
3738
{
3839
Name: "terraform_version",
39-
Type: schema.TypeString,
40+
Type: arrow.BinaryTypes.String,
4041
Description: "Terraform version",
4142
},
4243
{
4344
Name: "serial",
44-
Type: schema.TypeInt,
45+
Type: arrow.PrimitiveTypes.Int64,
4546
Description: "Incremental number which describe the state version",
4647
},
4748
{
4849
Name: "lineage",
49-
Type: schema.TypeString,
50+
Type: arrow.BinaryTypes.String,
5051
Description: "The \"lineage\" is a unique ID assigned to a state when it is created",
5152
},
5253
},

0 commit comments

Comments
 (0)