Skip to content

Commit be60546

Browse files
disqdisq
authored
feat(firestore)!: Migrate to Arrow native SDK (#11010)
Closes #10742 BEGIN_COMMIT_OVERRIDE feat: Update to use [Apache Arrow](https://arrow.apache.org/) type system (#11010) BREAKING-CHANGE: This release introduces an internal change to our type system to use [Apache Arrow](https://arrow.apache.org/). This should not have any visible breaking changes, however due to the size of the change we are introducing it under a major version bump to communicate that it might have some bugs that we weren't able to catch during our internal tests. If you encounter an issue during the upgrade, please submit a [bug report](https://github.com/cloudquery/cloudquery/issues/new/choose). You will also need to update destinations depending on which one you use: - Azure Blob Storage >= v3.2.0 - BigQuery >= v3.0.0 - ClickHouse >= v3.1.1 - DuckDB >= v1.1.6 - Elasticsearch >= v2.0.0 - File >= v3.2.0 - Firehose >= v2.0.2 - GCS >= v3.2.0 - Gremlin >= v2.1.10 - Kafka >= v3.0.1 - Meilisearch >= v2.0.1 - Microsoft SQL Server >= v4.2.0 - MongoDB >= v2.0.1 - MySQL >= v2.0.2 - Neo4j >= v3.0.0 - PostgreSQL >= v4.2.0 - S3 >= v4.4.0 - Snowflake >= v2.1.1 - SQLite >= v2.2.0 END_COMMIT_OVERRIDE
1 parent b00e212 commit be60546

File tree

8 files changed

+28
-24
lines changed

8 files changed

+28
-24
lines changed

‎plugins/source/firestore/client/client.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@ import (
66

77
"cloud.google.com/go/firestore"
88
"github.com/cloudquery/plugin-pb-go/specs"
9-
"github.com/cloudquery/plugin-sdk/v2/plugins/source"
10-
"github.com/cloudquery/plugin-sdk/v2/schema"
9+
"github.com/cloudquery/plugin-sdk/v3/plugins/source"
10+
"github.com/cloudquery/plugin-sdk/v3/schema"
1111
"github.com/rs/zerolog"
1212
"google.golang.org/api/option"
1313
)

‎plugins/source/firestore/client/list_tables.go‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,9 @@ import (
44
"context"
55

66
"cloud.google.com/go/firestore"
7-
"github.com/cloudquery/plugin-sdk/v2/schema"
7+
"github.com/apache/arrow/go/v13/arrow"
8+
"github.com/cloudquery/plugin-sdk/v3/schema"
9+
"github.com/cloudquery/plugin-sdk/v3/types"
810
"google.golang.org/api/iterator"
911
)
1012

@@ -29,25 +31,23 @@ func (*Client) listTables(ctx context.Context, client *firestore.Client) (schema
2931
}
3032
columns := make(schema.ColumnList, 0, 4)
3133
columns = append(columns, schema.Column{
32-
Name: "__id",
33-
Type: schema.TypeString,
34-
CreationOptions: schema.ColumnCreationOptions{
35-
PrimaryKey: true,
36-
NotNull: true,
37-
Unique: true,
38-
},
34+
Name: "__id",
35+
Type: arrow.BinaryTypes.String,
36+
PrimaryKey: true,
37+
Unique: true,
38+
NotNull: true,
3939
})
4040
columns = append(columns, schema.Column{
4141
Name: "__created_at",
42-
Type: schema.TypeTimestamp,
42+
Type: arrow.FixedWidthTypes.Timestamp_us,
4343
})
4444
columns = append(columns, schema.Column{
4545
Name: "__updated_at",
46-
Type: schema.TypeTimestamp,
46+
Type: arrow.FixedWidthTypes.Timestamp_us,
4747
})
4848
columns = append(columns, schema.Column{
4949
Name: "data",
50-
Type: schema.TypeJSON,
50+
Type: types.ExtensionTypes.JSON,
5151
})
5252

5353
schemaTables = append(schemaTables, &schema.Table{

‎plugins/source/firestore/client/sync.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@ import (
55
"time"
66

77
"cloud.google.com/go/firestore"
8-
"github.com/cloudquery/plugin-sdk/v2/plugins/source"
9-
"github.com/cloudquery/plugin-sdk/v2/schema"
8+
"github.com/cloudquery/plugin-sdk/v3/plugins/source"
9+
"github.com/cloudquery/plugin-sdk/v3/schema"
1010
"golang.org/x/sync/errgroup"
1111
"google.golang.org/api/iterator"
1212
)

‎plugins/source/firestore/go.mod‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,25 +4,25 @@ go 1.19
44

55
require (
66
cloud.google.com/go/firestore v1.9.0
7+
github.com/apache/arrow/go/v13 v13.0.0-20230509040948-de6c3cd2b604
78
github.com/cloudquery/plugin-pb-go v1.0.8
8-
github.com/cloudquery/plugin-sdk/v2 v2.7.0
9+
github.com/cloudquery/plugin-sdk/v3 v3.6.4
910
github.com/rs/zerolog v1.29.0
1011
github.com/stretchr/testify v1.8.2
1112
golang.org/x/sync v0.1.0
1213
google.golang.org/api v0.114.0
1314
)
1415

15-
// TODO: remove once all updates are merged
16-
replace github.com/apache/arrow/go/v13 => github.com/cloudquery/arrow/go/v13 v13.0.0-20230509053643-898a79b1d3c8
16+
replace github.com/apache/arrow/go/v13 => github.com/cloudquery/arrow/go/v13 v13.0.0-20230525142029-2d32efeedad8
1717

1818
require (
1919
cloud.google.com/go v0.110.0 // indirect
2020
cloud.google.com/go/compute v1.19.0 // indirect
2121
cloud.google.com/go/compute/metadata v0.2.3 // indirect
2222
cloud.google.com/go/longrunning v0.4.1 // indirect
2323
github.com/andybalholm/brotli v1.0.5 // indirect
24-
github.com/apache/arrow/go/v13 v13.0.0-20230509040948-de6c3cd2b604 // indirect
2524
github.com/apache/thrift v0.16.0 // indirect
25+
github.com/cloudquery/plugin-sdk/v2 v2.7.0 // indirect
2626
github.com/davecgh/go-spew v1.1.1 // indirect
2727
github.com/getsentry/sentry-go v0.20.0 // indirect
2828
github.com/ghodss/yaml v1.0.0 // indirect
@@ -46,6 +46,7 @@ require (
4646
github.com/mattn/go-isatty v0.0.18 // indirect
4747
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
4848
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
49+
github.com/pierrec/lz4/v4 v4.1.15 // indirect
4950
github.com/pmezard/go-difflib v1.0.0 // indirect
5051
github.com/spf13/cast v1.5.0 // indirect
5152
github.com/spf13/cobra v1.6.1 // indirect

‎plugins/source/firestore/go.sum‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,12 +54,14 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR
5454
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
5555
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
5656
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
57-
github.com/cloudquery/arrow/go/v13 v13.0.0-20230509053643-898a79b1d3c8 h1:CmgLSEGQNLHpUQ5cU4L4aF7cuJZRnc1toIIWqC1gmPg=
58-
github.com/cloudquery/arrow/go/v13 v13.0.0-20230509053643-898a79b1d3c8/go.mod h1:/XatdE3kDIBqZKhZ7OBUHwP2jaASDFZHqF4puOWM8po=
57+
github.com/cloudquery/arrow/go/v13 v13.0.0-20230525142029-2d32efeedad8 h1:/mgK+To5HNUzuZDWdVygbfn8oaPG5c7I/8qCxwwpWO8=
58+
github.com/cloudquery/arrow/go/v13 v13.0.0-20230525142029-2d32efeedad8/go.mod h1:/XatdE3kDIBqZKhZ7OBUHwP2jaASDFZHqF4puOWM8po=
5959
github.com/cloudquery/plugin-pb-go v1.0.8 h1:wn3GXhcNItcP+6wUUZuzUFbvdL59liKBO37/izMi+FQ=
6060
github.com/cloudquery/plugin-pb-go v1.0.8/go.mod h1:vAGA27psem7ZZNAY4a3S9TKuA/JDQWstjKcHPJX91Mc=
6161
github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U=
6262
github.com/cloudquery/plugin-sdk/v2 v2.7.0/go.mod h1:pAX6ojIW99b/Vg4CkhnsGkRIzNaVEceYMR+Bdit73ug=
63+
github.com/cloudquery/plugin-sdk/v3 v3.6.4 h1:P4OkS5tJYkv3OqeL60DAVqXXbFQUyPKJ5YDtAgjl9b4=
64+
github.com/cloudquery/plugin-sdk/v3 v3.6.4/go.mod h1:3JrZXEULmGXpkOukVaRIzaA63d7TJr9Ukp6hemTjbtc=
6365
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
6466
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
6567
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
@@ -195,6 +197,7 @@ github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8D
195197
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE=
196198
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
197199
github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
200+
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
198201
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
199202
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
200203
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

‎plugins/source/firestore/main.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ package main
22

33
import (
44
"github.com/cloudquery/cloudquery/plugins/source/firestore/resources/plugin"
5-
"github.com/cloudquery/plugin-sdk/v2/serve"
5+
"github.com/cloudquery/plugin-sdk/v3/serve"
66
)
77

88
const sentryDSN = "https://[email protected]/4505007730917376"

‎plugins/source/firestore/resources/plugin/plugin.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ package plugin
22

33
import (
44
"github.com/cloudquery/cloudquery/plugins/source/firestore/client"
5-
"github.com/cloudquery/plugin-sdk/v2/plugins/source"
5+
"github.com/cloudquery/plugin-sdk/v3/plugins/source"
66
)
77

88
var version = "development"

‎plugins/source/firestore/resources/plugin/tables.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import (
44
"context"
55

66
"github.com/cloudquery/cloudquery/plugins/source/firestore/client"
7-
"github.com/cloudquery/plugin-sdk/v2/schema"
7+
"github.com/cloudquery/plugin-sdk/v3/schema"
88
)
99

1010
func getDynamicTables(ctx context.Context, c schema.ClientMeta) (schema.Tables, error) {

0 commit comments

Comments
 (0)