Skip to content

Commit e3925ab

Browse files
erezrokaherezrokah
authored
fix(deps): Update module github.com/cloudquery/plugin-sdk/v3 to v3.10.3 (#11221)
 #### Summary Extracted from #11150 as BigQuery failed on decimal types and maps. I added support for decimals as it was not too much work, but skipped maps for now. See https://github.com/cloudquery/cloudquery/actions/runs/5176150317/jobs/9324607901. Please note the support for decimals is not a breaking change as before BigQuery panicked on those <!--
1 parent c0b9f0a commit e3925ab

File tree

5 files changed

+41
-6
lines changed

5 files changed

+41
-6
lines changed

‎plugins/destination/bigquery/client/client_test.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,5 +41,6 @@ func TestPlugin(t *testing.T) {
4141
MigrateStrategyAppend: migrateStrategy,
4242
},
4343
destination.WithTestIgnoreNullsInLists(),
44+
destination.WithTestSourceSkipMaps(),
4445
)
4546
}

‎plugins/destination/bigquery/client/read.go‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,16 @@ package client
33
import (
44
"context"
55
"fmt"
6+
"math/big"
67
"strings"
78
"time"
89

910
"cloud.google.com/go/bigquery"
1011
"cloud.google.com/go/civil"
1112
"github.com/apache/arrow/go/v13/arrow"
1213
"github.com/apache/arrow/go/v13/arrow/array"
14+
"github.com/apache/arrow/go/v13/arrow/decimal128"
15+
"github.com/apache/arrow/go/v13/arrow/decimal256"
1316
"github.com/apache/arrow/go/v13/arrow/memory"
1417
"github.com/cloudquery/plugin-sdk/v3/schema"
1518
"github.com/cloudquery/plugin-sdk/v3/types"
@@ -58,6 +61,15 @@ func (c *Client) Read(ctx context.Context, table *schema.Table, sourceName strin
5861
return nil
5962
}
6063

64+
func parseRat(v *big.Rat) (str string, precision int32, scale int32) {
65+
str = v.FloatString(10)
66+
parts := strings.Split(str, ".")
67+
if len(parts) == 2 {
68+
return str, int32(len(parts[0]) + len(parts[1])), int32(len(parts[1]))
69+
}
70+
return str, int32(len(parts[0])), 0
71+
}
72+
6173
func appendValue(builder array.Builder, value any) error {
6274
if value == nil {
6375
builder.AppendNull()
@@ -150,6 +162,24 @@ func appendValue(builder array.Builder, value any) error {
150162
bldr.Append(ts)
151163
return nil
152164
}
165+
case *array.Decimal128Builder:
166+
r := value.(*big.Rat)
167+
str, precision, scale := parseRat(r)
168+
dec, err := decimal128.FromString(str, precision, scale)
169+
if err != nil {
170+
return err
171+
}
172+
bldr.Append(dec)
173+
return nil
174+
case *array.Decimal256Builder:
175+
r := value.(*big.Rat)
176+
str, precision, scale := parseRat(r)
177+
dec, err := decimal256.FromString(str, precision, scale)
178+
if err != nil {
179+
return err
180+
}
181+
bldr.Append(dec)
182+
return nil
153183
case *types.JSONBuilder:
154184
return bldr.AppendValueFromString(value.(string))
155185
default:

‎plugins/destination/bigquery/client/types.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,10 @@ func (c *Client) DataTypeToBigQueryType(dataType arrow.DataType) bigquery.FieldT
127127
case typeOneOf(dataType,
128128
arrow.FixedWidthTypes.MonthDayNanoInterval):
129129
return bigquery.RecordFieldType
130+
// We don't use `typeOneOf` as `arrow.TypeEqual` checks for equality of precision and scale.
131+
case arrow.IsDecimal(dataType.ID()):
132+
// BigQuery NumericFieldType has a scale limit of 9, so we use BigNumeric for both decimal128 and decimal256.
133+
return bigquery.BigNumericFieldType
130134
default:
131135
panic("unsupported data type: " + dataType.String())
132136
}

‎plugins/destination/bigquery/go.mod‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ require (
66
cloud.google.com/go/bigquery v1.50.0
77
github.com/apache/arrow/go/v13 v13.0.0-20230601070034-e07e22c5580a
88
github.com/cloudquery/plugin-pb-go v1.0.8
9-
github.com/cloudquery/plugin-sdk/v3 v3.6.7
9+
github.com/cloudquery/plugin-sdk/v3 v3.10.3
1010
github.com/goccy/go-json v0.9.11
1111
github.com/rs/zerolog v1.29.0
1212
golang.org/x/sync v0.1.0
@@ -35,7 +35,7 @@ require (
3535
golang.org/x/mod v0.8.0 // indirect
3636
golang.org/x/tools v0.6.0 // indirect
3737
google.golang.org/genproto/googleapis/api v0.0.0-20230526203410-71b5a4ffd15e // indirect
38-
google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e // indirect
38+
google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
3939
gopkg.in/yaml.v3 v3.0.1 // indirect
4040
)
4141

‎plugins/destination/bigquery/go.sum‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -65,8 +65,8 @@ github.com/cloudquery/plugin-pb-go v1.0.8 h1:wn3GXhcNItcP+6wUUZuzUFbvdL59liKBO37
6565
github.com/cloudquery/plugin-pb-go v1.0.8/go.mod h1:vAGA27psem7ZZNAY4a3S9TKuA/JDQWstjKcHPJX91Mc=
6666
github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U=
6767
github.com/cloudquery/plugin-sdk/v2 v2.7.0/go.mod h1:pAX6ojIW99b/Vg4CkhnsGkRIzNaVEceYMR+Bdit73ug=
68-
github.com/cloudquery/plugin-sdk/v3 v3.6.7 h1:QJqZGHs+3uN+CE3y9oBOjkchH/v5XisRnVP2X4aj9Wo=
69-
github.com/cloudquery/plugin-sdk/v3 v3.6.7/go.mod h1:+ta6OETfGfzh6nCpCyZi5Er1rj+zvn7m2QR2wokEvA8=
68+
github.com/cloudquery/plugin-sdk/v3 v3.10.3 h1:aMofD3hHU4Dm+raxNgIOdSg+hGQrkTUTV2KXjxSwtqE=
69+
github.com/cloudquery/plugin-sdk/v3 v3.10.3/go.mod h1:P3zucEOH+IEhdM9FGD5q3WqciXIBOPCKw2kHZT4UrlQ=
7070
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
7171
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
7272
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
@@ -494,8 +494,8 @@ google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc h1:8DyZCyvI8mE1IdL
494494
google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64=
495495
google.golang.org/genproto/googleapis/api v0.0.0-20230526203410-71b5a4ffd15e h1:AZX1ra8YbFMSb7+1pI8S9v4rrgRR7jU1FmuFSSjTVcQ=
496496
google.golang.org/genproto/googleapis/api v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig=
497-
google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e h1:NumxXLPfHSndr3wBBdeKiVHjGVFzi9RX2HwwQke94iY=
498-
google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA=
497+
google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc=
498+
google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA=
499499
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
500500
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
501501
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=

0 commit comments

Comments
 (0)