Skip to content

Commit a109638

Browse files
erezrokaherezrokah
authored
fix(deps): Update module github.com/cloudquery/plugin-sdk/v3 to v3.10.3 (#11222)
 #### Summary Extracted from #11150 as MongoDB failed when trying to insert `unit64` that overflows `int64`, see https://github.com/cloudquery/cloudquery/actions/runs/5176150331/jobs/9324604882#step:8:14. The solution is to use a custom encoder to save the value as `int64`. When the value is read, if you cast it back to `unit64` you'd get the original value, so there's no dataloss. <!--
1 parent e3925ab commit a109638

File tree

5 files changed

+50
-8
lines changed

5 files changed

+50
-8
lines changed

‎plugins/destination/mongodb/client/client.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ func New(ctx context.Context, logger zerolog.Logger, destSpec specs.Destination)
3333
if err := spec.Validate(); err != nil {
3434
return nil, err
3535
}
36-
c.client, err = mongo.NewClient(options.Client().ApplyURI(spec.ConnectionString))
36+
c.client, err = mongo.NewClient(options.Client().ApplyURI(spec.ConnectionString).SetRegistry(getRegistry()))
3737
if err != nil {
3838
return nil, err
3939
}

‎plugins/destination/mongodb/client/uint64.go‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
package client
2+
3+
import (
4+
"reflect"
5+
6+
"go.mongodb.org/mongo-driver/bson"
7+
"go.mongodb.org/mongo-driver/bson/bsoncodec"
8+
"go.mongodb.org/mongo-driver/bson/bsonrw"
9+
)
10+
11+
type CustomUnit64 uint64
12+
13+
// See https://stackoverflow.com/questions/60520865/will-mongodb-go-driver-automatically-convert-uint64-to-bson-type-overflows-er
14+
// Please note don't need the decoder part from the above link, as we already cast back to uint64 when we pass the MongoDB saved value to arrow Uint64Builder
15+
func getRegistry() *bsoncodec.Registry {
16+
rb := bsoncodec.NewRegistryBuilder()
17+
var primitiveCodecs bson.PrimitiveCodecs
18+
bsoncodec.DefaultValueEncoders{}.RegisterDefaultEncoders(rb)
19+
bsoncodec.DefaultValueDecoders{}.RegisterDefaultDecoders(rb)
20+
primitiveCodecs.RegisterPrimitiveCodecs(rb)
21+
22+
customUnit64 := reflect.TypeOf(CustomUnit64(0))
23+
rb.RegisterTypeEncoder(
24+
customUnit64,
25+
bsoncodec.ValueEncoderFunc(func(_ bsoncodec.EncodeContext, vw bsonrw.ValueWriter, val reflect.Value) error {
26+
if !val.IsValid() || val.Type() != customUnit64 {
27+
return bsoncodec.ValueEncoderError{
28+
Name: "CustomUnit64EncodedValue",
29+
Types: []reflect.Type{customUnit64},
30+
Received: val,
31+
}
32+
}
33+
vw.WriteInt64(int64(val.Uint()))
34+
return nil
35+
}),
36+
)
37+
38+
reg := rb.Build()
39+
return reg
40+
}

‎plugins/destination/mongodb/client/write.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,9 @@ func transformArr(arr arrow.Array) []any {
3838
case *array.Uint32:
3939
dbArr[i] = a.Value(i)
4040
case *array.Uint64:
41-
dbArr[i] = a.Value(i)
41+
val := a.Value(i)
42+
var custom = CustomUnit64(val)
43+
dbArr[i] = custom
4244
case *array.Float32:
4345
dbArr[i] = a.Value(i)
4446
case *array.Float64:

‎plugins/destination/mongodb/go.mod‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ go 1.19
55
require (
66
github.com/apache/arrow/go/v13 v13.0.0-20230601070034-e07e22c5580a
77
github.com/cloudquery/plugin-pb-go v1.0.8
8-
github.com/cloudquery/plugin-sdk/v3 v3.6.7
8+
github.com/cloudquery/plugin-sdk/v3 v3.10.3
99
github.com/goccy/go-json v0.9.11
1010
github.com/rs/zerolog v1.29.0
1111
go.mongodb.org/mongo-driver v1.11.2
@@ -58,7 +58,7 @@ require (
5858
golang.org/x/text v0.9.0 // indirect
5959
golang.org/x/tools v0.6.0 // indirect
6060
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
61-
google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e // indirect
61+
google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
6262
google.golang.org/grpc v1.55.0 // indirect
6363
google.golang.org/protobuf v1.30.0 // indirect
6464
gopkg.in/yaml.v2 v2.4.0 // indirect

‎plugins/destination/mongodb/go.sum‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -50,8 +50,8 @@ github.com/cloudquery/plugin-pb-go v1.0.8 h1:wn3GXhcNItcP+6wUUZuzUFbvdL59liKBO37
5050
github.com/cloudquery/plugin-pb-go v1.0.8/go.mod h1:vAGA27psem7ZZNAY4a3S9TKuA/JDQWstjKcHPJX91Mc=
5151
github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U=
5252
github.com/cloudquery/plugin-sdk/v2 v2.7.0/go.mod h1:pAX6ojIW99b/Vg4CkhnsGkRIzNaVEceYMR+Bdit73ug=
53-
github.com/cloudquery/plugin-sdk/v3 v3.6.7 h1:QJqZGHs+3uN+CE3y9oBOjkchH/v5XisRnVP2X4aj9Wo=
54-
github.com/cloudquery/plugin-sdk/v3 v3.6.7/go.mod h1:+ta6OETfGfzh6nCpCyZi5Er1rj+zvn7m2QR2wokEvA8=
53+
github.com/cloudquery/plugin-sdk/v3 v3.10.3 h1:aMofD3hHU4Dm+raxNgIOdSg+hGQrkTUTV2KXjxSwtqE=
54+
github.com/cloudquery/plugin-sdk/v3 v3.10.3/go.mod h1:P3zucEOH+IEhdM9FGD5q3WqciXIBOPCKw2kHZT4UrlQ=
5555
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
5656
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
5757
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
@@ -493,8 +493,8 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D
493493
google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
494494
google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
495495
google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
496-
google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e h1:NumxXLPfHSndr3wBBdeKiVHjGVFzi9RX2HwwQke94iY=
497-
google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA=
496+
google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc=
497+
google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA=
498498
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
499499
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
500500
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=

0 commit comments

Comments
 (0)