Skip to content

Commit 9876bcb

Browse files
candiduslynxcandiduslynx
authored
fix: Use configtype.Duration (#11940)
Same as #11939 but for: * ClickHouse * Meilisearch * MSSQL
1 parent 77c931a commit 9876bcb

File tree

14 files changed

+66
-45
lines changed

14 files changed

+66
-45
lines changed

‎plugins/destination/clickhouse/client/client.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ func New(_ context.Context, logger zerolog.Logger, specBytes []byte, _ plugin.Ne
8686
batchwriter.WithLogger(l),
8787
batchwriter.WithBatchSize(spec.BatchSize),
8888
batchwriter.WithBatchSizeBytes(spec.BatchSizeBytes),
89-
batchwriter.WithBatchTimeout(spec.BatchTimeout),
89+
batchwriter.WithBatchTimeout(spec.BatchTimeout.Duration()),
9090
)
9191
if err != nil {
9292
return nil, err

‎plugins/destination/clickhouse/client/spec.go‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77

88
"github.com/ClickHouse/clickhouse-go/v2"
99
"github.com/cloudquery/cloudquery/plugins/destination/clickhouse/queries"
10+
"github.com/cloudquery/plugin-sdk/v4/configtype"
1011
)
1112

1213
type Spec struct {
@@ -16,9 +17,9 @@ type Spec struct {
1617

1718
Engine *queries.Engine `json:"engine,omitempty"`
1819

19-
BatchSize int `json:"batch_size,omitempty"`
20-
BatchSizeBytes int `json:"batch_size_bytes,omitempty"`
21-
BatchTimeout time.Duration `json:"batch_timeout,omitempty"`
20+
BatchSize int `json:"batch_size,omitempty"`
21+
BatchSizeBytes int `json:"batch_size_bytes,omitempty"`
22+
BatchTimeout *configtype.Duration `json:"batch_timeout,omitempty"`
2223
}
2324

2425
func (s *Spec) Options() (*clickhouse.Options, error) {
@@ -54,18 +55,16 @@ func (s *Spec) SetDefaults() {
5455
}
5556

5657
if s.BatchSize == 0 {
57-
const defaultBatchSize = 10_000 // 10K
58-
s.BatchSize = defaultBatchSize
58+
s.BatchSize = 10_000 // 10K
5959
}
6060

6161
if s.BatchSizeBytes == 0 {
62-
const defaultBatchSizeBytes = 5 << 20 // 5 MiB
63-
s.BatchSizeBytes = defaultBatchSizeBytes
62+
s.BatchSizeBytes = 5 << 20 // 5 MiB
6463
}
6564

66-
if s.BatchTimeout == 0 {
67-
const defaultBatchTimeout = 20 * time.Second // 20s
68-
s.BatchTimeout = defaultBatchTimeout
65+
if s.BatchTimeout == nil {
66+
d := configtype.NewDuration(20 * time.Second) // 20s
67+
s.BatchTimeout = &d
6968
}
7069
}
7170

‎plugins/destination/clickhouse/go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ require (
66
github.com/ClickHouse/clickhouse-go/v2 v2.10.1
77
github.com/apache/arrow/go/v13 v13.0.0-20230630125530-5a06b2ec2a8e
88
github.com/cloudquery/plugin-pb-go v1.6.0 // indirect
9-
github.com/cloudquery/plugin-sdk/v4 v4.6.1-rc1
9+
github.com/cloudquery/plugin-sdk/v4 v4.7.0-rc1
1010
github.com/google/uuid v1.3.0
1111
github.com/rs/zerolog v1.29.1
1212
github.com/stretchr/testify v1.8.4

‎plugins/destination/clickhouse/go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,8 +51,8 @@ github.com/cloudquery/plugin-pb-go v1.6.0 h1:4g+tPhRAQUWpGyQBMcj6D3yVwKqwEN3nynv
5151
github.com/cloudquery/plugin-pb-go v1.6.0/go.mod h1:R0Wse6NbJDZIHcRQjJ1sZGYDo3mrIDm4k3El1YUrvGA=
5252
github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U=
5353
github.com/cloudquery/plugin-sdk/v2 v2.7.0/go.mod h1:pAX6ojIW99b/Vg4CkhnsGkRIzNaVEceYMR+Bdit73ug=
54-
github.com/cloudquery/plugin-sdk/v4 v4.6.1-rc1 h1:oO1locVCUfi8tud9cwRPTO+hLyih2jgZ24nlvvuimh4=
55-
github.com/cloudquery/plugin-sdk/v4 v4.6.1-rc1/go.mod h1:Y5HzxesZrpmSTUrbvR8EmGCK5wXDAhqSqzNEXC2ouvI=
54+
github.com/cloudquery/plugin-sdk/v4 v4.7.0-rc1 h1:n8m4gdsVmnHrdn213OEBEnELqKldizy325U2SyoClpc=
55+
github.com/cloudquery/plugin-sdk/v4 v4.7.0-rc1/go.mod h1:Y5HzxesZrpmSTUrbvR8EmGCK5wXDAhqSqzNEXC2ouvI=
5656
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
5757
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
5858
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=

‎plugins/destination/meilisearch/client/client.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ func New(_ context.Context, logger zerolog.Logger, specBytes []byte, _ plugin.Ne
9696
client.writer, err = batchwriter.New(client,
9797
batchwriter.WithBatchSize(spec.BatchSize),
9898
batchwriter.WithBatchSizeBytes(spec.BatchSizeBytes),
99-
batchwriter.WithBatchTimeout(spec.Timeout),
99+
batchwriter.WithBatchTimeout(spec.BatchTimeout.Duration()),
100100
)
101101
if err != nil {
102102
return nil, err

‎plugins/destination/meilisearch/client/spec.go‎

Lines changed: 16 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -6,25 +6,23 @@ import (
66
"fmt"
77
"time"
88

9+
"github.com/cloudquery/plugin-sdk/v4/configtype"
910
"github.com/meilisearch/meilisearch-go"
1011
"github.com/valyala/fasthttp"
1112
)
1213

13-
const (
14-
defaultBatchSize = 1000
15-
defaultBatchSizeBytes = 4 * 1024 * 1024
16-
)
17-
1814
type Spec struct {
1915
// required
20-
Host string `json:"host,omitempty"`
21-
APIKey string `json:"api_key,omitempty"`
22-
BatchSize int `json:"batch_size,omitempty"`
23-
BatchSizeBytes int `json:"batch_size_bytes,omitempty"`
16+
Host string `json:"host,omitempty"`
17+
APIKey string `json:"api_key,omitempty"`
2418

2519
// optional
2620
Timeout time.Duration `json:"timeout,omitempty"`
2721
CACert string `json:"ca_cert,omitempty"`
22+
23+
BatchSize int `json:"batch_size,omitempty"`
24+
BatchSizeBytes int `json:"batch_size_bytes,omitempty"`
25+
BatchTimeout *configtype.Duration `json:"batch_timeout,omitempty"`
2826
}
2927

3028
func (s *Spec) validate() error {
@@ -42,11 +40,18 @@ func (s *Spec) setDefaults() {
4240
if s.Timeout == 0 {
4341
s.Timeout = 5 * time.Minute
4442
}
43+
4544
if s.BatchSize == 0 {
46-
s.BatchSize = defaultBatchSize
45+
s.BatchSize = 1000 // 1K
4746
}
47+
4848
if s.BatchSizeBytes == 0 {
49-
s.BatchSizeBytes = defaultBatchSizeBytes
49+
s.BatchSizeBytes = 4 << 20 // 4 MiB
50+
}
51+
52+
if s.BatchTimeout == nil {
53+
d := configtype.NewDuration(20 * time.Second) // 20s
54+
s.BatchTimeout = &d
5055
}
5156
}
5257

‎plugins/destination/meilisearch/go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ go 1.19
44

55
require (
66
github.com/apache/arrow/go/v13 v13.0.0-20230630125530-5a06b2ec2a8e
7-
github.com/cloudquery/plugin-sdk/v4 v4.6.1-rc1
7+
github.com/cloudquery/plugin-sdk/v4 v4.7.0-rc1
88
github.com/goccy/go-json v0.10.2
99
github.com/google/uuid v1.3.0
1010
github.com/meilisearch/meilisearch-go v0.24.0

‎plugins/destination/meilisearch/go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,8 @@ github.com/cloudquery/plugin-pb-go v1.6.0 h1:4g+tPhRAQUWpGyQBMcj6D3yVwKqwEN3nynv
4848
github.com/cloudquery/plugin-pb-go v1.6.0/go.mod h1:R0Wse6NbJDZIHcRQjJ1sZGYDo3mrIDm4k3El1YUrvGA=
4949
github.com/cloudquery/plugin-sdk/v2 v2.7.0 h1:hRXsdEiaOxJtsn/wZMFQC9/jPfU1MeMK3KF+gPGqm7U=
5050
github.com/cloudquery/plugin-sdk/v2 v2.7.0/go.mod h1:pAX6ojIW99b/Vg4CkhnsGkRIzNaVEceYMR+Bdit73ug=
51-
github.com/cloudquery/plugin-sdk/v4 v4.6.1-rc1 h1:oO1locVCUfi8tud9cwRPTO+hLyih2jgZ24nlvvuimh4=
52-
github.com/cloudquery/plugin-sdk/v4 v4.6.1-rc1/go.mod h1:Y5HzxesZrpmSTUrbvR8EmGCK5wXDAhqSqzNEXC2ouvI=
51+
github.com/cloudquery/plugin-sdk/v4 v4.7.0-rc1 h1:n8m4gdsVmnHrdn213OEBEnELqKldizy325U2SyoClpc=
52+
github.com/cloudquery/plugin-sdk/v4 v4.7.0-rc1/go.mod h1:Y5HzxesZrpmSTUrbvR8EmGCK5wXDAhqSqzNEXC2ouvI=
5353
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
5454
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
5555
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=

‎plugins/destination/mssql/client/client.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ func New(_ context.Context, logger zerolog.Logger, specBytes []byte, _ plugin.Ne
5656
batchwriter.WithLogger(c.logger),
5757
batchwriter.WithBatchSize(spec.BatchSize),
5858
batchwriter.WithBatchSizeBytes(spec.BatchSizeBytes),
59-
batchwriter.WithBatchTimeout(spec.BatchTimeout),
59+
batchwriter.WithBatchTimeout(spec.BatchTimeout.Duration()),
6060
)
6161
if err != nil {
6262
return nil, err

‎plugins/destination/mssql/client/spec.go‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"strings"
55
"time"
66

7+
"github.com/cloudquery/plugin-sdk/v4/configtype"
78
mssql "github.com/microsoft/go-mssqldb"
89
"github.com/microsoft/go-mssqldb/azuread"
910
)
@@ -20,9 +21,9 @@ type Spec struct {
2021
AuthMode AuthMode `json:"auth_mode,omitempty"`
2122
Schema string `json:"schema,omitempty"`
2223

23-
BatchSize int `json:"batch_size,omitempty"`
24-
BatchSizeBytes int `json:"batch_size_bytes,omitempty"`
25-
BatchTimeout time.Duration `json:"batch_timeout,omitempty"`
24+
BatchSize int `json:"batch_size,omitempty"`
25+
BatchSizeBytes int `json:"batch_size_bytes,omitempty"`
26+
BatchTimeout *configtype.Duration `json:"batch_timeout,omitempty"`
2627
}
2728

2829
func (s *Spec) SetDefaults() {
@@ -36,18 +37,16 @@ func (s *Spec) SetDefaults() {
3637
}
3738

3839
if s.BatchSize == 0 {
39-
const defaultBatchSize = 1000 // 1K
40-
s.BatchSize = defaultBatchSize
40+
s.BatchSize = 1000 // 1K
4141
}
4242

4343
if s.BatchSizeBytes == 0 {
44-
const defaultBatchSizeBytes = 5 << 20 // 5 MiB
45-
s.BatchSizeBytes = defaultBatchSizeBytes
44+
s.BatchSizeBytes = 5 << 20 // 5 MiB
4645
}
4746

48-
if s.BatchTimeout == 0 {
49-
const defaultBatchTimeout = 20 * time.Second // 20s
50-
s.BatchTimeout = defaultBatchTimeout
47+
if s.BatchTimeout == nil {
48+
d := configtype.NewDuration(20 * time.Second) // 20s
49+
s.BatchTimeout = &d
5150
}
5251
}
5352

0 commit comments

Comments
 (0)