Skip to content

Commit e2821fc

Browse files
benmcclellandbenmcclelland
committed
feat: add option to disable s3proxy client data integrity checks
AWS introduced a relatively newer option for data integrity checks that not all non-AWS server support yet. See this for mmore info: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html This change adds a new option: disable-data-integrity-check to disable the data integrity checks in the client sdk for the servers that may not yet support this. Use this only when the s3 service for the proxy does not support the data integrity features. Fixes #1867
1 parent a81f9e5 commit e2821fc

File tree

4 files changed

+55
-29
lines changed

4 files changed

+55
-29
lines changed

‎backend/s3proxy/client.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,11 @@ func (s *S3Proxy) getConfig(ctx context.Context, access, secret string) (aws.Con
7676
config.WithAPIOptions([]func(*middleware.Stack) error{v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware}))
7777
}
7878

79+
if s.disableDataIntegrityCheck {
80+
opts = append(opts,
81+
config.WithRequestChecksumCalculation(aws.RequestChecksumCalculationWhenRequired))
82+
}
83+
7984
if s.debug {
8085
opts = append(opts,
8186
config.WithClientLogMode(aws.LogSigning|aws.LogRetries|aws.LogRequest|aws.LogResponse|aws.LogRequestEventMessage|aws.LogResponseEventMessage))

‎backend/s3proxy/s3.go‎

Lines changed: 21 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -47,15 +47,16 @@ type S3Proxy struct {
4747

4848
client *s3.Client
4949

50-
access string
51-
secret string
52-
endpoint string
53-
awsRegion string
54-
metaBucket string
55-
disableChecksum bool
56-
sslSkipVerify bool
57-
usePathStyle bool
58-
debug bool
50+
access string
51+
secret string
52+
endpoint string
53+
awsRegion string
54+
metaBucket string
55+
disableChecksum bool
56+
disableDataIntegrityCheck bool
57+
sslSkipVerify bool
58+
usePathStyle bool
59+
debug bool
5960
}
6061

6162
var _ backend.Backend = &S3Proxy{}
@@ -68,17 +69,18 @@ func NewWithClient(ctx context.Context, client *s3.Client, metaBucket string) (*
6869
return s, s.validate(ctx)
6970
}
7071

71-
func New(ctx context.Context, access, secret, endpoint, region, metaBucket string, disableChecksum, sslSkipVerify, usePathStyle, debug bool) (*S3Proxy, error) {
72+
func New(ctx context.Context, access, secret, endpoint, region, metaBucket string, disableChecksum, disableDataIntegrityCheck, sslSkipVerify, usePathStyle, debug bool) (*S3Proxy, error) {
7273
s := &S3Proxy{
73-
access: access,
74-
secret: secret,
75-
endpoint: endpoint,
76-
awsRegion: region,
77-
metaBucket: metaBucket,
78-
disableChecksum: disableChecksum,
79-
sslSkipVerify: sslSkipVerify,
80-
usePathStyle: usePathStyle,
81-
debug: debug,
74+
access: access,
75+
secret: secret,
76+
endpoint: endpoint,
77+
awsRegion: region,
78+
metaBucket: metaBucket,
79+
disableChecksum: disableChecksum,
80+
disableDataIntegrityCheck: disableDataIntegrityCheck,
81+
sslSkipVerify: sslSkipVerify,
82+
usePathStyle: usePathStyle,
83+
debug: debug,
8284
}
8385
client, err := s.getClientWithCtx(ctx)
8486
if err != nil {

‎cmd/versitygw/s3.go‎

Lines changed: 18 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -22,15 +22,16 @@ import (
2222
)
2323

2424
var (
25-
s3proxyAccess string
26-
s3proxySecret string
27-
s3proxyEndpoint string
28-
s3proxyRegion string
29-
s3proxyMetaBucket string
30-
s3proxyDisableChecksum bool
31-
s3proxySslSkipVerify bool
32-
s3proxyUsePathStyle bool
33-
s3proxyDebug bool
25+
s3proxyAccess string
26+
s3proxySecret string
27+
s3proxyEndpoint string
28+
s3proxyRegion string
29+
s3proxyMetaBucket string
30+
s3proxyDisableChecksum bool
31+
s3proxyDisableDataIntegrityCheck bool
32+
s3proxySslSkipVerify bool
33+
s3proxyUsePathStyle bool
34+
s3proxyDebug bool
3435
)
3536

3637
func s3Command() *cli.Command {
@@ -84,6 +85,13 @@ to an s3 storage backend service.`,
8485
EnvVars: []string{"VGW_S3_DISABLE_CHECKSUM"},
8586
Destination: &s3proxyDisableChecksum,
8687
},
88+
&cli.BoolFlag{
89+
Name: "disable-data-integrity-check",
90+
Usage: "disable data integrity checks for requests (sets RequestChecksumCalculationWhenRequired)",
91+
Value: false,
92+
EnvVars: []string{"VGW_S3_DISABLE_DATA_INTEGRITY_CHECK"},
93+
Destination: &s3proxyDisableDataIntegrityCheck,
94+
},
8795
&cli.BoolFlag{
8896
Name: "ssl-skip-verify",
8997
Usage: "skip ssl cert verification for s3 service",
@@ -111,7 +119,7 @@ to an s3 storage backend service.`,
111119

112120
func runS3(ctx *cli.Context) error {
113121
be, err := s3proxy.New(ctx.Context, s3proxyAccess, s3proxySecret, s3proxyEndpoint, s3proxyRegion,
114-
s3proxyMetaBucket, s3proxyDisableChecksum, s3proxySslSkipVerify, s3proxyUsePathStyle, s3proxyDebug)
122+
s3proxyMetaBucket, s3proxyDisableChecksum, s3proxyDisableDataIntegrityCheck, s3proxySslSkipVerify, s3proxyUsePathStyle, s3proxyDebug)
115123
if err != nil {
116124
return fmt.Errorf("init s3 backend: %w", err)
117125
}

‎extra/example.conf‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -535,12 +535,23 @@ ROOT_SECRET_ACCESS_KEY=
535535
# be defined for an authorized access or both empty for an anonymous access.
536536
# The VGW_S3_REGION and VGW_S3_ENDPOINT are optional, and will default to
537537
# "us-east-1" and "https://s3.amazonaws.com" respectively.
538+
# VGW_S3_META_BUCKET specifies a bucket to store bucket ACL/policy metadata.
539+
# VGW_S3_DISABLE_CHECKSUM will disable the SHA256 checksum for unsigned payload.
540+
# VGW_S3_DISABLE_DATA_INTEGRITY_CHECK will disable data integrity checks on
541+
# requests by setting RequestChecksumCalculationWhenRequired, which only
542+
# calculates checksums when explicitly required (may improve performance).
543+
# VGW_S3_SSL_SKIP_VERIFY will skip SSL certificate verification.
544+
# VGW_S3_USE_PATH_STYLE will use path style addressing for the S3 proxy.
545+
# VGW_S3_DEBUG will enable debug logging for S3 requests.
538546
#VGW_S3_ACCESS_KEY=
539547
#VGW_S3_SECRET_KEY=
540548
#VGW_S3_REGION=
541549
#VGW_S3_ENDPOINT=
550+
#VGW_S3_META_BUCKET=
542551
#VGW_S3_DISABLE_CHECKSUM=false
552+
#VGW_S3_DISABLE_DATA_INTEGRITY_CHECK=false
543553
#VGW_S3_SSL_SKIP_VERIFY=false
554+
#VGW_S3_USE_PATH_STYLE=false
544555
#VGW_S3_DEBUG=false
545556

546557
########

0 commit comments

Comments
 (0)