Skip to content

Commit 61144d4

Browse files
Andrey Kolkovclaude
andcommitted
fix(agent): default S3 RequestChecksumCalculation to WhenRequired
aws-sdk-go-v2 defaults RequestChecksumCalculation to WhenSupported (since early 2025), attaching a flexible checksum to every PutObject. Non-AWS S3-compatible backends (Ceph RGW, some MinIO/R2) reject the accompanying header with "InvalidArgument: x-amz-content-sha256 must be UNSIGNED-PAYLOAD, STREAMING-AWS4-HMAC-SHA256-PAYLOAD or a valid sha256 value", so the snapshot-agent reads the etcd snapshot fine but fails at S3 upload. Set the S3 client to WhenRequired so a checksum is only added when the operation requires one; a plain PutObject then carries none, which both AWS S3 and the affected backends accept. No API/CRD change and no new knob. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]> Signed-off-by: Andrey Kolkov <[email protected]>
1 parent f0e3c86 commit 61144d4

2 files changed

Lines changed: 44 additions & 1 deletion

File tree

‎internal/agent/agent.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,18 @@ func (d destination) s3Client(ctx context.Context) (*s3.Client, error) {
166166
o.BaseEndpoint = aws.String(d.s3Endpoint)
167167
}
168168
o.UsePathStyle = d.s3PathStyle
169+
// Only add a request checksum when the operation actually requires one.
170+
//
171+
// Since early 2025, aws-sdk-go-v2 defaults RequestChecksumCalculation to
172+
// WhenSupported, which attaches a CRC32 to every PutObject. Many
173+
// S3-compatible backends that are not AWS S3 (Ceph RGW, some MinIO and
174+
// Cloudflare R2 versions, etc.) reject the accompanying header with
175+
// "InvalidArgument: x-amz-content-sha256 must be UNSIGNED-PAYLOAD,
176+
// STREAMING-AWS4-HMAC-SHA256-PAYLOAD or a valid sha256 value", so the
177+
// snapshot upload fails against them. WhenRequired keeps uploads working
178+
// on those backends and is equally accepted by real AWS S3, which does
179+
// not require a checksum on a plain PutObject.
180+
o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired
169181
}), nil
170182
}
171183

‎internal/agent/agent_test.go‎

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,12 @@ You may obtain a copy of the License at
1010

1111
package agent
1212

13-
import "testing"
13+
import (
14+
"context"
15+
"testing"
16+
17+
"github.com/aws/aws-sdk-go-v2/aws"
18+
)
1419

1520
func TestObjectKey(t *testing.T) {
1621
cases := []struct {
@@ -31,6 +36,32 @@ func TestObjectKey(t *testing.T) {
3136
}
3237
}
3338

39+
func TestS3ClientChecksumWhenRequired(t *testing.T) {
40+
d := destination{
41+
kind: "s3",
42+
s3Endpoint: "https://s3.example.internal",
43+
s3PathStyle: true,
44+
}
45+
c, err := d.s3Client(context.Background())
46+
if err != nil {
47+
t.Fatalf("s3Client: %v", err)
48+
}
49+
o := c.Options()
50+
// Non-AWS S3-compatible backends (Ceph RGW, some MinIO/R2) reject the
51+
// aws-sdk-go-v2 default (WhenSupported) flexible checksum on PutObject, so
52+
// the agent must only add a checksum when the operation requires one.
53+
if o.RequestChecksumCalculation != aws.RequestChecksumCalculationWhenRequired {
54+
t.Errorf("RequestChecksumCalculation = %v, want WhenRequired (%v)",
55+
o.RequestChecksumCalculation, aws.RequestChecksumCalculationWhenRequired)
56+
}
57+
if !o.UsePathStyle {
58+
t.Error("UsePathStyle = false, want true")
59+
}
60+
if o.BaseEndpoint == nil || *o.BaseEndpoint != d.s3Endpoint {
61+
t.Errorf("BaseEndpoint = %v, want %q", o.BaseEndpoint, d.s3Endpoint)
62+
}
63+
}
64+
3465
func TestURI(t *testing.T) {
3566
s3 := destination{kind: "s3", s3Bucket: "bucket", s3Key: "pre"}
3667
if got, want := s3.uri("snap"), "s3://bucket/pre/snap.db"; got != want {

0 commit comments

Comments
 (0)