Skip to content

Commit a90b910

Browse files
authored
fix(gcsartifact): prevent artifact version race in concurrent Save (#1172)
* fix(gcsartifact): prevent artifact version race in concurrent Save Save assigned versions non-atomically: it listed the existing versions, computed max+1, and wrote that blob. Two concurrent Save calls for the same artifact would read the same version set, pick the same next version, and the second write would silently overwrite the first — duplicate returned versions and a lost object. The code carried a TODO acknowledging this. Assign versions with optimistic concurrency instead: write the new blob with a "does not exist" generation precondition (x-goog-if-generation-match: 0). GCS has no transaction spanning the list and the write, so a colliding writer now fails the precondition (HTTP 412); Save re-reads the versions and retries with a fresh number, using a jittered backoff between attempts and bounded by maxSaveAttempts. It returns errVersionConflict if the budget is exhausted rather than overwriting an existing version. N concurrent saves therefore yield exactly versions 1..N with no lost writes. Testing Plan: - go build -mod=readonly ./... - go test -race -mod=readonly -count=1 -shuffle=on ./artifact/... - golangci-lint run ./artifact/... - go mod tidy -diff - TestGCSArtifactServiceConcurrentSave: concurrent savers assert distinct, contiguous versions and that nothing is overwritten (saver count kept within the retry budget). Verified it fails against the pre-fix code. - TestSaveReturnsWriteErrorWithoutRetry: a non-precondition write error is surfaced as-is and not retried. - TestSaveExhaustsRetriesOnPersistentConflict: persistent 412s give up with errVersionConflict after maxSaveAttempts. - TestBackoffDelayBounds: jittered backoff stays within bounds. * test(gcsartifact): cover the real precondition and zero-byte artifacts The does-not-exist precondition on the production ObjectHandle wrapper had no test: replacing it with a no-op left the whole suite green, so a regression there would silently restore last-writer-wins. Add an httptest-backed test that drives the real wrapper offline and asserts the upload carries ifGenerationMatch=0 (the client sends the condition as a query parameter, not a header). The fake also used data != nil as its existence test, so a zero-byte artifact -- reachable through the public API, since SaveRequest.Validate only requires Part.InlineData -- was modelled as not existing: saving it twice returned version 1 both times. Track existence in an explicit exists flag instead, which also subsumes the deleted flag. Also wrap the context error from the retry backoff, drop the unreachable nil check on the sleep hook, and note the adk-python parity gap and the per-attempt listing cost in the Save doc. * docs(gcsartifact): drop the cross-implementation claim from Save The caveat about other writers is a property of the scheme, so state it in those terms rather than pinning it to another implementation's current behaviour, which this repo cannot keep in sync.
1 parent 690fbd6 commit a90b910

5 files changed

Lines changed: 533 additions & 90 deletions

File tree

‎artifact/gcsartifact/gcs_client.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,9 @@ type gcsObject interface {
3939
newReader(ctx context.Context) (io.ReadCloser, error)
4040
delete(ctx context.Context) error
4141
attrs(ctx context.Context) (*storage.ObjectAttrs, error)
42+
// ifNotExist returns a handle whose write succeeds only if the object does
43+
// not already exist; writing over an existing object fails with HTTP 412.
44+
ifNotExist() gcsObject
4245
}
4346

4447
// gcsObjectIterator
@@ -97,6 +100,11 @@ func (w *gcsObjectWrapper) newWriter(ctx context.Context) gcsWriter {
97100
return &gcsWriterWrapper{w: w.object.NewWriter(ctx)}
98101
}
99102

103+
// ifNotExist implements the gcsObject interface for gcsObjectWrapper.
104+
func (w *gcsObjectWrapper) ifNotExist() gcsObject {
105+
return &gcsObjectWrapper{object: w.object.If(storage.Conditions{DoesNotExist: true})}
106+
}
107+
100108
// NewReader implements the gcsObject interface for gcsObjectWrapper.
101109
func (w *gcsObjectWrapper) newReader(ctx context.Context) (io.ReadCloser, error) {
102110
return w.object.NewReader(ctx)
Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
// Copyright 2025 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package gcsartifact
16+
17+
import (
18+
"io"
19+
"net/http"
20+
"net/http/httptest"
21+
"testing"
22+
23+
"cloud.google.com/go/storage"
24+
"google.golang.org/api/option"
25+
)
26+
27+
// TestObjectWrapperIfNotExist checks that the wrapper puts the does-not-exist
28+
// precondition on the wire. Everything else in this package tests against the
29+
// fake, so without this a wrapper that dropped the condition would still pass
30+
// while Save silently degraded to last-writer-wins. The client sends it as the
31+
// ifGenerationMatch=0 query parameter on the upload URL.
32+
func TestObjectWrapperIfNotExist(t *testing.T) {
33+
for _, tc := range []struct {
34+
name string
35+
conditional bool
36+
want string
37+
}{
38+
{"conditional", true, "0"},
39+
{"unconditional", false, ""},
40+
} {
41+
t.Run(tc.name, func(t *testing.T) {
42+
var got string
43+
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
44+
got = r.URL.Query().Get("ifGenerationMatch")
45+
if _, err := io.Copy(io.Discard, r.Body); err != nil {
46+
t.Errorf("draining request body failed: %v", err)
47+
}
48+
w.Header().Set("Content-Type", "application/json")
49+
if _, err := io.WriteString(w, `{"bucket":"bucket","name":"obj","generation":"1"}`); err != nil {
50+
t.Errorf("writing response failed: %v", err)
51+
}
52+
}))
53+
defer srv.Close()
54+
55+
client, err := storage.NewClient(t.Context(), option.WithEndpoint(srv.URL), option.WithoutAuthentication())
56+
if err != nil {
57+
t.Fatalf("storage.NewClient() failed: %v", err)
58+
}
59+
t.Cleanup(func() {
60+
if err := client.Close(); err != nil {
61+
t.Errorf("client.Close() failed: %v", err)
62+
}
63+
})
64+
65+
obj := (&gcsClientWrapper{client: client}).bucket("bucket").object("obj")
66+
if tc.conditional {
67+
obj = obj.ifNotExist()
68+
}
69+
w := obj.newWriter(t.Context())
70+
if _, err := w.Write([]byte("data")); err != nil {
71+
t.Fatalf("Write() failed: %v", err)
72+
}
73+
if err := w.Close(); err != nil {
74+
t.Fatalf("Close() failed: %v", err)
75+
}
76+
77+
if got != tc.want {
78+
t.Errorf("ifGenerationMatch = %q, want %q", got, tc.want)
79+
}
80+
})
81+
}
82+
}

0 commit comments

Comments
 (0)