Skip to content

Commit e36b29a

Browse files
authored
fix(storage): revise Reader to send XML preconditions (#4479)
This updates `Reader` to send precondition request headers to fit the XML API used for downloads. - adds a new function `setConditionsHeaders()` that parses `Conditions` and sets the corresponding precondition headers - revises `NewRangerReader` to call `setConditionsHeaders()` - if an object version is specified via `generation`, include `generation` as query string parameters - updates tests Fixes #4470
1 parent dec7a67 commit e36b29a

4 files changed

Lines changed: 109 additions & 68 deletions

File tree

‎storage/integration_test.go‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -744,6 +744,36 @@ func TestIntegration_ObjectsRangeReader(t *testing.T) {
744744
}
745745
}
746746

747+
func TestIntegration_ConditionalDownload(t *testing.T) {
748+
ctx := context.Background()
749+
client := testConfig(ctx, t)
750+
defer client.Close()
751+
h := testHelper{t}
752+
753+
o := client.Bucket(bucketName).Object("condread")
754+
defer o.Delete(ctx)
755+
756+
wc := o.NewWriter(ctx)
757+
wc.ContentType = "text/plain"
758+
h.mustWrite(wc, []byte("foo"))
759+
760+
gen := wc.Attrs().Generation
761+
metaGen := wc.Attrs().Metageneration
762+
763+
if _, err := o.Generation(gen + 1).NewReader(ctx); err == nil {
764+
t.Fatalf("Unexpected successful download with nonexistent Generation")
765+
}
766+
if _, err := o.If(Conditions{MetagenerationMatch: metaGen + 1}).NewReader(ctx); err == nil {
767+
t.Fatalf("Unexpected successful download with failed preconditions IfMetaGenerationMatch")
768+
}
769+
if _, err := o.If(Conditions{GenerationMatch: gen + 1}).NewReader(ctx); err == nil {
770+
t.Fatalf("Unexpected successful download with failed preconditions IfGenerationMatch")
771+
}
772+
if _, err := o.If(Conditions{GenerationMatch: gen}).NewReader(ctx); err != nil {
773+
t.Fatalf("Download failed: %v", err)
774+
}
775+
}
776+
747777
func TestIntegration_Objects(t *testing.T) {
748778
// TODO(jba): Use subtests (Go 1.7).
749779
ctx := context.Background()

‎storage/reader.go‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,14 @@ func (o *ObjectHandle) NewRangeReader(ctx context.Context, offset, length int64)
149149
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, offset+length-1))
150150
}
151151
// We wait to assign conditions here because the generation number can change in between reopen() runs.
152-
req.URL.RawQuery = conditionsQuery(gen, o.conds)
152+
if err := setConditionsHeaders(req.Header, o.conds); err != nil {
153+
return nil, err
154+
}
155+
// If an object generation is specified, include generation as query string parameters.
156+
if gen >= 0 {
157+
req.URL.RawQuery = fmt.Sprintf("generation=%d", gen)
158+
}
159+
153160
var res *http.Response
154161
err = runWithRetry(ctx, func() error {
155162
res, err = o.c.hc.Do(req)
@@ -324,6 +331,24 @@ func parseCRC32c(res *http.Response) (uint32, bool) {
324331
return 0, false
325332
}
326333

334+
// setConditionsHeaders sets precondition request headers for downloads
335+
// using the XML API. It assumes that the conditions have been validated.
336+
func setConditionsHeaders(headers http.Header, conds *Conditions) error {
337+
if conds == nil {
338+
return nil
339+
}
340+
if conds.MetagenerationMatch != 0 {
341+
headers.Set("x-goog-if-metageneration-match", fmt.Sprint(conds.MetagenerationMatch))
342+
}
343+
switch {
344+
case conds.GenerationMatch != 0:
345+
headers.Set("x-goog-if-generation-match", fmt.Sprint(conds.GenerationMatch))
346+
case conds.DoesNotExist:
347+
headers.Set("x-goog-if-generation-match", "0")
348+
}
349+
return nil
350+
}
351+
327352
var emptyBody = ioutil.NopCloser(strings.NewReader(""))
328353

329354
// Reader reads a Cloud Storage object.

‎storage/storage.go‎

Lines changed: 0 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@ import (
3333
"reflect"
3434
"regexp"
3535
"sort"
36-
"strconv"
3736
"strings"
3837
"time"
3938
"unicode/utf8"
@@ -1596,44 +1595,6 @@ func setConditionField(call reflect.Value, name string, value interface{}) bool
15961595
return true
15971596
}
15981597

1599-
// conditionsQuery returns the generation and conditions as a URL query
1600-
// string suitable for URL.RawQuery. It assumes that the conditions
1601-
// have been validated.
1602-
func conditionsQuery(gen int64, conds *Conditions) string {
1603-
// URL escapes are elided because integer strings are URL-safe.
1604-
var buf []byte
1605-
1606-
appendParam := func(s string, n int64) {
1607-
if len(buf) > 0 {
1608-
buf = append(buf, '&')
1609-
}
1610-
buf = append(buf, s...)
1611-
buf = strconv.AppendInt(buf, n, 10)
1612-
}
1613-
1614-
if gen >= 0 {
1615-
appendParam("generation=", gen)
1616-
}
1617-
if conds == nil {
1618-
return string(buf)
1619-
}
1620-
switch {
1621-
case conds.GenerationMatch != 0:
1622-
appendParam("ifGenerationMatch=", conds.GenerationMatch)
1623-
case conds.GenerationNotMatch != 0:
1624-
appendParam("ifGenerationNotMatch=", conds.GenerationNotMatch)
1625-
case conds.DoesNotExist:
1626-
appendParam("ifGenerationMatch=", 0)
1627-
}
1628-
switch {
1629-
case conds.MetagenerationMatch != 0:
1630-
appendParam("ifMetagenerationMatch=", conds.MetagenerationMatch)
1631-
case conds.MetagenerationNotMatch != 0:
1632-
appendParam("ifMetagenerationNotMatch=", conds.MetagenerationNotMatch)
1633-
}
1634-
return string(buf)
1635-
}
1636-
16371598
// composeSourceObj wraps a *raw.ComposeRequestSourceObjects, but adds the methods
16381599
// that modifyCall searches for by name.
16391600
type composeSourceObj struct {

‎storage/storage_test.go‎

Lines changed: 53 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -644,34 +644,6 @@ func TestCondition(t *testing.T) {
644644
},
645645
"GET /buck/obj?generation=1234",
646646
},
647-
{
648-
func() error {
649-
_, err := obj.If(Conditions{GenerationMatch: 1234}).NewReader(ctx)
650-
return err
651-
},
652-
"GET /buck/obj?ifGenerationMatch=1234",
653-
},
654-
{
655-
func() error {
656-
_, err := obj.If(Conditions{GenerationNotMatch: 1234}).NewReader(ctx)
657-
return err
658-
},
659-
"GET /buck/obj?ifGenerationNotMatch=1234",
660-
},
661-
{
662-
func() error {
663-
_, err := obj.If(Conditions{MetagenerationMatch: 1234}).NewReader(ctx)
664-
return err
665-
},
666-
"GET /buck/obj?ifMetagenerationMatch=1234",
667-
},
668-
{
669-
func() error {
670-
_, err := obj.If(Conditions{MetagenerationNotMatch: 1234}).NewReader(ctx)
671-
return err
672-
},
673-
"GET /buck/obj?ifMetagenerationNotMatch=1234",
674-
},
675647
{
676648
func() error {
677649
_, err := obj.If(Conditions{MetagenerationNotMatch: 1234}).Attrs(ctx)
@@ -734,6 +706,59 @@ func TestCondition(t *testing.T) {
734706
}
735707
}
736708

709+
readerTests := []struct {
710+
fn func() error
711+
want string
712+
}{
713+
{
714+
func() error {
715+
_, err := obj.If(Conditions{GenerationMatch: 1234}).NewReader(ctx)
716+
return err
717+
},
718+
"x-goog-if-generation-match: 1234, x-goog-if-metageneration-match: ",
719+
},
720+
{
721+
func() error {
722+
_, err := obj.If(Conditions{MetagenerationMatch: 5}).NewReader(ctx)
723+
return err
724+
},
725+
"x-goog-if-generation-match: , x-goog-if-metageneration-match: 5",
726+
},
727+
{
728+
func() error {
729+
_, err := obj.If(Conditions{GenerationMatch: 1234, MetagenerationMatch: 5}).NewReader(ctx)
730+
return err
731+
},
732+
"x-goog-if-generation-match: 1234, x-goog-if-metageneration-match: 5",
733+
},
734+
}
735+
736+
for i, tt := range readerTests {
737+
if err := tt.fn(); err != nil && err != io.EOF {
738+
t.Error(err)
739+
continue
740+
}
741+
742+
select {
743+
case r := <-gotReq:
744+
generationConds := r.Header.Get("x-goog-if-generation-match")
745+
metagenerationConds := r.Header.Get("x-goog-if-metageneration-match")
746+
got := fmt.Sprintf(
747+
"x-goog-if-generation-match: %s, x-goog-if-metageneration-match: %s",
748+
generationConds,
749+
metagenerationConds,
750+
)
751+
if got != tt.want {
752+
t.Errorf("%d. RequestHeaders = %q; want %q", i, got, tt.want)
753+
}
754+
case <-time.After(5 * time.Second):
755+
t.Fatalf("%d. timeout", i)
756+
}
757+
if err != nil {
758+
t.Fatal(err)
759+
}
760+
}
761+
737762
// Test an error, too:
738763
err = obj.Generation(1234).NewWriter(ctx).Close()
739764
if err == nil || !strings.Contains(err.Error(), "NewWriter: generation not supported") {

0 commit comments

Comments
 (0)