feat(storage): add Custom Placement Config Dual Region Support (#6294)

cojenco · noahdietz · commit 455ae028d173 · 2022-07-18T14:26:13.000-07:00
* feat(storage): support Custom Dual Regions with CustomPlacementConfig

* fix typo

* add comments

* address pr comments

* new sublink
diff --git a/storage/bucket.go b/storage/bucket.go
@@ -460,8 +460,13 @@ type BucketAttrs struct {
 	PredefinedDefaultObjectACL string
 
 	// Location is the location of the bucket. It defaults to "US".
+	// If specifying a dual-region, CustomPlacementConfig should be set in conjunction.
 	Location string
 
+	// The bucket's custom placement configuration that holds a list of
+	// regional locations for custom dual regions.
+	CustomPlacementConfig *CustomPlacementConfig
+
 	// MetaGeneration is the metadata generation of the bucket.
 	// This field is read-only.
 	MetaGeneration int64
@@ -781,6 +786,15 @@ type BucketWebsite struct {
 	NotFoundPage string
 }
 
+// CustomPlacementConfig holds the bucket's custom placement
+// configuration for Custom Dual Regions. See
+// https://cloud.google.com/storage/docs/locations#location-dr for more information.
+type CustomPlacementConfig struct {
+	// The list of regional locations in which data is placed.
+	// Custom Dual Regions require exactly 2 regional locations.
+	DataLocations []string
+}
+
 func newBucket(b *raw.Bucket) (*BucketAttrs, error) {
 	if b == nil {
 		return nil, nil
@@ -814,6 +828,7 @@ func newBucket(b *raw.Bucket) (*BucketAttrs, error) {
 		LocationType:             b.LocationType,
 		ProjectNumber:            b.ProjectNumber,
 		RPO:                      toRPO(b),
+		CustomPlacementConfig:    customPlacementFromRaw(b.CustomPlacementConfig),
 	}, nil
 }
 
@@ -844,6 +859,7 @@ func newBucketFromProto(b *storagepb.Bucket) *BucketAttrs {
 		PublicAccessPrevention:   toPublicAccessPreventionFromProto(b.GetIamConfig()),
 		LocationType:             b.GetLocationType(),
 		RPO:                      toRPOFromProto(b),
+		CustomPlacementConfig:    customPlacementFromProto(b.GetCustomPlacementConfig()),
 	}
 }
 
@@ -881,22 +897,23 @@ func (b *BucketAttrs) toRawBucket() *raw.Bucket {
 		}
 	}
 	return &raw.Bucket{
-		Name:             b.Name,
-		Location:         b.Location,
-		StorageClass:     b.StorageClass,
-		Acl:              toRawBucketACL(b.ACL),
-		DefaultObjectAcl: toRawObjectACL(b.DefaultObjectACL),
-		Versioning:       v,
-		Labels:           labels,
-		Billing:          bb,
-		Lifecycle:        toRawLifecycle(b.Lifecycle),
-		RetentionPolicy:  b.RetentionPolicy.toRawRetentionPolicy(),
-		Cors:             toRawCORS(b.CORS),
-		Encryption:       b.Encryption.toRawBucketEncryption(),
-		Logging:          b.Logging.toRawBucketLogging(),
-		Website:          b.Website.toRawBucketWebsite(),
-		IamConfiguration: bktIAM,
-		Rpo:              b.RPO.String(),
+		Name:                  b.Name,
+		Location:              b.Location,
+		StorageClass:          b.StorageClass,
+		Acl:                   toRawBucketACL(b.ACL),
+		DefaultObjectAcl:      toRawObjectACL(b.DefaultObjectACL),
+		Versioning:            v,
+		Labels:                labels,
+		Billing:               bb,
+		Lifecycle:             toRawLifecycle(b.Lifecycle),
+		RetentionPolicy:       b.RetentionPolicy.toRawRetentionPolicy(),
+		Cors:                  toRawCORS(b.CORS),
+		Encryption:            b.Encryption.toRawBucketEncryption(),
+		Logging:               b.Logging.toRawBucketLogging(),
+		Website:               b.Website.toRawBucketWebsite(),
+		IamConfiguration:      bktIAM,
+		Rpo:                   b.RPO.String(),
+		CustomPlacementConfig: b.CustomPlacementConfig.toRawCustomPlacement(),
 	}
 }
 
@@ -939,22 +956,23 @@ func (b *BucketAttrs) toProtoBucket() *storagepb.Bucket {
 	}
 
 	return &storagepb.Bucket{
-		Name:             b.Name,
-		Location:         b.Location,
-		StorageClass:     b.StorageClass,
-		Acl:              toProtoBucketACL(b.ACL),
-		DefaultObjectAcl: toProtoObjectACL(b.DefaultObjectACL),
-		Versioning:       v,
-		Labels:           labels,
-		Billing:          bb,
-		Lifecycle:        toProtoLifecycle(b.Lifecycle),
-		RetentionPolicy:  b.RetentionPolicy.toProtoRetentionPolicy(),
-		Cors:             toProtoCORS(b.CORS),
-		Encryption:       b.Encryption.toProtoBucketEncryption(),
-		Logging:          b.Logging.toProtoBucketLogging(),
-		Website:          b.Website.toProtoBucketWebsite(),
-		IamConfig:        bktIAM,
-		Rpo:              b.RPO.String(),
+		Name:                  b.Name,
+		Location:              b.Location,
+		StorageClass:          b.StorageClass,
+		Acl:                   toProtoBucketACL(b.ACL),
+		DefaultObjectAcl:      toProtoObjectACL(b.DefaultObjectACL),
+		Versioning:            v,
+		Labels:                labels,
+		Billing:               bb,
+		Lifecycle:             toProtoLifecycle(b.Lifecycle),
+		RetentionPolicy:       b.RetentionPolicy.toProtoRetentionPolicy(),
+		Cors:                  toProtoCORS(b.CORS),
+		Encryption:            b.Encryption.toProtoBucketEncryption(),
+		Logging:               b.Logging.toProtoBucketLogging(),
+		Website:               b.Website.toProtoBucketWebsite(),
+		IamConfig:             bktIAM,
+		Rpo:                   b.RPO.String(),
+		CustomPlacementConfig: b.CustomPlacementConfig.toProtoCustomPlacement(),
 	}
 }
 
@@ -1932,6 +1950,38 @@ func toRPOFromProto(b *storagepb.Bucket) RPO {
 	}
 }
 
+func customPlacementFromRaw(c *raw.BucketCustomPlacementConfig) *CustomPlacementConfig {
+	if c == nil {
+		return nil
+	}
+	return &CustomPlacementConfig{DataLocations: c.DataLocations}
+}
+
+func (c *CustomPlacementConfig) toRawCustomPlacement() *raw.BucketCustomPlacementConfig {
+	if c == nil {
+		return nil
+	}
+	return &raw.BucketCustomPlacementConfig{
+		DataLocations: c.DataLocations,
+	}
+}
+
+func (c *CustomPlacementConfig) toProtoCustomPlacement() *storagepb.Bucket_CustomPlacementConfig {
+	if c == nil {
+		return nil
+	}
+	return &storagepb.Bucket_CustomPlacementConfig{
+		DataLocations: c.DataLocations,
+	}
+}
+
+func customPlacementFromProto(c *storagepb.Bucket_CustomPlacementConfig) *CustomPlacementConfig {
+	if c == nil {
+		return nil
+	}
+	return &CustomPlacementConfig{DataLocations: c.GetDataLocations()}
+}
+
 // Objects returns an iterator over the objects in the bucket that match the
 // Query q. If q is nil, no filtering is done. Objects will be iterated over
 // lexicographically by name.
diff --git a/storage/integration_test.go b/storage/integration_test.go
@@ -314,12 +314,13 @@ func TestIntegration_BucketCreateDelete(t *testing.T) {
 
 	// testedAttrs are the bucket attrs directly compared in this test
 	type testedAttrs struct {
-		StorageClass      string
-		VersioningEnabled bool
-		LocationType      string
-		Labels            map[string]string
-		Location          string
-		Lifecycle         Lifecycle
+		StorageClass          string
+		VersioningEnabled     bool
+		LocationType          string
+		Labels                map[string]string
+		Location              string
+		Lifecycle             Lifecycle
+		CustomPlacementConfig *CustomPlacementConfig
 	}
 
 	for _, test := range []struct {
@@ -358,12 +359,18 @@ func TestIntegration_BucketCreateDelete(t *testing.T) {
 		{
 			name: "dual-region",
 			attrs: &BucketAttrs{
-				Location: "US-EAST1+US-WEST1",
+				Location: "US",
+				CustomPlacementConfig: &CustomPlacementConfig{
+					DataLocations: []string{"US-EAST1", "US-WEST1"},
+				},
 			},
 			wantAttrs: testedAttrs{
-				Location:     "US-EAST1+US-WEST1",
+				Location:     "US",
 				LocationType: "dual-region",
 				StorageClass: "STANDARD",
+				CustomPlacementConfig: &CustomPlacementConfig{
+					DataLocations: []string{"US-EAST1", "US-WEST1"},
+				},
 			},
 		},
 	} {
@@ -407,6 +414,9 @@ func TestIntegration_BucketCreateDelete(t *testing.T) {
 			if gotAttrs.Location != test.wantAttrs.Location {
 				t.Errorf("location: got %s, want %s", gotAttrs.Location, test.wantAttrs.Location)
 			}
+			if got, want := gotAttrs.CustomPlacementConfig, test.wantAttrs.CustomPlacementConfig; !testutil.Equal(got, want) {
+				t.Errorf("customPlacementConfig: \ngot\t%v\nwant\t%v", got, want)
+			}
 
 			// Delete the bucket and check that the deletion was succesful
 			if err := b.Delete(ctx); err != nil {
