Skip to content

Commit ca8510e

Browse files
BenWhiteheadBenWhitehead
authored
fix: update GrpcStorageImpl handling to be aware of quotaProjectId (#1877)
Add integration tests to verify desired precedence when resolving userProject options. Fixes #1736
1 parent 9220e28 commit ca8510e

5 files changed

Lines changed: 307 additions & 37 deletions

File tree

‎google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageImpl.java‎

Lines changed: 43 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@
6767
import com.google.cloud.storage.UnifiedOpts.ObjectTargetOpt;
6868
import com.google.cloud.storage.UnifiedOpts.Opts;
6969
import com.google.cloud.storage.UnifiedOpts.ProjectId;
70+
import com.google.cloud.storage.UnifiedOpts.UserProject;
7071
import com.google.common.annotations.VisibleForTesting;
7172
import com.google.common.collect.ImmutableList;
7273
import com.google.common.collect.ImmutableSet;
@@ -159,11 +160,15 @@ final class GrpcStorageImpl extends BaseService<StorageOptions> implements Stora
159160
final GrpcRetryAlgorithmManager retryAlgorithmManager;
160161
final SyntaxDecoders syntaxDecoders;
161162

163+
// workaround for https://github.com/googleapis/java-storage/issues/1736
164+
private final Opts<UserProject> defaultOpts;
162165
@Deprecated private final ProjectId defaultProjectId;
163166

164-
GrpcStorageImpl(GrpcStorageOptions options, StorageClient storageClient) {
167+
GrpcStorageImpl(
168+
GrpcStorageOptions options, StorageClient storageClient, Opts<UserProject> defaultOpts) {
165169
super(options);
166170
this.storageClient = storageClient;
171+
this.defaultOpts = defaultOpts;
167172
this.codecs = Conversions.grpc();
168173
this.retryAlgorithmManager = options.getRetryAlgorithmManager();
169174
this.syntaxDecoders = new SyntaxDecoders();
@@ -182,7 +187,7 @@ public void close() throws Exception {
182187

183188
@Override
184189
public Bucket create(BucketInfo bucketInfo, BucketTargetOption... options) {
185-
Opts<BucketTargetOpt> opts = Opts.unwrap(options).resolveFrom(bucketInfo);
190+
Opts<BucketTargetOpt> opts = Opts.unwrap(options).resolveFrom(bucketInfo).prepend(defaultOpts);
186191
GrpcCallContext grpcCallContext =
187192
opts.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
188193
com.google.storage.v2.Bucket bucket = codecs.bucketInfo().encode(bucketInfo);
@@ -215,7 +220,7 @@ public Blob create(
215220
BlobInfo blobInfo, byte[] content, int offset, int length, BlobTargetOption... options) {
216221
requireNonNull(blobInfo, "blobInfo must be non null");
217222
requireNonNull(content, "content must be non null");
218-
Opts<ObjectTargetOpt> opts = Opts.unwrap(options).resolveFrom(blobInfo);
223+
Opts<ObjectTargetOpt> opts = Opts.unwrap(options).resolveFrom(blobInfo).prepend(defaultOpts);
219224
GrpcCallContext grpcCallContext =
220225
opts.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
221226
WriteObjectRequest req = getWriteObjectRequest(blobInfo, opts);
@@ -267,7 +272,7 @@ public Blob createFrom(BlobInfo blobInfo, Path path, int bufferSize, BlobWriteOp
267272
throw new StorageException(0, path + " is a directory");
268273
}
269274

270-
Opts<ObjectTargetOpt> opts = Opts.unwrap(options).resolveFrom(blobInfo);
275+
Opts<ObjectTargetOpt> opts = Opts.unwrap(options).resolveFrom(blobInfo).prepend(defaultOpts);
271276
GrpcCallContext grpcCallContext =
272277
opts.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
273278
WriteObjectRequest req = getWriteObjectRequest(blobInfo, opts);
@@ -335,7 +340,7 @@ public Blob createFrom(
335340
throws IOException {
336341
requireNonNull(blobInfo, "blobInfo must be non null");
337342

338-
Opts<ObjectTargetOpt> opts = Opts.unwrap(options).resolveFrom(blobInfo);
343+
Opts<ObjectTargetOpt> opts = Opts.unwrap(options).resolveFrom(blobInfo).prepend(defaultOpts);
339344
GrpcCallContext grpcCallContext =
340345
opts.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
341346
WriteObjectRequest req = getWriteObjectRequest(blobInfo, opts);
@@ -369,7 +374,7 @@ public Blob createFrom(
369374

370375
@Override
371376
public Bucket get(String bucket, BucketGetOption... options) {
372-
Opts<BucketSourceOpt> opts = Opts.unwrap(options);
377+
Opts<BucketSourceOpt> opts = Opts.unwrap(options).prepend(defaultOpts);
373378
GrpcCallContext grpcCallContext =
374379
opts.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
375380
GetBucketRequest.Builder builder =
@@ -384,7 +389,7 @@ public Bucket get(String bucket, BucketGetOption... options) {
384389

385390
@Override
386391
public Bucket lockRetentionPolicy(BucketInfo bucket, BucketTargetOption... options) {
387-
Opts<BucketTargetOpt> opts = Opts.unwrap(options).resolveFrom(bucket);
392+
Opts<BucketTargetOpt> opts = Opts.unwrap(options).resolveFrom(bucket).prepend(defaultOpts);
388393
GrpcCallContext grpcCallContext =
389394
opts.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
390395
LockBucketRetentionPolicyRequest.Builder builder =
@@ -406,7 +411,7 @@ public Blob get(String bucket, String blob, BlobGetOption... options) {
406411

407412
@Override
408413
public Blob get(BlobId blob, BlobGetOption... options) {
409-
Opts<ObjectSourceOpt> opts = Opts.unwrap(options).resolveFrom(blob);
414+
Opts<ObjectSourceOpt> opts = Opts.unwrap(options).resolveFrom(blob).prepend(defaultOpts);
410415
GrpcCallContext grpcCallContext =
411416
opts.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
412417
GetObjectRequest.Builder builder =
@@ -435,7 +440,7 @@ public Blob get(BlobId blob) {
435440

436441
@Override
437442
public Page<Bucket> list(BucketListOption... options) {
438-
Opts<BucketListOpt> opts = Opts.unwrap(options);
443+
Opts<BucketListOpt> opts = Opts.unwrap(options).prepend(defaultOpts);
439444
GrpcCallContext grpcCallContext =
440445
opts.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
441446
ListBucketsRequest request =
@@ -462,7 +467,7 @@ public Page<Bucket> list(BucketListOption... options) {
462467

463468
@Override
464469
public Page<Blob> list(String bucket, BlobListOption... options) {
465-
Opts<ObjectListOpt> opts = Opts.unwrap(options);
470+
Opts<ObjectListOpt> opts = Opts.unwrap(options).prepend(defaultOpts);
466471
GrpcCallContext grpcCallContext =
467472
opts.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
468473
ListObjectsRequest.Builder builder =
@@ -481,7 +486,7 @@ public Page<Blob> list(String bucket, BlobListOption... options) {
481486

482487
@Override
483488
public Bucket update(BucketInfo bucketInfo, BucketTargetOption... options) {
484-
Opts<BucketTargetOpt> opts = Opts.unwrap(options).resolveFrom(bucketInfo);
489+
Opts<BucketTargetOpt> opts = Opts.unwrap(options).resolveFrom(bucketInfo).prepend(defaultOpts);
485490
GrpcCallContext grpcCallContext =
486491
opts.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
487492
com.google.storage.v2.Bucket bucket = codecs.bucketInfo().encode(bucketInfo);
@@ -503,7 +508,7 @@ public Bucket update(BucketInfo bucketInfo, BucketTargetOption... options) {
503508

504509
@Override
505510
public Blob update(BlobInfo blobInfo, BlobTargetOption... options) {
506-
Opts<ObjectTargetOpt> opts = Opts.unwrap(options).resolveFrom(blobInfo);
511+
Opts<ObjectTargetOpt> opts = Opts.unwrap(options).resolveFrom(blobInfo).prepend(defaultOpts);
507512
GrpcCallContext grpcCallContext =
508513
opts.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
509514
Object object = codecs.blobInfo().encode(blobInfo);
@@ -530,7 +535,7 @@ public Blob update(BlobInfo blobInfo) {
530535

531536
@Override
532537
public boolean delete(String bucket, BucketSourceOption... options) {
533-
Opts<BucketSourceOpt> opts = Opts.unwrap(options);
538+
Opts<BucketSourceOpt> opts = Opts.unwrap(options).prepend(defaultOpts);
534539
GrpcCallContext grpcCallContext =
535540
opts.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
536541
DeleteBucketRequest.Builder builder =
@@ -555,7 +560,7 @@ public boolean delete(String bucket, String blob, BlobSourceOption... options) {
555560

556561
@Override
557562
public boolean delete(BlobId blob, BlobSourceOption... options) {
558-
Opts<ObjectSourceOpt> opts = Opts.unwrap(options).resolveFrom(blob);
563+
Opts<ObjectSourceOpt> opts = Opts.unwrap(options).resolveFrom(blob).prepend(defaultOpts);
559564
GrpcCallContext grpcCallContext =
560565
opts.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
561566
DeleteObjectRequest.Builder builder =
@@ -587,7 +592,9 @@ public boolean delete(BlobId blob) {
587592
@Override
588593
public Blob compose(ComposeRequest composeRequest) {
589594
Opts<ObjectTargetOpt> opts =
590-
Opts.unwrap(composeRequest.getTargetOptions()).resolveFrom(composeRequest.getTarget());
595+
Opts.unwrap(composeRequest.getTargetOptions())
596+
.resolveFrom(composeRequest.getTarget())
597+
.prepend(defaultOpts);
591598
GrpcCallContext grpcCallContext =
592599
opts.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
593600
ComposeObjectRequest.Builder builder = ComposeObjectRequest.newBuilder();
@@ -609,8 +616,12 @@ public CopyWriter copy(CopyRequest copyRequest) {
609616
BlobId src = copyRequest.getSource();
610617
BlobInfo dst = copyRequest.getTarget();
611618
Opts<ObjectSourceOpt> srcOpts =
612-
Opts.unwrap(copyRequest.getSourceOptions()).projectAsSource().resolveFrom(src);
613-
Opts<ObjectTargetOpt> dstOpts = Opts.unwrap(copyRequest.getTargetOptions()).resolveFrom(dst);
619+
Opts.unwrap(copyRequest.getSourceOptions())
620+
.projectAsSource()
621+
.resolveFrom(src)
622+
.prepend(defaultOpts);
623+
Opts<ObjectTargetOpt> dstOpts =
624+
Opts.unwrap(copyRequest.getTargetOptions()).resolveFrom(dst).prepend(defaultOpts);
614625

615626
Mapper<RewriteObjectRequest.Builder> mapper =
616627
srcOpts.rewriteObjectsRequest().andThen(dstOpts.rewriteObjectsRequest());
@@ -684,7 +695,7 @@ public GrpcBlobReadChannel reader(String bucket, String blob, BlobSourceOption..
684695

685696
@Override
686697
public GrpcBlobReadChannel reader(BlobId blob, BlobSourceOption... options) {
687-
Opts<ObjectSourceOpt> opts = Opts.unwrap(options).resolveFrom(blob);
698+
Opts<ObjectSourceOpt> opts = Opts.unwrap(options).resolveFrom(blob).prepend(defaultOpts);
688699
ReadObjectRequest request = getReadObjectRequest(blob, opts);
689700
Set<StatusCode.Code> codes = resultRetryAlgorithmToCodes(retryAlgorithmManager.getFor(request));
690701
GrpcCallContext grpcCallContext = GrpcCallContext.createDefault().withRetryableCodes(codes);
@@ -722,7 +733,7 @@ public void downloadTo(BlobId blob, OutputStream outputStream, BlobSourceOption.
722733

723734
@Override
724735
public GrpcBlobWriteChannel writer(BlobInfo blobInfo, BlobWriteOption... options) {
725-
Opts<ObjectTargetOpt> opts = Opts.unwrap(options).resolveFrom(blobInfo);
736+
Opts<ObjectTargetOpt> opts = Opts.unwrap(options).resolveFrom(blobInfo).prepend(defaultOpts);
726737
GrpcCallContext grpcCallContext =
727738
opts.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
728739
WriteObjectRequest req = getWriteObjectRequest(blobInfo, opts);
@@ -844,7 +855,7 @@ public List<Boolean> delete(Iterable<BlobId> blobIds) {
844855
@Override
845856
public Acl getAcl(String bucket, Entity entity, BucketSourceOption... options) {
846857
try {
847-
Opts<BucketSourceOpt> opts = Opts.unwrap(options);
858+
Opts<BucketSourceOpt> opts = Opts.unwrap(options).prepend(defaultOpts);
848859
com.google.storage.v2.Bucket resp = getBucketWithAcls(bucket, opts);
849860

850861
Predicate<BucketAccessControl> entityPredicate =
@@ -874,7 +885,7 @@ public Acl getAcl(String bucket, Entity entity) {
874885
@Override
875886
public boolean deleteAcl(String bucket, Entity entity, BucketSourceOption... options) {
876887
try {
877-
Opts<BucketSourceOpt> opts = Opts.unwrap(options);
888+
Opts<BucketSourceOpt> opts = Opts.unwrap(options).prepend(defaultOpts);
878889
com.google.storage.v2.Bucket resp = getBucketWithAcls(bucket, opts);
879890
String encode = codecs.entity().encode(entity);
880891

@@ -928,7 +939,7 @@ public Acl createAcl(String bucket, Acl acl) {
928939
@Override
929940
public Acl updateAcl(String bucket, Acl acl, BucketSourceOption... options) {
930941
try {
931-
Opts<BucketSourceOpt> opts = Opts.unwrap(options);
942+
Opts<BucketSourceOpt> opts = Opts.unwrap(options).prepend(defaultOpts);
932943
com.google.storage.v2.Bucket resp = getBucketWithAcls(bucket, opts);
933944
BucketAccessControl encode = codecs.bucketAcl().encode(acl);
934945
String entity = encode.getEntity();
@@ -966,7 +977,7 @@ public Acl updateAcl(String bucket, Acl acl) {
966977
@Override
967978
public List<Acl> listAcls(String bucket, BucketSourceOption... options) {
968979
try {
969-
Opts<BucketSourceOpt> opts = Opts.unwrap(options);
980+
Opts<BucketSourceOpt> opts = Opts.unwrap(options).prepend(defaultOpts);
970981
com.google.storage.v2.Bucket resp = getBucketWithAcls(bucket, opts);
971982
return resp.getAclList().stream()
972983
.map(codecs.bucketAcl()::decode)
@@ -1211,7 +1222,7 @@ public List<Acl> listAcls(BlobId blob) {
12111222

12121223
@Override
12131224
public HmacKey createHmacKey(ServiceAccount serviceAccount, CreateHmacKeyOption... options) {
1214-
Opts<HmacKeyTargetOpt> opts = Opts.unwrap(options);
1225+
Opts<HmacKeyTargetOpt> opts = Opts.unwrap(options).prepend(defaultOpts);
12151226
GrpcCallContext grpcCallContext =
12161227
opts.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
12171228
CreateHmacKeyRequest request =
@@ -1236,7 +1247,7 @@ public HmacKey createHmacKey(ServiceAccount serviceAccount, CreateHmacKeyOption.
12361247

12371248
@Override
12381249
public Page<HmacKeyMetadata> listHmacKeys(ListHmacKeysOption... options) {
1239-
Opts<HmacKeyListOpt> opts = Opts.unwrap(options);
1250+
Opts<HmacKeyListOpt> opts = Opts.unwrap(options).prepend(defaultOpts);
12401251
GrpcCallContext grpcCallContext =
12411252
opts.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
12421253

@@ -1264,7 +1275,7 @@ public Page<HmacKeyMetadata> listHmacKeys(ListHmacKeysOption... options) {
12641275

12651276
@Override
12661277
public HmacKeyMetadata getHmacKey(String accessId, GetHmacKeyOption... options) {
1267-
Opts<HmacKeySourceOpt> opts = Opts.unwrap(options);
1278+
Opts<HmacKeySourceOpt> opts = Opts.unwrap(options).prepend(defaultOpts);
12681279
GrpcCallContext grpcCallContext =
12691280
opts.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
12701281
GetHmacKeyRequest request =
@@ -1283,7 +1294,7 @@ public HmacKeyMetadata getHmacKey(String accessId, GetHmacKeyOption... options)
12831294

12841295
@Override
12851296
public void deleteHmacKey(HmacKeyMetadata hmacKeyMetadata, DeleteHmacKeyOption... options) {
1286-
Opts<HmacKeyTargetOpt> opts = Opts.unwrap(options);
1297+
Opts<HmacKeyTargetOpt> opts = Opts.unwrap(options).prepend(defaultOpts);
12871298
GrpcCallContext grpcCallContext =
12881299
opts.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
12891300
DeleteHmacKeyRequest req =
@@ -1304,7 +1315,7 @@ public void deleteHmacKey(HmacKeyMetadata hmacKeyMetadata, DeleteHmacKeyOption..
13041315
@Override
13051316
public HmacKeyMetadata updateHmacKeyState(
13061317
HmacKeyMetadata hmacKeyMetadata, HmacKeyState state, UpdateHmacKeyOption... options) {
1307-
Opts<HmacKeyTargetOpt> opts = Opts.unwrap(options);
1318+
Opts<HmacKeyTargetOpt> opts = Opts.unwrap(options).prepend(defaultOpts);
13081319
GrpcCallContext grpcCallContext =
13091320
opts.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
13101321
com.google.storage.v2.HmacKeyMetadata encode =
@@ -1323,7 +1334,7 @@ public HmacKeyMetadata updateHmacKeyState(
13231334

13241335
@Override
13251336
public Policy getIamPolicy(String bucket, BucketSourceOption... options) {
1326-
Opts<BucketSourceOpt> opts = Opts.unwrap(options);
1337+
Opts<BucketSourceOpt> opts = Opts.unwrap(options).prepend(defaultOpts);
13271338
GrpcCallContext grpcCallContext =
13281339
opts.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
13291340
GetIamPolicyRequest.Builder builder =
@@ -1338,7 +1349,7 @@ public Policy getIamPolicy(String bucket, BucketSourceOption... options) {
13381349

13391350
@Override
13401351
public Policy setIamPolicy(String bucket, Policy policy, BucketSourceOption... options) {
1341-
Opts<BucketSourceOpt> opts = Opts.unwrap(options);
1352+
Opts<BucketSourceOpt> opts = Opts.unwrap(options).prepend(defaultOpts);
13421353
GrpcCallContext grpcCallContext =
13431354
opts.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
13441355
SetIamPolicyRequest req =
@@ -1356,7 +1367,7 @@ public Policy setIamPolicy(String bucket, Policy policy, BucketSourceOption... o
13561367
@Override
13571368
public List<Boolean> testIamPermissions(
13581369
String bucket, List<String> permissions, BucketSourceOption... options) {
1359-
Opts<BucketSourceOpt> opts = Opts.unwrap(options);
1370+
Opts<BucketSourceOpt> opts = Opts.unwrap(options).prepend(defaultOpts);
13601371
GrpcCallContext grpcCallContext =
13611372
opts.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
13621373
TestIamPermissionsRequest req =
@@ -1706,7 +1717,7 @@ private WriteObjectRequest getWriteObjectRequest(BlobInfo info, Opts<ObjectTarge
17061717

17071718
private UnbufferedReadableByteChannelSession<Object> unbufferedReadSession(
17081719
BlobId blob, BlobSourceOption[] options) {
1709-
Opts<ObjectSourceOpt> opts = Opts.unwrap(options).resolveFrom(blob);
1720+
Opts<ObjectSourceOpt> opts = Opts.unwrap(options).resolveFrom(blob).prepend(defaultOpts);
17101721
ReadObjectRequest readObjectRequest = getReadObjectRequest(blob, opts);
17111722
Set<StatusCode.Code> codes =
17121723
resultRetryAlgorithmToCodes(retryAlgorithmManager.getFor(readObjectRequest));

0 commit comments

Comments
 (0)