Skip to content

Commit fc3a2d7

Browse files
mziccardmziccard
committed
Add batch get, update, delete methods to storage
- Update StorageRpc.BatchRequest to check for null to* lists - Add storage.get, storage.update and storage.delete methods - Add unit tests to StorageImplTest - Add integration tests to ITStorageTest
1 parent 92b6fce commit fc3a2d7

5 files changed

Lines changed: 331 additions & 3 deletions

File tree

‎gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616

1717
package com.google.gcloud.spi;
1818

19+
import static com.google.common.base.MoreObjects.firstNonNull;
20+
1921
import com.google.api.services.storage.model.Bucket;
2022
import com.google.api.services.storage.model.StorageObject;
2123
import com.google.common.collect.ImmutableList;
@@ -106,9 +108,12 @@ class BatchRequest {
106108
public BatchRequest(Iterable<Tuple<StorageObject, Map<Option, ?>>> toDelete,
107109
Iterable<Tuple<StorageObject, Map<Option, ?>>> toUpdate,
108110
Iterable<Tuple<StorageObject, Map<Option, ?>>> toGet) {
109-
this.toDelete = ImmutableList.copyOf(toDelete);
110-
this.toUpdate = ImmutableList.copyOf(toUpdate);
111-
this.toGet = ImmutableList.copyOf(toGet);
111+
this.toDelete = ImmutableList.copyOf(
112+
firstNonNull(toDelete, ImmutableList.<Tuple<StorageObject, Map<Option, ?>>>of()));
113+
this.toUpdate = ImmutableList.copyOf(
114+
firstNonNull(toUpdate, ImmutableList.<Tuple<StorageObject, Map<Option, ?>>>of()));
115+
this.toGet = ImmutableList.copyOf(
116+
firstNonNull(toGet, ImmutableList.<Tuple<StorageObject, Map<Option, ?>>>of()));
112117
}
113118
}
114119

‎gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -638,4 +638,37 @@ public static Builder builder() {
638638
* @see <a href="https://cloud.google.com/storage/docs/access-control#Signed-URLs">Signed-URLs</a>
639639
*/
640640
URL signUrl(BlobInfo blobInfo, long expirationTimeInSeconds, SignUrlOption... options);
641+
642+
/**
643+
* Gets the requested blobs. A batch request is used to perform this call.
644+
*
645+
* @param blobInfo1 first blob to get
646+
* @param blobInfo2 second blob to get
647+
* @param blobInfos other blobs to get
648+
* @return a list of {@code BlobInfo} objects. If a blob does not exist the corresponding item in
649+
* the list is {@code null}.
650+
*/
651+
List<BlobInfo> get(BlobInfo blobInfo1, BlobInfo blobInfo2, BlobInfo... blobInfos);
652+
653+
/**
654+
* Updates the requested blobs. A batch request is used to perform this call.
655+
*
656+
* @param blobInfo1 first blob to update
657+
* @param blobInfo2 second blob to update
658+
* @param blobInfos other blobs to update
659+
* @return a list of {@code BlobInfo} objects. If a blob does not exist the corresponding item in
660+
* the list is {@code null}.
661+
*/
662+
List<BlobInfo> update(BlobInfo blobInfo1, BlobInfo blobInfo2, BlobInfo... blobInfos);
663+
664+
/**
665+
* Deletes the requested blobs. A batch request is used to perform this call.
666+
*
667+
* @param blobInfo1 first blob to delete
668+
* @param blobInfo2 second blob to delete
669+
* @param blobInfos other blobs to delete
670+
* @return a list of booleans. If a blob has been deleted the corresponding item in the list is
671+
* {@code true}. If deletion failed the item is {@code false}.
672+
*/
673+
List<Boolean> delete(BlobInfo blobInfo1, BlobInfo blobInfo2, BlobInfo... blobInfos);
641674
}

‎gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -577,6 +577,68 @@ public URL signUrl(BlobInfo blobInfo, long expiration, SignUrlOption... options)
577577
}
578578
}
579579

580+
@Override
581+
public List<BlobInfo> get(BlobInfo blobInfo1, BlobInfo blobInfo2, BlobInfo... blobInfos) {
582+
List<Tuple<StorageObject, Map<StorageRpc.Option, ?>>> toGet =
583+
Lists.newArrayListWithCapacity(blobInfos.length + 2);
584+
toGet.add(Tuple.<StorageObject, Map<StorageRpc.Option, ?>>of(blobInfo1.toPb(), optionMap()));
585+
toGet.add(Tuple.<StorageObject, Map<StorageRpc.Option, ?>>of(blobInfo2.toPb(), optionMap()));
586+
for (BlobInfo blobInfo : blobInfos) {
587+
toGet.add(Tuple.<StorageObject, Map<StorageRpc.Option, ?>>of(blobInfo.toPb(), optionMap()));
588+
}
589+
StorageRpc.BatchResponse response =
590+
storageRpc.batch(new StorageRpc.BatchRequest(null, null, toGet));
591+
return transformBatchResult(toGet, response.gets, BlobInfo.FROM_PB_FUNCTION, (BlobInfo) null);
592+
}
593+
594+
@Override
595+
public List<BlobInfo> update(BlobInfo blobInfo1, BlobInfo blobInfo2, BlobInfo... blobInfos) {
596+
List<Tuple<StorageObject, Map<StorageRpc.Option, ?>>> toUpdate =
597+
Lists.newArrayListWithCapacity(blobInfos.length + 2);
598+
toUpdate.add(Tuple.<StorageObject, Map<StorageRpc.Option, ?>>of(blobInfo1.toPb(), optionMap()));
599+
toUpdate.add(Tuple.<StorageObject, Map<StorageRpc.Option, ?>>of(blobInfo2.toPb(), optionMap()));
600+
for (BlobInfo blobInfo : blobInfos) {
601+
toUpdate
602+
.add(Tuple.<StorageObject, Map<StorageRpc.Option, ?>>of(blobInfo.toPb(), optionMap()));
603+
}
604+
StorageRpc.BatchResponse response =
605+
storageRpc.batch(new StorageRpc.BatchRequest(null, toUpdate, null));
606+
return transformBatchResult(toUpdate, response.updates, BlobInfo.FROM_PB_FUNCTION,
607+
(BlobInfo) null);
608+
}
609+
610+
@Override
611+
public List<Boolean> delete(BlobInfo blobInfo1, BlobInfo blobInfo2, BlobInfo... blobInfos) {
612+
List<Tuple<StorageObject, Map<StorageRpc.Option, ?>>> toDelete =
613+
Lists.newArrayListWithCapacity(blobInfos.length + 2);
614+
toDelete.add(Tuple.<StorageObject, Map<StorageRpc.Option, ?>>of(blobInfo1.toPb(), optionMap()));
615+
toDelete.add(Tuple.<StorageObject, Map<StorageRpc.Option, ?>>of(blobInfo2.toPb(), optionMap()));
616+
for (BlobInfo blobInfo : blobInfos) {
617+
toDelete
618+
.add(Tuple.<StorageObject, Map<StorageRpc.Option, ?>>of(blobInfo.toPb(), optionMap()));
619+
}
620+
StorageRpc.BatchResponse response =
621+
storageRpc.batch(new StorageRpc.BatchRequest(toDelete, null, null));
622+
return transformBatchResult(toDelete, response.deletes, Functions.<Boolean>identity(),
623+
Boolean.FALSE);
624+
}
625+
626+
private <I, O extends Serializable> List<O> transformBatchResult(
627+
Iterable<Tuple<StorageObject, Map<StorageRpc.Option, ?>>> request,
628+
Map<StorageObject, Tuple<I, StorageException>> results, Function<I, O> transform,
629+
O errorValue) {
630+
List<O> response = Lists.newArrayListWithCapacity(results.size());
631+
for (Tuple<StorageObject, ?> tuple : request) {
632+
Tuple<I, StorageException> result = results.get(tuple.x());
633+
if (result.x() != null) {
634+
response.add(transform.apply(result.x()));
635+
} else {
636+
response.add(errorValue);
637+
}
638+
}
639+
return response;
640+
}
641+
580642
private Map<StorageRpc.Option, ?> optionMap(Long generation, Long metaGeneration,
581643
Iterable<? extends Option> options) {
582644
return optionMap(generation, metaGeneration, options, false);

‎gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import static org.junit.Assert.assertArrayEquals;
2121
import static org.junit.Assert.assertEquals;
2222
import static org.junit.Assert.assertNotNull;
23+
import static org.junit.Assert.assertNull;
2324
import static org.junit.Assert.assertTrue;
2425
import static org.junit.Assert.fail;
2526

@@ -36,6 +37,7 @@
3637
import java.util.Arrays;
3738
import java.util.Calendar;
3839
import java.util.Iterator;
40+
import java.util.List;
3941
import java.util.concurrent.ExecutionException;
4042
import java.util.concurrent.TimeUnit;
4143
import java.util.concurrent.TimeoutException;
@@ -464,4 +466,101 @@ public void testPostSignedUrl() throws IOException {
464466
assertEquals(blob.name(), remoteBlob.name());
465467
assertTrue(storage.delete(bucket, blobName));
466468
}
469+
470+
@Test
471+
public void testGetBlobs() {
472+
String sourceBlobName1 = "test-get-blobs-1";
473+
String sourceBlobName2 = "test-get-blobs-2";
474+
BlobInfo sourceBlob1 = BlobInfo.of(bucket, sourceBlobName1);
475+
BlobInfo sourceBlob2 = BlobInfo.of(bucket, sourceBlobName2);
476+
assertNotNull(storage.create(sourceBlob1));
477+
assertNotNull(storage.create(sourceBlob2));
478+
List<BlobInfo> remoteInfos = storage.get(sourceBlob1, sourceBlob2);
479+
assertEquals(sourceBlob1.bucket(), remoteInfos.get(0).bucket());
480+
assertEquals(sourceBlob1.name(), remoteInfos.get(0).name());
481+
assertEquals(sourceBlob2.bucket(), remoteInfos.get(1).bucket());
482+
assertEquals(sourceBlob2.name(), remoteInfos.get(1).name());
483+
assertTrue(storage.delete(bucket, sourceBlobName1));
484+
assertTrue(storage.delete(bucket, sourceBlobName2));
485+
}
486+
487+
@Test
488+
public void testGetBlobsFail() {
489+
String sourceBlobName1 = "test-get-blobs-fail-1";
490+
String sourceBlobName2 = "test-get-blobs-fail-2";
491+
BlobInfo sourceBlob1 = BlobInfo.of(bucket, sourceBlobName1);
492+
BlobInfo sourceBlob2 = BlobInfo.of(bucket, sourceBlobName2);
493+
assertNotNull(storage.create(sourceBlob1));
494+
List<BlobInfo> remoteBlobs = storage.get(sourceBlob1, sourceBlob2);
495+
assertEquals(sourceBlob1.bucket(), remoteBlobs.get(0).bucket());
496+
assertEquals(sourceBlob1.name(), remoteBlobs.get(0).name());
497+
assertNull(remoteBlobs.get(1));
498+
assertTrue(storage.delete(bucket, sourceBlobName1));
499+
}
500+
501+
@Test
502+
public void testDeleteBlobs() {
503+
String sourceBlobName1 = "test-delete-blobs-1";
504+
String sourceBlobName2 = "test-delete-blobs-2";
505+
BlobInfo sourceBlob1 = BlobInfo.of(bucket, sourceBlobName1);
506+
BlobInfo sourceBlob2 = BlobInfo.of(bucket, sourceBlobName2);
507+
assertNotNull(storage.create(sourceBlob1));
508+
assertNotNull(storage.create(sourceBlob2));
509+
List<Boolean> deleteStatus = storage.delete(sourceBlob1, sourceBlob2);
510+
assertTrue(deleteStatus.get(0));
511+
assertTrue(deleteStatus.get(1));
512+
}
513+
514+
@Test
515+
public void testDeleteBlobsFail() {
516+
String sourceBlobName1 = "test-delete-blobs-fail-1";
517+
String sourceBlobName2 = "test-delete-blobs-fail-2";
518+
BlobInfo sourceBlob1 = BlobInfo.of(bucket, sourceBlobName1);
519+
BlobInfo sourceBlob2 = BlobInfo.of(bucket, sourceBlobName2);
520+
assertNotNull(storage.create(sourceBlob1));
521+
List<Boolean> deleteStatus = storage.delete(sourceBlob1, sourceBlob2);
522+
assertTrue(deleteStatus.get(0));
523+
assertTrue(!deleteStatus.get(1));
524+
}
525+
526+
@Test
527+
public void testUpdateBlobs() {
528+
String sourceBlobName1 = "test-update-blobs-1";
529+
String sourceBlobName2 = "test-update-blobs-2";
530+
BlobInfo sourceBlob1 = BlobInfo.of(bucket, sourceBlobName1);
531+
BlobInfo sourceBlob2 = BlobInfo.of(bucket, sourceBlobName2);
532+
BlobInfo remoteBlob1 = storage.create(sourceBlob1);
533+
BlobInfo remoteBlob2 = storage.create(sourceBlob2);
534+
assertNotNull(remoteBlob1);
535+
assertNotNull(remoteBlob2);
536+
List<BlobInfo> updatedBlobs = storage.update(
537+
remoteBlob1.toBuilder().contentType(CONTENT_TYPE).build(),
538+
remoteBlob2.toBuilder().contentType(CONTENT_TYPE).build());
539+
assertEquals(sourceBlob1.bucket(), updatedBlobs.get(0).bucket());
540+
assertEquals(sourceBlob1.name(), updatedBlobs.get(0).name());
541+
assertEquals(CONTENT_TYPE, updatedBlobs.get(0).contentType());
542+
assertEquals(sourceBlob2.bucket(), updatedBlobs.get(1).bucket());
543+
assertEquals(sourceBlob2.name(), updatedBlobs.get(1).name());
544+
assertEquals(CONTENT_TYPE, updatedBlobs.get(1).contentType());
545+
assertTrue(storage.delete(bucket, sourceBlobName1));
546+
assertTrue(storage.delete(bucket, sourceBlobName2));
547+
}
548+
549+
@Test
550+
public void testUpdateBlobsFail() {
551+
String sourceBlobName1 = "test-update-blobs-fail-1";
552+
String sourceBlobName2 = "test-update-blobs-fail-2";
553+
BlobInfo sourceBlob1 = BlobInfo.of(bucket, sourceBlobName1);
554+
BlobInfo sourceBlob2 = BlobInfo.of(bucket, sourceBlobName2);
555+
BlobInfo remoteBlob1 = storage.create(sourceBlob1);
556+
assertNotNull(remoteBlob1);
557+
List<BlobInfo> updatedBlobs = storage.update(
558+
remoteBlob1.toBuilder().contentType(CONTENT_TYPE).build(),
559+
sourceBlob2.toBuilder().contentType(CONTENT_TYPE).build());
560+
assertEquals(sourceBlob1.bucket(), updatedBlobs.get(0).bucket());
561+
assertEquals(sourceBlob1.name(), updatedBlobs.get(0).name());
562+
assertEquals(CONTENT_TYPE, updatedBlobs.get(0).contentType());
563+
assertNull(updatedBlobs.get(1));
564+
assertTrue(storage.delete(bucket, sourceBlobName1));
565+
}
467566
}

‎gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -857,6 +857,135 @@ public void testSignUrlWithOptions() throws NoSuchAlgorithmException, InvalidKey
857857
EasyMock.verify(credentialsMock);
858858
}
859859

860+
@Test
861+
public void testGetAll() {
862+
BlobInfo blobInfo1 = BlobInfo.of(BUCKET_NAME1, BLOB_NAME1);
863+
BlobInfo blobInfo2 = BlobInfo.of(BUCKET_NAME1, BLOB_NAME2);
864+
StorageObject storageObject1 = blobInfo1.toPb();
865+
StorageObject storageObject2 = blobInfo2.toPb();
866+
List<StorageObject> toGet = ImmutableList.of(storageObject1, storageObject2);
867+
868+
Map<StorageObject, Tuple<Boolean, StorageException>> deleteResult = ImmutableMap.of();
869+
Map<StorageObject, Tuple<StorageObject, StorageException>> updateResult = ImmutableMap.of();
870+
Map<StorageObject, Tuple<StorageObject, StorageException>> getResult = Maps.toMap(toGet,
871+
new Function<StorageObject, Tuple<StorageObject, StorageException>>() {
872+
@Override
873+
public Tuple<StorageObject, StorageException> apply(StorageObject f) {
874+
return Tuple.of(f, null);
875+
}
876+
});
877+
StorageRpc.BatchResponse res =
878+
new StorageRpc.BatchResponse(deleteResult, updateResult, getResult);
879+
880+
EasyMock.expect(optionsMock.storageRpc()).andReturn(storageRpcMock);
881+
Capture<StorageRpc.BatchRequest> capturedBatchRequest = Capture.newInstance();
882+
EasyMock.expect(storageRpcMock.batch(EasyMock.capture(capturedBatchRequest))).andReturn(res);
883+
EasyMock.replay(optionsMock, storageRpcMock);
884+
storage = StorageFactory.instance().get(optionsMock);
885+
List<BlobInfo> resultBlobs = storage.get(blobInfo1, blobInfo2);
886+
887+
// Verify captured StorageRpc.BatchRequest
888+
List<Tuple<StorageObject, Map<StorageRpc.Option, ?>>> capturedToGet =
889+
capturedBatchRequest.getValue().toGet;
890+
assertTrue(capturedBatchRequest.getValue().toDelete.isEmpty());
891+
assertTrue(capturedBatchRequest.getValue().toUpdate.isEmpty());
892+
for (int i = 0; i < capturedToGet.size(); i++) {
893+
assertEquals(toGet.get(i), capturedToGet.get(i).x());
894+
assertTrue(capturedToGet.get(i).y().isEmpty());
895+
}
896+
897+
// Verify result
898+
for (int i = 0; i < resultBlobs.size(); i++) {
899+
assertEquals(toGet.get(i), resultBlobs.get(i).toPb());
900+
}
901+
}
902+
903+
@Test
904+
public void testUpdateAll() {
905+
BlobInfo blobInfo1 = BlobInfo.builder(BUCKET_NAME1, BLOB_NAME1).contentType("type").build();
906+
BlobInfo blobInfo2 = BlobInfo.builder(BUCKET_NAME1, BLOB_NAME2).contentType("type").build();
907+
StorageObject storageObject1 = blobInfo1.toPb();
908+
StorageObject storageObject2 = blobInfo2.toPb();
909+
List<StorageObject> toUpdate = ImmutableList.of(storageObject1, storageObject2);
910+
911+
Map<StorageObject, Tuple<Boolean, StorageException>> deleteResult = ImmutableMap.of();
912+
Map<StorageObject, Tuple<StorageObject, StorageException>> getResult = ImmutableMap.of();
913+
Map<StorageObject, Tuple<StorageObject, StorageException>> updateResult = Maps.toMap(toUpdate,
914+
new Function<StorageObject, Tuple<StorageObject, StorageException>>() {
915+
@Override
916+
public Tuple<StorageObject, StorageException> apply(StorageObject f) {
917+
return Tuple.of(f, null);
918+
}
919+
});
920+
StorageRpc.BatchResponse res =
921+
new StorageRpc.BatchResponse(deleteResult, updateResult, getResult);
922+
923+
EasyMock.expect(optionsMock.storageRpc()).andReturn(storageRpcMock);
924+
Capture<StorageRpc.BatchRequest> capturedBatchRequest = Capture.newInstance();
925+
EasyMock.expect(storageRpcMock.batch(EasyMock.capture(capturedBatchRequest))).andReturn(res);
926+
EasyMock.replay(optionsMock, storageRpcMock);
927+
storage = StorageFactory.instance().get(optionsMock);
928+
List<BlobInfo> resultBlobs = storage.update(blobInfo1, blobInfo2);
929+
930+
// Verify captured StorageRpc.BatchRequest
931+
List<Tuple<StorageObject, Map<StorageRpc.Option, ?>>> capturedToUpdate =
932+
capturedBatchRequest.getValue().toUpdate;
933+
assertTrue(capturedBatchRequest.getValue().toDelete.isEmpty());
934+
assertTrue(capturedBatchRequest.getValue().toGet.isEmpty());
935+
for (int i = 0; i < capturedToUpdate.size(); i++) {
936+
assertEquals(toUpdate.get(i), capturedToUpdate.get(i).x());
937+
assertTrue(capturedToUpdate.get(i).y().isEmpty());
938+
}
939+
940+
// Verify result
941+
for (int i = 0; i < resultBlobs.size(); i++) {
942+
assertEquals(toUpdate.get(i), resultBlobs.get(i).toPb());
943+
}
944+
}
945+
946+
@Test
947+
public void testDeleteAll() {
948+
BlobInfo blobInfo1 = BlobInfo.builder(BUCKET_NAME1, BLOB_NAME1).contentType("type").build();
949+
BlobInfo blobInfo2 = BlobInfo.builder(BUCKET_NAME1, BLOB_NAME2).contentType("type").build();
950+
StorageObject storageObject1 = blobInfo1.toPb();
951+
StorageObject storageObject2 = blobInfo2.toPb();
952+
List<StorageObject> toUpdate = ImmutableList.of(storageObject1, storageObject2);
953+
954+
Map<StorageObject, Tuple<StorageObject, StorageException>> updateResult = ImmutableMap.of();
955+
Map<StorageObject, Tuple<StorageObject, StorageException>> getResult = ImmutableMap.of();
956+
Map<StorageObject, Tuple<Boolean, StorageException>> deleteResult = Maps.toMap(toUpdate,
957+
new Function<StorageObject, Tuple<Boolean, StorageException>>() {
958+
@Override
959+
public Tuple<Boolean, StorageException> apply(StorageObject f) {
960+
return Tuple.of(true, null);
961+
}
962+
});
963+
StorageRpc.BatchResponse res =
964+
new StorageRpc.BatchResponse(deleteResult, updateResult, getResult);
965+
966+
EasyMock.expect(optionsMock.storageRpc()).andReturn(storageRpcMock);
967+
Capture<StorageRpc.BatchRequest> capturedBatchRequest = Capture.newInstance();
968+
EasyMock.expect(storageRpcMock.batch(EasyMock.capture(capturedBatchRequest))).andReturn(res);
969+
EasyMock.replay(optionsMock, storageRpcMock);
970+
storage = StorageFactory.instance().get(optionsMock);
971+
List<Boolean> deleteResults = storage.delete(blobInfo1, blobInfo2);
972+
973+
// Verify captured StorageRpc.BatchRequest
974+
List<Tuple<StorageObject, Map<StorageRpc.Option, ?>>> capturedToDelete =
975+
capturedBatchRequest.getValue().toDelete;
976+
assertTrue(capturedBatchRequest.getValue().toUpdate.isEmpty());
977+
assertTrue(capturedBatchRequest.getValue().toGet.isEmpty());
978+
for (int i = 0; i < capturedToDelete.size(); i++) {
979+
assertEquals(toUpdate.get(i), capturedToDelete.get(i).x());
980+
assertTrue(capturedToDelete.get(i).y().isEmpty());
981+
}
982+
983+
// Verify result
984+
for (Boolean deleteStatus : deleteResults) {
985+
assertTrue(deleteStatus);
986+
}
987+
}
988+
860989
@Test
861990
public void testRetryableException() {
862991
BlobInfo blob = BlobInfo.of(BUCKET_NAME1, BLOB_NAME1);

0 commit comments

Comments
 (0)