Skip to content

Commit 352e097

Browse files
mziccardmziccard
committed
Refactor CopyWriter, Storage and StorageRpc
- remove StorageRpc.copy - make CopyWriter.result issue requests until copy completes - make RewriteResponse and CopyWriter blobSize isDone and totalBytesCopied primitive - rename Storage.RewriteRequest megabytesRewrittenPerCall to megabytesCopiedPerChunk - update tests - update StorageExample
1 parent f90e70c commit 352e097

11 files changed

Lines changed: 94 additions & 137 deletions

File tree

‎gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -368,9 +368,6 @@ private static class CopyAction extends StorageAction<CopyRequest> {
368368
@Override
369369
public void run(Storage storage, CopyRequest request) {
370370
CopyWriter copyWriter = storage.copy(request);
371-
while (!copyWriter.isDone()) {
372-
copyWriter.copyChunk();
373-
}
374371
System.out.println("Copied " + copyWriter.result());
375372
}
376373

‎gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java‎

Lines changed: 0 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -321,30 +321,6 @@ public StorageObject compose(Iterable<StorageObject> sources, StorageObject targ
321321
}
322322
}
323323

324-
@Override
325-
public StorageObject copy(StorageObject source, Map<Option, ?> sourceOptions,
326-
StorageObject target, Map<Option, ?> targetOptions) throws StorageException {
327-
try {
328-
return storage
329-
.objects()
330-
.copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(),
331-
target.getContentType() != null ? target : null)
332-
.setProjection(DEFAULT_PROJECTION)
333-
.setIfSourceMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions))
334-
.setIfSourceMetagenerationNotMatch(
335-
IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions))
336-
.setIfSourceGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions))
337-
.setIfSourceGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions))
338-
.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
339-
.setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions))
340-
.setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
341-
.setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions))
342-
.execute();
343-
} catch (IOException ex) {
344-
throw translate(ex);
345-
}
346-
}
347-
348324
@Override
349325
public byte[] load(StorageObject from, Map<Option, ?> options)
350326
throws StorageException {

‎gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java‎

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -177,13 +177,13 @@ class RewriteResponse {
177177

178178
public final RewriteRequest rewriteRequest;
179179
public final StorageObject result;
180-
public final Long blobSize;
181-
public final Boolean isDone;
180+
public final long blobSize;
181+
public final boolean isDone;
182182
public final String rewriteToken;
183-
public final Long totalBytesRewritten;
183+
public final long totalBytesRewritten;
184184

185-
public RewriteResponse(RewriteRequest rewriteRequest, StorageObject result, Long blobSize,
186-
Boolean isDone, String rewriteToken, Long totalBytesRewritten) {
185+
public RewriteResponse(RewriteRequest rewriteRequest, StorageObject result, long blobSize,
186+
boolean isDone, String rewriteToken, long totalBytesRewritten) {
187187
this.rewriteRequest = rewriteRequest;
188188
this.result = result;
189189
this.blobSize = blobSize;
@@ -204,9 +204,9 @@ public boolean equals(Object obj) {
204204
return Objects.equals(this.rewriteRequest, other.rewriteRequest)
205205
&& Objects.equals(this.result, other.result)
206206
&& Objects.equals(this.rewriteToken, other.rewriteToken)
207-
&& Objects.equals(this.blobSize, other.blobSize)
207+
&& this.blobSize == other.blobSize
208208
&& Objects.equals(this.isDone, other.isDone)
209-
&& Objects.equals(this.totalBytesRewritten, other.totalBytesRewritten);
209+
&& this.totalBytesRewritten == other.totalBytesRewritten;
210210
}
211211

212212
@Override
@@ -245,9 +245,6 @@ StorageObject patch(StorageObject storageObject, Map<Option, ?> options)
245245
StorageObject compose(Iterable<StorageObject> sources, StorageObject target,
246246
Map<Option, ?> targetOptions) throws StorageException;
247247

248-
StorageObject copy(StorageObject source, Map<Option, ?> sourceOptions,
249-
StorageObject target, Map<Option, ?> targetOptions) throws StorageException;
250-
251248
byte[] load(StorageObject storageObject, Map<Option, ?> options)
252249
throws StorageException;
253250

‎gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -223,7 +223,7 @@ public Blob update(BlobInfo blobInfo, BlobTargetOption... options) {
223223
* @throws StorageException upon failure
224224
*/
225225
public CopyWriter copyTo(BlobId targetBlob, BlobSourceOption... options) {
226-
BlobInfo updatedInfo = info.toBuilder().blobId(targetBlob).build();
226+
BlobInfo updatedInfo = BlobInfo.builder(targetBlob).build();
227227
CopyRequest copyRequest = CopyRequest.builder().source(info.bucket(), info.name())
228228
.sourceOptions(convert(info, options)).target(updatedInfo).build();
229229
return storage.copy(copyRequest);
@@ -266,7 +266,7 @@ public CopyWriter copyTo(String targetBucket, BlobSourceOption... options) {
266266
* @throws StorageException upon failure
267267
*/
268268
public CopyWriter copyTo(String targetBucket, String targetBlob, BlobSourceOption... options) {
269-
BlobInfo updatedInfo = info.toBuilder().blobId(BlobId.of(targetBucket, targetBlob)).build();
269+
BlobInfo updatedInfo = BlobInfo.builder(targetBucket, targetBlob).build();
270270
CopyRequest copyRequest = CopyRequest.builder().source(info.bucket(), info.name())
271271
.sourceOptions(convert(info, options)).target(updatedInfo).build();
272272
return storage.copy(copyRequest);

‎gcloud-java-storage/src/main/java/com/google/gcloud/storage/CopyWriter.java‎

Lines changed: 34 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,8 @@
3434
/**
3535
* Google Storage blob copy writer. This class holds the result of a copy request.
3636
* If source and destination blobs do not share the same location or storage class more than one
37-
* RPC request is needed to copy the blob. When this is the case {@link #copyChunk()} can be used
38-
* to copy to destination other chunks of the source blob.
37+
* RPC request is needed to copy the blob otherwise one or more {@link #copyChunk()} calls are
38+
* necessary to complete the copy.
3939
*
4040
* @see <a href="https://cloud.google.com/storage/docs/json_api/v1/objects/rewrite">Rewrite</a>
4141
*/
@@ -52,31 +52,36 @@ public class CopyWriter implements Restorable<CopyWriter> {
5252
}
5353

5454
/**
55-
* Returns the updated information for the just written blob when {@link #isDone} is {@code true}.
56-
* Returns {@code null} otherwise.
55+
* Returns the updated information for the just written blob. This method might block and issue
56+
* several RPC requests to complete blob copy.
57+
*
58+
* @throws StorageException upon failure
5759
*/
5860
public BlobInfo result() {
59-
return rewriteResponse.result != null ? BlobInfo.fromPb(rewriteResponse.result) : null;
61+
while(!isDone()) {
62+
copyChunk();
63+
}
64+
return BlobInfo.fromPb(rewriteResponse.result);
6065
}
6166

6267
/**
6368
* Size of the blob being copied.
6469
*/
65-
public Long blobSize() {
70+
public long blobSize() {
6671
return rewriteResponse.blobSize;
6772
}
6873

6974
/**
7075
* Returns {@code true} of blob rewrite finished, {@code false} otherwise.
7176
*/
72-
public Boolean isDone() {
77+
public boolean isDone() {
7378
return rewriteResponse.isDone;
7479
}
7580

7681
/**
7782
* Returns the number of bytes copied.
7883
*/
79-
public Long totalBytesCopied() {
84+
public long totalBytesCopied() {
8085
return rewriteResponse.totalBytesRewritten;
8186
}
8287

@@ -111,7 +116,7 @@ public RestorableState<CopyWriter> capture() {
111116
rewriteResponse.rewriteRequest.targetOptions)
112117
.blobSize(blobSize())
113118
.isDone(isDone())
114-
.megabytesRewrittenPerCall(rewriteResponse.rewriteRequest.megabytesRewrittenPerCall)
119+
.megabytesCopiedPerChunk(rewriteResponse.rewriteRequest.megabytesRewrittenPerCall)
115120
.rewriteToken(rewriteResponse.rewriteToken)
116121
.totalBytesRewritten(totalBytesCopied())
117122
.build();
@@ -127,11 +132,11 @@ static class StateImpl implements RestorableState<CopyWriter>, Serializable {
127132
private final BlobInfo target;
128133
private final Map<StorageRpc.Option, ?> targetOptions;
129134
private final BlobInfo result;
130-
private final Long blobSize;
131-
private final Boolean isDone;
135+
private final long blobSize;
136+
private final boolean isDone;
132137
private final String rewriteToken;
133-
private final Long totalBytesCopied;
134-
private final Long megabytesRewrittenPerCall;
138+
private final long totalBytesCopied;
139+
private final Long megabytesCopiedPerChunk;
135140

136141
StateImpl(Builder builder) {
137142
this.serviceOptions = builder.serviceOptions;
@@ -144,7 +149,7 @@ static class StateImpl implements RestorableState<CopyWriter>, Serializable {
144149
this.isDone = builder.isDone;
145150
this.rewriteToken = builder.rewriteToken;
146151
this.totalBytesCopied = builder.totalBytesCopied;
147-
this.megabytesRewrittenPerCall = builder.megabytesRewrittenPerCall;
152+
this.megabytesCopiedPerChunk = builder.megabytesCopiedPerChunk;
148153
}
149154

150155
static class Builder {
@@ -155,11 +160,11 @@ static class Builder {
155160
private final BlobInfo target;
156161
private final Map<StorageRpc.Option, ?> targetOptions;
157162
private BlobInfo result;
158-
private Long blobSize;
159-
private Boolean isDone;
163+
private long blobSize;
164+
private boolean isDone;
160165
private String rewriteToken;
161-
private Long totalBytesCopied;
162-
private Long megabytesRewrittenPerCall;
166+
private long totalBytesCopied;
167+
private Long megabytesCopiedPerChunk;
163168

164169
private Builder(StorageOptions options, BlobId source,
165170
Map<StorageRpc.Option, ?> sourceOptions,
@@ -176,12 +181,12 @@ Builder result(BlobInfo result) {
176181
return this;
177182
}
178183

179-
Builder blobSize(Long blobSize) {
184+
Builder blobSize(long blobSize) {
180185
this.blobSize = blobSize;
181186
return this;
182187
}
183188

184-
Builder isDone(Boolean isDone) {
189+
Builder isDone(boolean isDone) {
185190
this.isDone = isDone;
186191
return this;
187192
}
@@ -191,13 +196,13 @@ Builder rewriteToken(String rewriteToken) {
191196
return this;
192197
}
193198

194-
Builder totalBytesRewritten(Long totalBytesRewritten) {
199+
Builder totalBytesRewritten(long totalBytesRewritten) {
195200
this.totalBytesCopied = totalBytesRewritten;
196201
return this;
197202
}
198203

199-
Builder megabytesRewrittenPerCall(Long megabytesRewrittenPerCall) {
200-
this.megabytesRewrittenPerCall = megabytesRewrittenPerCall;
204+
Builder megabytesCopiedPerChunk(Long megabytesCopiedPerChunk) {
205+
this.megabytesCopiedPerChunk = megabytesCopiedPerChunk;
201206
return this;
202207
}
203208

@@ -215,7 +220,7 @@ static Builder builder(StorageOptions options, BlobId source,
215220
@Override
216221
public CopyWriter restore() {
217222
RewriteRequest rewriteRequest = new RewriteRequest(
218-
source.toPb(), sourceOptions, target.toPb(), targetOptions, megabytesRewrittenPerCall);
223+
source.toPb(), sourceOptions, target.toPb(), targetOptions, megabytesCopiedPerChunk);
219224
RewriteResponse rewriteResponse = new RewriteResponse(rewriteRequest,
220225
result != null ? result.toPb() : null, blobSize, isDone, rewriteToken,
221226
totalBytesCopied);
@@ -225,7 +230,7 @@ public CopyWriter restore() {
225230
@Override
226231
public int hashCode() {
227232
return Objects.hash(serviceOptions, source, sourceOptions, target, targetOptions, result,
228-
blobSize, isDone, megabytesRewrittenPerCall, rewriteToken, totalBytesCopied);
233+
blobSize, isDone, megabytesCopiedPerChunk, rewriteToken, totalBytesCopied);
229234
}
230235

231236
@Override
@@ -244,10 +249,10 @@ public boolean equals(Object obj) {
244249
&& Objects.equals(this.targetOptions, other.targetOptions)
245250
&& Objects.equals(this.result, other.result)
246251
&& Objects.equals(this.rewriteToken, other.rewriteToken)
247-
&& Objects.equals(this.blobSize, other.blobSize)
248-
&& Objects.equals(this.isDone, other.isDone)
249-
&& Objects.equals(this.megabytesRewrittenPerCall, other.megabytesRewrittenPerCall)
250-
&& Objects.equals(this.totalBytesCopied, other.totalBytesCopied);
252+
&& Objects.equals(this.megabytesCopiedPerChunk, other.megabytesCopiedPerChunk)
253+
&& this.blobSize == other.blobSize
254+
&& this.isDone == other.isDone
255+
&& this.totalBytesCopied == other.totalBytesCopied;
251256
}
252257

253258
@Override

‎gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java‎

Lines changed: 18 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -500,18 +500,18 @@ class CopyRequest implements Serializable {
500500
private final List<BlobSourceOption> sourceOptions;
501501
private final BlobInfo target;
502502
private final List<BlobTargetOption> targetOptions;
503-
private final Long megabytesRewrittenPerCall;
503+
private final Long megabytesCopiedPerChunk;
504504

505505
public static class Builder {
506506

507507
private final Set<BlobSourceOption> sourceOptions = new LinkedHashSet<>();
508508
private final Set<BlobTargetOption> targetOptions = new LinkedHashSet<>();
509509
private BlobId source;
510510
private BlobInfo target;
511-
private Long megabytesRewrittenPerCall;
511+
private Long megabytesCopiedPerChunk;
512512

513513
/**
514-
* Sets the blob to rewrite given bucket and blob name.
514+
* Sets the blob to copy given bucket and blob name.
515515
*
516516
* @return the builder.
517517
*/
@@ -521,7 +521,7 @@ public Builder source(String bucket, String blob) {
521521
}
522522

523523
/**
524-
* Sets the blob to rewrite given a {@link BlobId}.
524+
* Sets the blob to copy given a {@link BlobId}.
525525
*
526526
* @return the builder.
527527
*/
@@ -551,7 +551,7 @@ public Builder sourceOptions(Iterable<BlobSourceOption> options) {
551551
}
552552

553553
/**
554-
* Sets the rewrite target.
554+
* Sets the copy target.
555555
*
556556
* @return the builder.
557557
*/
@@ -582,18 +582,18 @@ public Builder targetOptions(Iterable<BlobTargetOption> options) {
582582

583583
/**
584584
* Sets the maximum number of megabytes to copy for each RPC call. This parameter is ignored
585-
* if source and target blob share the same location and storage class as rewrite is made with
585+
* if source and target blob share the same location and storage class as copy is made with
586586
* one single RPC.
587587
*
588588
* @return the builder.
589589
*/
590-
public Builder megabytesRewrittenPerCall(Long megabytesRewrittenPerCall) {
591-
this.megabytesRewrittenPerCall = megabytesRewrittenPerCall;
590+
public Builder megabytesCopiedPerChunk(Long megabytesCopiedPerChunk) {
591+
this.megabytesCopiedPerChunk = megabytesCopiedPerChunk;
592592
return this;
593593
}
594594

595595
/**
596-
* Creates a {@code RewriteRequest}.
596+
* Creates a {@code CopyRequest}.
597597
*/
598598
public CopyRequest build() {
599599
checkNotNull(source);
@@ -607,7 +607,7 @@ private CopyRequest(Builder builder) {
607607
sourceOptions = ImmutableList.copyOf(builder.sourceOptions);
608608
target = checkNotNull(builder.target);
609609
targetOptions = ImmutableList.copyOf(builder.targetOptions);
610-
megabytesRewrittenPerCall = builder.megabytesRewrittenPerCall;
610+
megabytesCopiedPerChunk = builder.megabytesCopiedPerChunk;
611611
}
612612

613613
/**
@@ -640,11 +640,11 @@ public List<BlobTargetOption> targetOptions() {
640640

641641
/**
642642
* Returns the maximum number of megabytes to copy for each RPC call. This parameter is ignored
643-
* if source and target blob share the same location and storage class as rewrite is made with
643+
* if source and target blob share the same location and storage class as copy is made with
644644
* one single RPC.
645645
*/
646-
public Long megabytesRewrittenPerCall() {
647-
return megabytesRewrittenPerCall;
646+
public Long megabytesCopiedPerChunk() {
647+
return megabytesCopiedPerChunk;
648648
}
649649

650650
public static CopyRequest of(String sourceBucket, String sourceBlob, BlobInfo target) {
@@ -831,16 +831,13 @@ public static Builder builder() {
831831
/**
832832
* Sends a copy request. Returns a {@link CopyWriter} object for the provided
833833
* {@code CopyRequest}. If source and destination objects share the same location and storage
834-
* class the source blob is copied and its information can be accessed with
835-
* {@link CopyWriter#result()}, regardless of the {@link CopyRequest#megabytesRewrittenPerCall}
836-
* parameter. If source and destination have different location or storage class multiple RPC
837-
* calls might be needed depending on blob's size.
834+
* class the source blob is copied with one request and {@link CopyWriter#result()} immediately
835+
* returns, regardless of the {@link CopyRequest#megabytesCopiedPerChunk} parameter.
836+
* If source and destination have different location or storage class {@link CopyWriter#result()}
837+
* might issue multiple RPC calls depending on blob's size.
838838
* <p>
839839
* Example usage of copy:
840-
* <pre> {@code CopyWriter copyWriter = service.copy(copyRequest);}
841-
* {@code while(!copyWriter.isDone()) {
842-
* copyWriter.copyChunk();
843-
* }}
840+
* <pre> {@code BlobInfo blob = service.copy(copyRequest).result();}
844841
* </pre>
845842
*
846843
* @return a {@link CopyWriter} object that can be used to get information on the newly created

‎gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -457,7 +457,7 @@ public CopyWriter copy(final CopyRequest copyRequest) {
457457
@Override
458458
public RewriteResponse call() {
459459
return storageRpc.openRewrite(new StorageRpc.RewriteRequest(source, sourceOptions, target,
460-
targetOptions, copyRequest.megabytesRewrittenPerCall()));
460+
targetOptions, copyRequest.megabytesCopiedPerChunk()));
461461
}
462462
}, options().retryParams(), EXCEPTION_HANDLER);
463463
return new CopyWriter(options(), rewriteResponse);

0 commit comments

Comments
 (0)