Skip to content

Commit 15372f1

Browse files
mziccardmziccard
committed
Refactor StorageRpc and Storage rewrite
- Split StorageRpc.rewrite into openRewrite and continueRewrite - Add StorageRpc.RewriteResponse and RewriteRequest classe - Move first rewrite request from BlobRewriter to StorageImpl - Change maxBytesRewrittenPerCall to megabytesRewrittenPerCall - Change BlobWriter.blobSize for BigInteger to Long - BlobRewriter extends Restorable - Refactor tests and add StorageImpl unit tests
1 parent fdb2402 commit 15372f1

8 files changed

Lines changed: 489 additions & 258 deletions

File tree

‎gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java‎

Lines changed: 32 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,6 @@
5555
import com.google.api.services.storage.model.ComposeRequest;
5656
import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions;
5757
import com.google.api.services.storage.model.Objects;
58-
import com.google.api.services.storage.model.RewriteResponse;
5958
import com.google.api.services.storage.model.StorageObject;
6059
import com.google.common.base.MoreObjects;
6160
import com.google.common.collect.ImmutableSet;
@@ -79,6 +78,7 @@ public class DefaultStorageRpc implements StorageRpc {
7978

8079
// see: https://cloud.google.com/storage/docs/concepts-techniques#practices
8180
private static final Set<Integer> RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 429, 408);
81+
private static final long MEGABYTE = 1024L * 1024L;
8282

8383
public DefaultStorageRpc(StorageOptions options) {
8484
HttpTransport transport = options.httpTransportFactory().create();
@@ -524,27 +524,42 @@ public String open(StorageObject object, Map<Option, ?> options)
524524
}
525525

526526
@Override
527-
public RewriteResponse rewrite(StorageObject source, Map<Option, ?> sourceOptions,
528-
StorageObject target, Map<Option, ?> targetOptions, String token, Long maxByteRewrittenPerCall)
529-
throws StorageException {
527+
public RewriteResponse openRewrite(RewriteRequest rewriteRequest) throws StorageException {
528+
return rewrite(rewriteRequest, null);
529+
}
530+
531+
@Override
532+
public RewriteResponse continueRewrite(RewriteResponse previousResponse) throws StorageException {
533+
return rewrite(previousResponse.rewriteRequest, previousResponse.rewriteToken);
534+
}
535+
536+
private RewriteResponse rewrite(RewriteRequest req, String token) throws StorageException {
530537
try {
531-
return storage
532-
.objects()
533-
.rewrite(source.getBucket(), source.getName(), target.getBucket(), target.getName(),
534-
target)
538+
Long maxBytesRewrittenPerCall = req.megabytesRewrittenPerCall != null
539+
? req.megabytesRewrittenPerCall * MEGABYTE : null;
540+
com.google.api.services.storage.model.RewriteResponse rewriteReponse = storage.objects()
541+
.rewrite(req.source.getBucket(), req.source.getName(), req.target.getBucket(),
542+
req.target.getName(), req.target)
535543
.setRewriteToken(token)
536-
.setMaxBytesRewrittenPerCall(maxByteRewrittenPerCall)
544+
.setMaxBytesRewrittenPerCall(maxBytesRewrittenPerCall)
537545
.setProjection(DEFAULT_PROJECTION)
538-
.setIfSourceMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions))
546+
.setIfSourceMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(req.sourceOptions))
539547
.setIfSourceMetagenerationNotMatch(
540-
IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions))
541-
.setIfSourceGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions))
542-
.setIfSourceGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions))
543-
.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
544-
.setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions))
545-
.setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
546-
.setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions))
548+
IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(req.sourceOptions))
549+
.setIfSourceGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(req.sourceOptions))
550+
.setIfSourceGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(req.sourceOptions))
551+
.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(req.targetOptions))
552+
.setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(req.targetOptions))
553+
.setIfGenerationMatch(IF_GENERATION_MATCH.getLong(req.targetOptions))
554+
.setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(req.targetOptions))
547555
.execute();
556+
return new RewriteResponse(
557+
req,
558+
rewriteReponse.getResource(),
559+
rewriteReponse.getObjectSize().longValue(),
560+
rewriteReponse.getDone(),
561+
rewriteReponse.getRewriteToken(),
562+
rewriteReponse.getTotalBytesRewritten().longValue());
548563
} catch (IOException ex) {
549564
throw translate(ex);
550565
}

‎gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java‎

Lines changed: 87 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@
1919
import static com.google.common.base.MoreObjects.firstNonNull;
2020

2121
import com.google.api.services.storage.model.Bucket;
22-
import com.google.api.services.storage.model.RewriteResponse;
2322
import com.google.api.services.storage.model.StorageObject;
2423
import com.google.common.collect.ImmutableList;
2524
import com.google.common.collect.ImmutableMap;
@@ -28,6 +27,7 @@
2827
import java.io.InputStream;
2928
import java.util.List;
3029
import java.util.Map;
30+
import java.util.Objects;
3131

3232
public interface StorageRpc {
3333

@@ -133,6 +133,89 @@ public BatchResponse(Map<StorageObject, Tuple<Boolean, StorageException>> delete
133133
}
134134
}
135135

136+
class RewriteRequest {
137+
138+
public final StorageObject source;
139+
public final Map<StorageRpc.Option, ?> sourceOptions;
140+
public final StorageObject target;
141+
public final Map<StorageRpc.Option, ?> targetOptions;
142+
public final Long megabytesRewrittenPerCall;
143+
144+
public RewriteRequest(StorageObject source, Map<StorageRpc.Option, ?> sourceOptions,
145+
StorageObject target, Map<StorageRpc.Option, ?> targetOptions,
146+
Long megabytesRewrittenPerCall) {
147+
this.source = source;
148+
this.sourceOptions = sourceOptions;
149+
this.target = target;
150+
this.targetOptions = targetOptions;
151+
this.megabytesRewrittenPerCall = megabytesRewrittenPerCall;
152+
}
153+
154+
@Override
155+
public boolean equals(Object obj) {
156+
if (obj == null) {
157+
return false;
158+
}
159+
if (!(obj instanceof RewriteRequest)) {
160+
return false;
161+
}
162+
final RewriteRequest other = (RewriteRequest) obj;
163+
return Objects.equals(this.source, other.source)
164+
&& Objects.equals(this.sourceOptions, other.sourceOptions)
165+
&& Objects.equals(this.target, other.target)
166+
&& Objects.equals(this.targetOptions, other.targetOptions)
167+
&& Objects.equals(this.megabytesRewrittenPerCall, other.megabytesRewrittenPerCall);
168+
}
169+
170+
@Override
171+
public int hashCode() {
172+
return Objects.hash(source, sourceOptions, target, targetOptions, megabytesRewrittenPerCall);
173+
}
174+
}
175+
176+
class RewriteResponse {
177+
178+
public final RewriteRequest rewriteRequest;
179+
public final StorageObject result;
180+
public final Long blobSize;
181+
public final Boolean isDone;
182+
public final String rewriteToken;
183+
public final Long totalBytesRewritten;
184+
185+
public RewriteResponse(RewriteRequest rewriteRequest, StorageObject result, Long blobSize,
186+
Boolean isDone, String rewriteToken, Long totalBytesRewritten) {
187+
this.rewriteRequest = rewriteRequest;
188+
this.result = result;
189+
this.blobSize = blobSize;
190+
this.isDone = isDone;
191+
this.rewriteToken = rewriteToken;
192+
this.totalBytesRewritten = totalBytesRewritten;
193+
}
194+
195+
@Override
196+
public boolean equals(Object obj) {
197+
if (obj == null) {
198+
return false;
199+
}
200+
if (!(obj instanceof RewriteResponse)) {
201+
return false;
202+
}
203+
final RewriteResponse other = (RewriteResponse) obj;
204+
return Objects.equals(this.rewriteRequest, other.rewriteRequest)
205+
&& Objects.equals(this.result, other.result)
206+
&& Objects.equals(this.rewriteToken, other.rewriteToken)
207+
&& Objects.equals(this.blobSize, other.blobSize)
208+
&& Objects.equals(this.isDone, other.isDone)
209+
&& Objects.equals(this.totalBytesRewritten, other.totalBytesRewritten);
210+
}
211+
212+
@Override
213+
public int hashCode() {
214+
return Objects.hash(rewriteRequest, result, blobSize, isDone, rewriteToken,
215+
totalBytesRewritten);
216+
}
217+
}
218+
136219
Bucket create(Bucket bucket, Map<Option, ?> options) throws StorageException;
137220

138221
StorageObject create(StorageObject object, InputStream content, Map<Option, ?> options)
@@ -176,7 +259,7 @@ byte[] read(StorageObject from, Map<Option, ?> options, long position, int bytes
176259
void write(String uploadId, byte[] toWrite, int toWriteOffset, StorageObject dest,
177260
long destOffset, int length, boolean last) throws StorageException;
178261

179-
RewriteResponse rewrite(StorageObject source, Map<Option, ?> sourceOptions,
180-
StorageObject target, Map<Option, ?> targetOptions, String token, Long maxByteRewrittenPerCall)
181-
throws StorageException;
262+
RewriteResponse openRewrite(RewriteRequest rewriteRequest) throws StorageException;
263+
264+
RewriteResponse continueRewrite(RewriteResponse previousResponse) throws StorageException;
182265
}

0 commit comments

Comments
 (0)