Skip to content

Commit d930b4b

Browse files
mziccardmziccard
committed
Remove BlobListOption.recursive option and fix delimiter handling
- Add BlobListOption.currentDirectory option that sets the '/' delimiter - Add isDirectory method to BlobInfo objects - Change StorageRpc.list(bucket) method to add prefixes to the list of storage objects - Add unit and integration tests
1 parent e70387d commit d930b4b

12 files changed

Lines changed: 228 additions & 41 deletions

File tree

‎gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -57,8 +57,10 @@
5757
import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions;
5858
import com.google.api.services.storage.model.Objects;
5959
import com.google.api.services.storage.model.StorageObject;
60+
import com.google.common.base.Function;
6061
import com.google.common.base.MoreObjects;
6162
import com.google.common.collect.ImmutableList;
63+
import com.google.common.collect.Iterables;
6264
import com.google.common.collect.Lists;
6365
import com.google.common.collect.Maps;
6466
import com.google.gcloud.storage.StorageException;
@@ -67,6 +69,7 @@
6769
import java.io.ByteArrayOutputStream;
6870
import java.io.IOException;
6971
import java.io.InputStream;
72+
import java.math.BigInteger;
7073
import java.util.ArrayList;
7174
import java.util.Iterator;
7275
import java.util.List;
@@ -151,7 +154,7 @@ public Tuple<String, Iterable<Bucket>> list(Map<Option, ?> options) {
151154
}
152155

153156
@Override
154-
public Tuple<String, Iterable<StorageObject>> list(String bucket, Map<Option, ?> options) {
157+
public Tuple<String, Iterable<StorageObject>> list(final String bucket, Map<Option, ?> options) {
155158
try {
156159
Objects objects = storage.objects()
157160
.list(bucket)
@@ -163,8 +166,20 @@ public Tuple<String, Iterable<StorageObject>> list(String bucket, Map<Option, ?>
163166
.setPageToken(PAGE_TOKEN.getString(options))
164167
.setFields(FIELDS.getString(options))
165168
.execute();
166-
return Tuple.<String, Iterable<StorageObject>>of(
167-
objects.getNextPageToken(), objects.getItems());
169+
Iterable<StorageObject> storageObjects = Iterables.concat(
170+
objects.getItems() != null ? objects.getItems() : ImmutableList.<StorageObject>of(),
171+
objects.getPrefixes() != null
172+
? Lists.transform(objects.getPrefixes(), new Function<String, StorageObject>() {
173+
@Override
174+
public StorageObject apply(String prefix) {
175+
return new StorageObject()
176+
.set("isDirectory", true)
177+
.setBucket(bucket)
178+
.setName(prefix)
179+
.setSize(BigInteger.ZERO);
180+
}
181+
}) : ImmutableList.<StorageObject>of());
182+
return Tuple.of(objects.getNextPageToken(), storageObjects);
168183
} catch (IOException ex) {
169184
throw translate(ex);
170185
}

‎gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -344,7 +344,7 @@ void write(String uploadId, byte[] toWrite, int toWriteOffset, long destOffset,
344344
/**
345345
* Continues rewriting on an already open rewrite channel.
346346
*
347-
* @throws StorageException
347+
* @throws StorageException upon failure
348348
*/
349349
RewriteResponse continueRewrite(RewriteResponse previousResponse);
350350
}

‎gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -290,6 +290,12 @@ Builder updateTime(Long updateTime) {
290290
return this;
291291
}
292292

293+
@Override
294+
Builder isDirectory(boolean isDirectory) {
295+
infoBuilder.isDirectory(isDirectory);
296+
return this;
297+
}
298+
293299
@Override
294300
public Blob build() {
295301
return new Blob(storage, infoBuilder);

‎gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobInfo.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ public StorageObject apply(BlobInfo blobInfo) {
7878
private final String contentDisposition;
7979
private final String contentLanguage;
8080
private final Integer componentCount;
81+
private final boolean isDirectory;
8182

8283
/**
8384
* This class is meant for internal use only. Users are discouraged from using this class.
@@ -187,6 +188,8 @@ public abstract static class Builder {
187188

188189
abstract Builder updateTime(Long updateTime);
189190

191+
abstract Builder isDirectory(boolean isDirectory);
192+
190193
/**
191194
* Creates a {@code BlobInfo} object.
192195
*/
@@ -215,6 +218,7 @@ static final class BuilderImpl extends Builder {
215218
private Long metageneration;
216219
private Long deleteTime;
217220
private Long updateTime;
221+
private Boolean isDirectory;
218222

219223
BuilderImpl(BlobId blobId) {
220224
this.blobId = blobId;
@@ -241,6 +245,7 @@ static final class BuilderImpl extends Builder {
241245
metageneration = blobInfo.metageneration;
242246
deleteTime = blobInfo.deleteTime;
243247
updateTime = blobInfo.updateTime;
248+
isDirectory = blobInfo.isDirectory;
244249
}
245250

246251
@Override
@@ -364,6 +369,12 @@ Builder updateTime(Long updateTime) {
364369
return this;
365370
}
366371

372+
@Override
373+
Builder isDirectory(boolean isDirectory) {
374+
this.isDirectory = isDirectory;
375+
return this;
376+
}
377+
367378
@Override
368379
public BlobInfo build() {
369380
checkNotNull(blobId);
@@ -392,6 +403,7 @@ public BlobInfo build() {
392403
metageneration = builder.metageneration;
393404
deleteTime = builder.deleteTime;
394405
updateTime = builder.updateTime;
406+
isDirectory = firstNonNull(builder.isDirectory, Boolean.FALSE);
395407
}
396408

397409
/**
@@ -588,6 +600,18 @@ public Long updateTime() {
588600
return updateTime;
589601
}
590602

603+
/**
604+
* Returns {@code true} if the current blob represents a directory. This can only happen if the
605+
* blob is returned by {@link Storage#list(String, Storage.BlobListOption...)} when the
606+
* {@link Storage.BlobListOption#currentDirectory()} option is used. If {@code true} only
607+
* {@link #blobId()} and {@link #size()} are set for the current blob: {@link BlobId#name()} ends
608+
* with the '/' character, {@link BlobId#generation()} returns {@code null} and {@link #size()} is
609+
* {@code 0}.
610+
*/
611+
public boolean isDirectory() {
612+
return isDirectory;
613+
}
614+
591615
/**
592616
* Returns a builder for the current blob.
593617
*/
@@ -761,6 +785,9 @@ public Acl apply(ObjectAccessControl objectAccessControl) {
761785
}
762786
}));
763787
}
788+
if (storageObject.get("isDirectory") != null) {
789+
builder.isDirectory(Boolean.TRUE);
790+
}
764791
return builder.build();
765792
}
766793
}

‎gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -694,10 +694,17 @@ public static BlobListOption prefix(String prefix) {
694694
}
695695

696696
/**
697-
* Returns an option to specify whether blob listing should include subdirectories or not.
697+
* If specified, results are returned in a directory-like mode. Blobs whose names, aside from
698+
* a possible {@link #prefix(String)}, do not contain the '/' delimiter are returned as is.
699+
* Blobs whose names, aside from a possible {@link #prefix(String)}, contain the '/' delimiter,
700+
* will have their name truncated after the delimiter and will be returned as {@link Blob}
701+
* objects where only {@link Blob#blobId()}, {@link Blob#size()} and {@link Blob#isDirectory()}
702+
* are set. For such directory blobs, ({@link BlobId#generation()} returns {@code null}),
703+
* {@link Blob#size()} returns {@code 0} while {@link Blob#isDirectory()} returns {@code true}.
704+
* Duplicate directory blobs are omitted.
698705
*/
699-
public static BlobListOption recursive(boolean recursive) {
700-
return new BlobListOption(StorageRpc.Option.DELIMITER, recursive);
706+
public static BlobListOption currentDirectory() {
707+
return new BlobListOption(StorageRpc.Option.DELIMITER, true);
701708
}
702709

703710
/**
@@ -1289,7 +1296,8 @@ private static void checkContentType(BlobInfo blobInfo) throws IllegalArgumentEx
12891296
Page<Bucket> list(BucketListOption... options);
12901297

12911298
/**
1292-
* Lists the bucket's blobs.
1299+
* Lists the bucket's blobs. If the {@link BlobListOption#currentDirectory()} option is provided,
1300+
* results are returned in a directory-like mode.
12931301
*
12941302
* @throws StorageException upon failure
12951303
*/

‎gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ final class StorageImpl extends BaseService<StorageOptions> implements Storage {
7676
private static final byte[] EMPTY_BYTE_ARRAY = {};
7777
private static final String EMPTY_BYTE_ARRAY_MD5 = "1B2M2Y8AsgTpgAmY7PhCfg==";
7878
private static final String EMPTY_BYTE_ARRAY_CRC32C = "AAAAAA==";
79+
private static final String PATH_DELIMITER = "/";
7980

8081
private static final Function<Tuple<Storage, Boolean>, Boolean> DELETE_FUNCTION =
8182
new Function<Tuple<Storage, Boolean>, Boolean>() {
@@ -669,7 +670,7 @@ private static <T> void addToOptionMap(StorageRpc.Option getOption, StorageRpc.O
669670
}
670671
Boolean value = (Boolean) temp.remove(DELIMITER);
671672
if (Boolean.TRUE.equals(value)) {
672-
temp.put(DELIMITER, options().pathDelimiter());
673+
temp.put(DELIMITER, PATH_DELIMITER);
673674
}
674675
if (useAsSource) {
675676
addToOptionMap(IF_GENERATION_MATCH, IF_SOURCE_GENERATION_MATCH, generation, temp);

‎gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageOptions.java‎

Lines changed: 2 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -16,24 +16,19 @@
1616

1717
package com.google.gcloud.storage;
1818

19-
import com.google.common.base.MoreObjects;
2019
import com.google.common.collect.ImmutableSet;
2120
import com.google.gcloud.ServiceOptions;
2221
import com.google.gcloud.spi.DefaultStorageRpc;
2322
import com.google.gcloud.spi.StorageRpc;
2423
import com.google.gcloud.spi.StorageRpcFactory;
2524

26-
import java.util.Objects;
2725
import java.util.Set;
2826

2927
public class StorageOptions extends ServiceOptions<Storage, StorageRpc, StorageOptions> {
3028

3129
private static final long serialVersionUID = -7804860602287801084L;
3230
private static final String GCS_SCOPE = "https://www.googleapis.com/auth/devstorage.full_control";
3331
private static final Set<String> SCOPES = ImmutableSet.of(GCS_SCOPE);
34-
private static final String DEFAULT_PATH_DELIMITER = "/";
35-
36-
private final String pathDelimiter;
3732

3833
public static class DefaultStorageFactory implements StorageFactory {
3934

@@ -58,24 +53,10 @@ public StorageRpc create(StorageOptions options) {
5853
public static class Builder extends
5954
ServiceOptions.Builder<Storage, StorageRpc, StorageOptions, Builder> {
6055

61-
private String pathDelimiter;
62-
6356
private Builder() {}
6457

6558
private Builder(StorageOptions options) {
6659
super(options);
67-
pathDelimiter = options.pathDelimiter;
68-
}
69-
70-
/**
71-
* Sets the path delimiter for the storage service.
72-
*
73-
* @param pathDelimiter the path delimiter to set
74-
* @return the builder
75-
*/
76-
public Builder pathDelimiter(String pathDelimiter) {
77-
this.pathDelimiter = pathDelimiter;
78-
return this;
7960
}
8061

8162
@Override
@@ -86,7 +67,6 @@ public StorageOptions build() {
8667

8768
private StorageOptions(Builder builder) {
8869
super(StorageFactory.class, StorageRpcFactory.class, builder);
89-
pathDelimiter = MoreObjects.firstNonNull(builder.pathDelimiter, DEFAULT_PATH_DELIMITER);
9070
}
9171

9272
@SuppressWarnings("unchecked")
@@ -106,13 +86,6 @@ protected Set<String> scopes() {
10686
return SCOPES;
10787
}
10888

109-
/**
110-
* Returns the storage service's path delimiter.
111-
*/
112-
public String pathDelimiter() {
113-
return pathDelimiter;
114-
}
115-
11689
/**
11790
* Returns a default {@code StorageOptions} instance.
11891
*/
@@ -128,7 +101,7 @@ public Builder toBuilder() {
128101

129102
@Override
130103
public int hashCode() {
131-
return baseHashCode() ^ Objects.hash(pathDelimiter);
104+
return baseHashCode();
132105
}
133106

134107
@Override
@@ -137,7 +110,7 @@ public boolean equals(Object obj) {
137110
return false;
138111
}
139112
StorageOptions other = (StorageOptions) obj;
140-
return baseEquals(other) && Objects.equals(pathDelimiter, other.pathDelimiter);
113+
return baseEquals(other);
141114
}
142115

143116
public static Builder builder() {

‎gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobInfoTest.java‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,14 +20,19 @@
2020
import static com.google.gcloud.storage.Acl.Role.READER;
2121
import static com.google.gcloud.storage.Acl.Role.WRITER;
2222
import static org.junit.Assert.assertEquals;
23+
import static org.junit.Assert.assertFalse;
24+
import static org.junit.Assert.assertNull;
25+
import static org.junit.Assert.assertTrue;
2326

27+
import com.google.api.services.storage.model.StorageObject;
2428
import com.google.common.collect.ImmutableList;
2529
import com.google.common.collect.ImmutableMap;
2630
import com.google.gcloud.storage.Acl.Project;
2731
import com.google.gcloud.storage.Acl.User;
2832

2933
import org.junit.Test;
3034

35+
import java.math.BigInteger;
3136
import java.util.List;
3237
import java.util.Map;
3338

@@ -76,6 +81,10 @@ public class BlobInfoTest {
7681
.size(SIZE)
7782
.updateTime(UPDATE_TIME)
7883
.build();
84+
private static final BlobInfo DIRECTORY_INFO = BlobInfo.builder("b", "n/")
85+
.size(0L)
86+
.isDirectory(true)
87+
.build();
7988

8089
@Test
8190
public void testToBuilder() {
@@ -118,6 +127,30 @@ public void testBuilder() {
118127
assertEquals(SELF_LINK, BLOB_INFO.selfLink());
119128
assertEquals(SIZE, BLOB_INFO.size());
120129
assertEquals(UPDATE_TIME, BLOB_INFO.updateTime());
130+
assertFalse(BLOB_INFO.isDirectory());
131+
assertEquals("b", DIRECTORY_INFO.bucket());
132+
assertEquals("n/", DIRECTORY_INFO.name());
133+
assertNull(DIRECTORY_INFO.acl());
134+
assertNull(DIRECTORY_INFO.componentCount());
135+
assertNull(DIRECTORY_INFO.contentType());
136+
assertNull(DIRECTORY_INFO.cacheControl());
137+
assertNull(DIRECTORY_INFO.contentDisposition());
138+
assertNull(DIRECTORY_INFO.contentEncoding());
139+
assertNull(DIRECTORY_INFO.contentLanguage());
140+
assertNull(DIRECTORY_INFO.crc32c());
141+
assertNull(DIRECTORY_INFO.deleteTime());
142+
assertNull(DIRECTORY_INFO.etag());
143+
assertNull(DIRECTORY_INFO.generation());
144+
assertNull(DIRECTORY_INFO.id());
145+
assertNull(DIRECTORY_INFO.md5());
146+
assertNull(DIRECTORY_INFO.mediaLink());
147+
assertNull(DIRECTORY_INFO.metadata());
148+
assertNull(DIRECTORY_INFO.metageneration());
149+
assertNull(DIRECTORY_INFO.owner());
150+
assertNull(DIRECTORY_INFO.selfLink());
151+
assertEquals(0L, (long) DIRECTORY_INFO.size());
152+
assertNull(DIRECTORY_INFO.updateTime());
153+
assertTrue(DIRECTORY_INFO.isDirectory());
121154
}
122155

123156
private void compareBlobs(BlobInfo expected, BlobInfo value) {
@@ -151,6 +184,35 @@ public void testToPbAndFromPb() {
151184
compareBlobs(BLOB_INFO, BlobInfo.fromPb(BLOB_INFO.toPb()));
152185
BlobInfo blobInfo = BlobInfo.builder(BlobId.of("b", "n")).build();
153186
compareBlobs(blobInfo, BlobInfo.fromPb(blobInfo.toPb()));
187+
StorageObject object = new StorageObject()
188+
.setName("n/")
189+
.setBucket("b")
190+
.setSize(BigInteger.ZERO)
191+
.set("isDirectory", true);
192+
blobInfo = BlobInfo.fromPb(object);
193+
assertEquals("b", blobInfo.bucket());
194+
assertEquals("n/", blobInfo.name());
195+
assertNull(blobInfo.acl());
196+
assertNull(blobInfo.componentCount());
197+
assertNull(blobInfo.contentType());
198+
assertNull(blobInfo.cacheControl());
199+
assertNull(blobInfo.contentDisposition());
200+
assertNull(blobInfo.contentEncoding());
201+
assertNull(blobInfo.contentLanguage());
202+
assertNull(blobInfo.crc32c());
203+
assertNull(blobInfo.deleteTime());
204+
assertNull(blobInfo.etag());
205+
assertNull(blobInfo.generation());
206+
assertNull(blobInfo.id());
207+
assertNull(blobInfo.md5());
208+
assertNull(blobInfo.mediaLink());
209+
assertNull(blobInfo.metadata());
210+
assertNull(blobInfo.metageneration());
211+
assertNull(blobInfo.owner());
212+
assertNull(blobInfo.selfLink());
213+
assertEquals(0L, (long) blobInfo.size());
214+
assertNull(blobInfo.updateTime());
215+
assertTrue(blobInfo.isDirectory());
154216
}
155217

156218
@Test

0 commit comments

Comments
 (0)