Skip to content

Commit 7b25185

Browse files
jean-philippe-martinmziccard
jean-philippe-martin
authored and
mziccard
committed
---
yaml --- r: 4307 b: refs/heads/gcs-nio c: eceec71 h: refs/heads/master i: 4305: 17abd13 4303: 3c74cd4
1 parent f834440 commit 7b25185

10 files changed

Lines changed: 76 additions & 30 deletions

File tree

‎[refs]‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ refs/tags/v0.0.12: 2fd8066e891fb3dfea69b65f6bf6461db79342b9
1111
refs/heads/compute-alpha: 969cba2627f1d53d352cc4a5ffe0879dacf65e6c
1212
refs/heads/dns-alpha: 2f90e7e338349287ace33375896907af0f032ca1
1313
refs/heads/dns-alpha-batch: 17442b07867021b85d0452f5f3eda29a3413288f
14-
refs/heads/gcs-nio: 43f966a04b1ebe51c738751bceed8f11bccea6e9
14+
refs/heads/gcs-nio: eceec7148c81612e402a4242cdf9656df5e2b146
1515
refs/heads/logging-alpha: db5312bffa7fccac194f6a7feb8cc3066de16aff
1616
refs/tags/v0.1.0: a615317f7424ed58621b1f65d5c4d8cbbe8a6ed8
1717
refs/tags/v0.1.1: 7a7f6985fe465e9dd6a075af55493f42b4933be0

‎branches/gcs-nio/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileAttributeView.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,9 @@
1818

1919
import static com.google.common.base.Preconditions.checkNotNull;
2020

21-
import com.google.common.base.MoreObjects;
2221
import com.google.cloud.storage.BlobInfo;
2322
import com.google.cloud.storage.Storage;
23+
import com.google.common.base.MoreObjects;
2424

2525
import java.io.IOException;
2626
import java.nio.file.NoSuchFileException;

‎branches/gcs-nio/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileAttributes.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,9 @@
1616

1717
package com.google.cloud.storage.contrib.nio;
1818

19+
import com.google.cloud.storage.Acl;
1920
import com.google.common.base.Optional;
2021
import com.google.common.collect.ImmutableMap;
21-
import com.google.cloud.storage.Acl;
2222

2323
import java.nio.file.attribute.BasicFileAttributes;
2424
import java.util.List;

‎branches/gcs-nio/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystem.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,8 @@
1919
import static com.google.common.base.Preconditions.checkArgument;
2020
import static com.google.common.base.Preconditions.checkNotNull;
2121

22-
import com.google.common.collect.ImmutableSet;
2322
import com.google.cloud.storage.StorageOptions;
23+
import com.google.common.collect.ImmutableSet;
2424

2525
import java.io.IOException;
2626
import java.net.URI;

‎branches/gcs-nio/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemProvider.java‎

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,6 @@
2121
import static com.google.common.base.Strings.isNullOrEmpty;
2222

2323
import com.google.auto.service.AutoService;
24-
import com.google.common.annotations.VisibleForTesting;
25-
import com.google.common.base.MoreObjects;
26-
import com.google.common.base.Throwables;
27-
import com.google.common.collect.AbstractIterator;
28-
import com.google.common.primitives.Ints;
2924
import com.google.cloud.storage.Acl;
3025
import com.google.cloud.storage.Blob;
3126
import com.google.cloud.storage.BlobId;
@@ -34,6 +29,11 @@
3429
import com.google.cloud.storage.Storage;
3530
import com.google.cloud.storage.StorageException;
3631
import com.google.cloud.storage.StorageOptions;
32+
import com.google.common.annotations.VisibleForTesting;
33+
import com.google.common.base.MoreObjects;
34+
import com.google.common.base.Throwables;
35+
import com.google.common.collect.AbstractIterator;
36+
import com.google.common.primitives.Ints;
3737

3838
import java.io.BufferedInputStream;
3939
import java.io.IOException;
@@ -87,7 +87,8 @@ private static class LazyPathIterator extends AbstractIterator<Path> {
8787
private final Filter<? super Path> filter;
8888
private final CloudStorageFileSystem fileSystem;
8989

90-
LazyPathIterator(CloudStorageFileSystem fileSystem, Iterator<Blob> blobIterator, Filter<? super Path> filter) {
90+
LazyPathIterator(CloudStorageFileSystem fileSystem, Iterator<Blob> blobIterator,
91+
Filter<? super Path> filter) {
9192
this.blobIterator = blobIterator;
9293
this.filter = filter;
9394
this.fileSystem = fileSystem;
@@ -164,7 +165,8 @@ public CloudStorageFileSystem newFileSystem(URI uri, Map<String, ?> env) {
164165
CloudStorageFileSystem.URI_SCHEME,
165166
uri);
166167
checkArgument(
167-
!isNullOrEmpty(uri.getHost()), "%s:// URIs must have a host: %s", CloudStorageFileSystem.URI_SCHEME, uri);
168+
!isNullOrEmpty(uri.getHost()), "%s:// URIs must have a host: %s",
169+
CloudStorageFileSystem.URI_SCHEME, uri);
168170
checkArgument(
169171
uri.getPort() == -1
170172
&& isNullOrEmpty(uri.getPath())
@@ -179,7 +181,8 @@ && isNullOrEmpty(uri.getUserInfo()),
179181

180182
@Override
181183
public CloudStoragePath getPath(URI uri) {
182-
return CloudStoragePath.getPath(getFileSystem(CloudStorageUtil.stripPathFromUri(uri)), uri.getPath());
184+
return CloudStoragePath.getPath(
185+
getFileSystem(CloudStorageUtil.stripPathFromUri(uri)), uri.getPath());
183186
}
184187

185188
@Override
@@ -562,7 +565,9 @@ public DirectoryStream<Path> newDirectoryStream(Path dir, final Filter<? super P
562565
final CloudStoragePath cloudPath = CloudStorageUtil.checkPath(dir);
563566
checkNotNull(filter);
564567
String prefix = cloudPath.toString();
565-
final Iterator<Blob> blobIterator = storage.list(cloudPath.bucket(), Storage.BlobListOption.prefix(prefix), Storage.BlobListOption.fields()).iterateAll();
568+
final Iterator<Blob> blobIterator = storage.list(cloudPath.bucket(),
569+
Storage.BlobListOption.prefix(prefix), Storage.BlobListOption.currentDirectory(),
570+
Storage.BlobListOption.fields()).iterateAll();
566571
return new DirectoryStream<Path>() {
567572
@Override
568573
public Iterator<Path> iterator() {

‎branches/gcs-nio/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageObjectAttributes.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,14 +16,14 @@
1616

1717
package com.google.cloud.storage.contrib.nio;
1818

19-
import static com.google.common.base.Preconditions.checkNotNull;
2019
import static com.google.cloud.storage.contrib.nio.CloudStorageFileSystem.FILE_TIME_UNKNOWN;
20+
import static com.google.common.base.Preconditions.checkNotNull;
2121

22+
import com.google.cloud.storage.Acl;
23+
import com.google.cloud.storage.BlobInfo;
2224
import com.google.common.base.MoreObjects;
2325
import com.google.common.base.Optional;
2426
import com.google.common.collect.ImmutableMap;
25-
import com.google.cloud.storage.Acl;
26-
import com.google.cloud.storage.BlobInfo;
2727

2828
import java.nio.file.attribute.FileTime;
2929
import java.util.List;

‎branches/gcs-nio/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStoragePath.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,8 @@
1919
import static com.google.common.base.Preconditions.checkArgument;
2020
import static com.google.common.base.Preconditions.checkNotNull;
2121

22-
import com.google.common.collect.UnmodifiableIterator;
2322
import com.google.cloud.storage.BlobId;
23+
import com.google.common.collect.UnmodifiableIterator;
2424

2525
import java.io.File;
2626
import java.net.URI;
@@ -308,7 +308,8 @@ public String toString() {
308308
@Override
309309
public URI toUri() {
310310
try {
311-
return new URI(CloudStorageFileSystem.URI_SCHEME, bucket(), path.toAbsolutePath().toString(), null);
311+
return new URI(
312+
CloudStorageFileSystem.URI_SCHEME, bucket(), path.toAbsolutePath().toString(), null);
312313
} catch (URISyntaxException e) {
313314
throw new AssertionError(e);
314315
}

‎branches/gcs-nio/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStoragePseudoDirectoryAttributes.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,9 @@
1616

1717
package com.google.cloud.storage.contrib.nio;
1818

19+
import com.google.cloud.storage.Acl;
1920
import com.google.common.base.Optional;
2021
import com.google.common.collect.ImmutableMap;
21-
import com.google.cloud.storage.Acl;
2222

2323
import java.nio.file.attribute.FileTime;
2424
import java.util.List;

‎branches/gcs-nio/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/it/ITGcsNio.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,10 +27,12 @@
2727
import java.nio.ByteBuffer;
2828
import java.nio.channels.ReadableByteChannel;
2929
import java.nio.channels.SeekableByteChannel;
30+
import java.nio.file.FileSystem;
3031
import java.nio.file.Files;
3132
import java.nio.file.NoSuchFileException;
3233
import java.nio.file.Path;
3334
import java.nio.file.StandardOpenOption;
35+
import java.util.ArrayList;
3436
import java.util.Arrays;
3537
import java.util.List;
3638
import java.util.Random;
@@ -317,6 +319,29 @@ public void testCopy() throws IOException {
317319
}
318320
}
319321

322+
@Test
323+
public void testListFiles() throws IOException {
324+
try (FileSystem fs = getTestBucket()) {
325+
List<Path> goodPaths = new ArrayList<>();
326+
List<Path> paths = new ArrayList<>();
327+
goodPaths.add(fs.getPath("dir/angel"));
328+
goodPaths.add(fs.getPath("dir/alone"));
329+
paths.add(fs.getPath("dir/dir2/another_angel"));
330+
paths.add(fs.getPath("atroot"));
331+
paths.addAll(goodPaths);
332+
goodPaths.add(fs.getPath("dir/dir2/"));
333+
for (Path path : paths) {
334+
fillFile(storage, path.toString(), SML_SIZE);
335+
}
336+
337+
List<Path> got = new ArrayList<>();
338+
for (Path path : Files.newDirectoryStream(fs.getPath("dir/"))) {
339+
got.add(path);
340+
}
341+
assertThat(got).containsExactlyElementsIn(goodPaths);
342+
}
343+
}
344+
320345
private int readFully(ReadableByteChannel chan, byte[] outputBuf) throws IOException {
321346
ByteBuffer buf = ByteBuffer.wrap(outputBuf);
322347
int sofar = 0;

‎branches/gcs-nio/gcloud-java-storage/src/main/java/com/google/cloud/storage/testing/FakeStorageRpc.java‎

Lines changed: 26 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,7 @@ public Tuple<String, Iterable<Bucket>> list(Map<Option, ?> options) throws Stora
9393
@Override
9494
public Tuple<String, Iterable<StorageObject>> list(String bucket, Map<Option, ?> options)
9595
throws StorageException {
96+
String delimiter = null;
9697
String preprefix = "";
9798
for (Map.Entry<Option, ?> e : options.entrySet()) {
9899
switch (e.getKey()) {
@@ -102,6 +103,9 @@ public Tuple<String, Iterable<StorageObject>> list(String bucket, Map<Option, ?>
102103
preprefix = preprefix.substring(1);
103104
}
104105
break;
106+
case DELIMITER:
107+
delimiter = (String) e.getValue();
108+
break;
105109
case FIELDS:
106110
// ignore and return all the fields
107111
break;
@@ -117,17 +121,7 @@ public Tuple<String, Iterable<StorageObject>> list(String bucket, Map<Option, ?>
117121
if (!so.getName().startsWith(prefix)) {
118122
continue;
119123
}
120-
int nextSlash = so.getName().indexOf("/", prefix.length());
121-
if (nextSlash >= 0) {
122-
String folderName = so.getName().substring(0, nextSlash + 1);
123-
if (folders.containsKey(folderName)) {
124-
continue;
125-
}
126-
StorageObject fakeFolder = new StorageObject();
127-
fakeFolder.setName(folderName);
128-
fakeFolder.setBucket(so.getBucket());
129-
fakeFolder.setGeneration(so.getGeneration());
130-
folders.put(folderName, fakeFolder);
124+
if (processedAsFolder(so, delimiter, prefix, folders)) {
131125
continue;
132126
}
133127
values.add(so);
@@ -333,4 +327,25 @@ private void potentiallyThrow(Map<Option, ?> options) throws UnsupportedOperatio
333327
throw new UnsupportedOperationException();
334328
}
335329
}
330+
331+
// Returns true if this is a folder. Adds it to folders if it isn't already there.
332+
private static boolean processedAsFolder(StorageObject so, String delimiter, String prefix, /* inout */ Map<String, StorageObject> folders) {
333+
if (delimiter == null) {
334+
return false;
335+
}
336+
int nextSlash = so.getName().indexOf(delimiter, prefix.length());
337+
if (nextSlash < 0) {
338+
return false;
339+
}
340+
String folderName = so.getName().substring(0, nextSlash + 1);
341+
if (folders.containsKey(folderName)) {
342+
return true;
343+
}
344+
StorageObject fakeFolder = new StorageObject();
345+
fakeFolder.setName(folderName);
346+
fakeFolder.setBucket(so.getBucket());
347+
fakeFolder.setGeneration(so.getGeneration());
348+
folders.put(folderName, fakeFolder);
349+
return true;
350+
}
336351
}

0 commit comments

Comments
 (0)