Skip to content

Commit 10180eb

Browse files
jean-philippe-martinmziccard
jean-philippe-martin
authored and
mziccard
committed
Integration test for ls (#931)
1 parent e874b76 commit 10180eb

4 files changed

Lines changed: 60 additions & 15 deletions

File tree

‎gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemProvider.java‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,8 @@ private static class LazyPathIterator extends AbstractIterator<Path> {
8888
private final Filter<? super Path> filter;
8989
private final CloudStorageFileSystem fileSystem;
9090

91-
LazyPathIterator(CloudStorageFileSystem fileSystem, Iterator<Blob> blobIterator, Filter<? super Path> filter) {
91+
LazyPathIterator(CloudStorageFileSystem fileSystem, Iterator<Blob> blobIterator,
92+
Filter<? super Path> filter) {
9293
this.blobIterator = blobIterator;
9394
this.filter = filter;
9495
this.fileSystem = fileSystem;
@@ -176,7 +177,8 @@ && isNullOrEmpty(uri.getUserInfo()),
176177

177178
@Override
178179
public CloudStoragePath getPath(URI uri) {
179-
return CloudStoragePath.getPath(getFileSystem(CloudStorageUtil.stripPathFromUri(uri)), uri.getPath());
180+
return CloudStoragePath.getPath(
181+
getFileSystem(CloudStorageUtil.stripPathFromUri(uri)), uri.getPath());
180182
}
181183

182184
@Override
@@ -559,7 +561,9 @@ public DirectoryStream<Path> newDirectoryStream(Path dir, final Filter<? super P
559561
final CloudStoragePath cloudPath = CloudStorageUtil.checkPath(dir);
560562
checkNotNull(filter);
561563
String prefix = cloudPath.toString();
562-
final Iterator<Blob> blobIterator = storage.list(cloudPath.bucket(), Storage.BlobListOption.prefix(prefix), Storage.BlobListOption.fields()).iterateAll();
564+
final Iterator<Blob> blobIterator = storage.list(cloudPath.bucket(),
565+
Storage.BlobListOption.prefix(prefix), Storage.BlobListOption.currentDirectory(),
566+
Storage.BlobListOption.fields()).iterateAll();
563567
return new DirectoryStream<Path>() {
564568
@Override
565569
public Iterator<Path> iterator() {

‎gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStoragePath.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -308,7 +308,8 @@ public String toString() {
308308
@Override
309309
public URI toUri() {
310310
try {
311-
return new URI(CloudStorageFileSystem.URI_SCHEME, bucket(), path.toAbsolutePath().toString(), null);
311+
return new URI(
312+
CloudStorageFileSystem.URI_SCHEME, bucket(), path.toAbsolutePath().toString(), null);
312313
} catch (URISyntaxException e) {
313314
throw new AssertionError(e);
314315
}

‎gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/it/ITGcsNio.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,10 +27,12 @@
2727
import java.nio.ByteBuffer;
2828
import java.nio.channels.ReadableByteChannel;
2929
import java.nio.channels.SeekableByteChannel;
30+
import java.nio.file.FileSystem;
3031
import java.nio.file.Files;
3132
import java.nio.file.NoSuchFileException;
3233
import java.nio.file.Path;
3334
import java.nio.file.StandardOpenOption;
35+
import java.util.ArrayList;
3436
import java.util.Arrays;
3537
import java.util.List;
3638
import java.util.Random;
@@ -316,6 +318,29 @@ public void testCopy() throws IOException {
316318
}
317319
}
318320

321+
@Test
322+
public void testListFiles() throws IOException {
323+
try (FileSystem fs = getTestBucket()) {
324+
List<Path> goodPaths = new ArrayList<>();
325+
List<Path> paths = new ArrayList<>();
326+
goodPaths.add(fs.getPath("dir/angel"));
327+
goodPaths.add(fs.getPath("dir/alone"));
328+
paths.add(fs.getPath("dir/dir2/another_angel"));
329+
paths.add(fs.getPath("atroot"));
330+
paths.addAll(goodPaths);
331+
goodPaths.add(fs.getPath("dir/dir2/"));
332+
for (Path path : paths) {
333+
fillFile(storage, path.toString(), SML_SIZE);
334+
}
335+
336+
List<Path> got = new ArrayList<>();
337+
for (Path path : Files.newDirectoryStream(fs.getPath("dir/"))) {
338+
got.add(path);
339+
}
340+
assertThat(got).containsExactlyElementsIn(goodPaths);
341+
}
342+
}
343+
319344
private int readFully(ReadableByteChannel chan, byte[] outputBuf) throws IOException {
320345
ByteBuffer buf = ByteBuffer.wrap(outputBuf);
321346
int sofar = 0;

‎gcloud-java-storage/src/main/java/com/google/cloud/storage/testing/FakeStorageRpc.java‎

Lines changed: 26 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,7 @@ public Tuple<String, Iterable<Bucket>> list(Map<Option, ?> options) throws Stora
9494
@Override
9595
public Tuple<String, Iterable<StorageObject>> list(String bucket, Map<Option, ?> options)
9696
throws StorageException {
97+
String delimiter = null;
9798
String preprefix = "";
9899
for (Map.Entry<Option, ?> e : options.entrySet()) {
99100
switch (e.getKey()) {
@@ -103,6 +104,9 @@ public Tuple<String, Iterable<StorageObject>> list(String bucket, Map<Option, ?>
103104
preprefix = preprefix.substring(1);
104105
}
105106
break;
107+
case DELIMITER:
108+
delimiter = (String) e.getValue();
109+
break;
106110
case FIELDS:
107111
// ignore and return all the fields
108112
break;
@@ -118,17 +122,7 @@ public Tuple<String, Iterable<StorageObject>> list(String bucket, Map<Option, ?>
118122
if (!so.getName().startsWith(prefix)) {
119123
continue;
120124
}
121-
int nextSlash = so.getName().indexOf("/", prefix.length());
122-
if (nextSlash >= 0) {
123-
String folderName = so.getName().substring(0, nextSlash + 1);
124-
if (folders.containsKey(folderName)) {
125-
continue;
126-
}
127-
StorageObject fakeFolder = new StorageObject();
128-
fakeFolder.setName(folderName);
129-
fakeFolder.setBucket(so.getBucket());
130-
fakeFolder.setGeneration(so.getGeneration());
131-
folders.put(folderName, fakeFolder);
125+
if (processedAsFolder(so, delimiter, prefix, folders)) {
132126
continue;
133127
}
134128
values.add(so);
@@ -334,4 +328,25 @@ private void potentiallyThrow(Map<Option, ?> options) throws UnsupportedOperatio
334328
throw new UnsupportedOperationException();
335329
}
336330
}
331+
332+
// Returns true if this is a folder. Adds it to folders if it isn't already there.
333+
private static boolean processedAsFolder(StorageObject so, String delimiter, String prefix, /* inout */ Map<String, StorageObject> folders) {
334+
if (delimiter == null) {
335+
return false;
336+
}
337+
int nextSlash = so.getName().indexOf(delimiter, prefix.length());
338+
if (nextSlash < 0) {
339+
return false;
340+
}
341+
String folderName = so.getName().substring(0, nextSlash + 1);
342+
if (folders.containsKey(folderName)) {
343+
return true;
344+
}
345+
StorageObject fakeFolder = new StorageObject();
346+
fakeFolder.setName(folderName);
347+
fakeFolder.setBucket(so.getBucket());
348+
fakeFolder.setGeneration(so.getGeneration());
349+
folders.put(folderName, fakeFolder);
350+
return true;
351+
}
337352
}

0 commit comments

Comments
 (0)