Skip to content

Commit 73b2917

Browse files
jean-philippe-martinmziccard
jean-philippe-martin
authored and
mziccard
committed
Implement bare-bones blob listing
1 parent 2125f8a commit 73b2917

3 files changed

Lines changed: 119 additions & 10 deletions

File tree

‎gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemProvider.java‎

Lines changed: 47 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626

2727
import com.google.auto.service.AutoService;
2828
import com.google.cloud.storage.Acl;
29+
import com.google.cloud.storage.Blob;
2930
import com.google.cloud.storage.BlobId;
3031
import com.google.cloud.storage.BlobInfo;
3132
import com.google.cloud.storage.CopyWriter;
@@ -35,6 +36,7 @@
3536
import com.google.common.annotations.VisibleForTesting;
3637
import com.google.common.base.MoreObjects;
3738
import com.google.common.base.Throwables;
39+
import com.google.common.collect.AbstractIterator;
3840
import com.google.common.primitives.Ints;
3941

4042
import java.io.BufferedInputStream;
@@ -45,6 +47,7 @@
4547
import java.nio.file.AccessMode;
4648
import java.nio.file.AtomicMoveNotSupportedException;
4749
import java.nio.file.CopyOption;
50+
import java.nio.file.DirectoryIteratorException;
4851
import java.nio.file.DirectoryStream;
4952
import java.nio.file.DirectoryStream.Filter;
5053
import java.nio.file.FileAlreadyExistsException;
@@ -63,11 +66,11 @@
6366
import java.util.ArrayList;
6467
import java.util.Collections;
6568
import java.util.HashMap;
69+
import java.util.Iterator;
6670
import java.util.List;
6771
import java.util.Map;
6872
import java.util.Objects;
6973
import java.util.Set;
70-
7174
import javax.annotation.Nullable;
7275
import javax.annotation.concurrent.ThreadSafe;
7376

@@ -83,6 +86,33 @@ public final class CloudStorageFileSystemProvider extends FileSystemProvider {
8386
// used only when we create a new instance of CloudStorageFileSystemProvider.
8487
private static StorageOptions storageOptions;
8588

89+
private static class LazyPathIterator extends AbstractIterator<Path> {
90+
private final Iterator<Blob> blobIterator;
91+
private final Filter<? super Path> filter;
92+
private final CloudStorageFileSystem fileSystem;
93+
94+
LazyPathIterator(CloudStorageFileSystem fileSystem, Iterator<Blob> blobIterator, Filter<? super Path> filter) {
95+
this.blobIterator = blobIterator;
96+
this.filter = filter;
97+
this.fileSystem = fileSystem;
98+
}
99+
100+
@Override
101+
protected Path computeNext() {
102+
while (blobIterator.hasNext()) {
103+
Path path = fileSystem.getPath(blobIterator.next().name());
104+
try {
105+
if (filter.accept(path)) {
106+
return path;
107+
}
108+
} catch (IOException ex) {
109+
throw new DirectoryIteratorException(ex);
110+
}
111+
}
112+
return endOfData();
113+
}
114+
}
115+
86116
/**
87117
* Sets options that are only used by the constructor.
88118
*/
@@ -527,13 +557,23 @@ public void createDirectory(Path dir, FileAttribute<?>... attrs) {
527557
checkNotNullArray(attrs);
528558
}
529559

530-
/**
531-
* Throws {@link UnsupportedOperationException} because this feature hasn't been implemented yet.
532-
*/
533560
@Override
534-
public DirectoryStream<Path> newDirectoryStream(Path dir, Filter<? super Path> filter) {
535-
// TODO: Implement me.
536-
throw new UnsupportedOperationException();
561+
public DirectoryStream<Path> newDirectoryStream(Path dir, final Filter<? super Path> filter) {
562+
final CloudStoragePath cloudPath = checkPath(dir);
563+
checkNotNull(filter);
564+
String prefix = cloudPath.toString();
565+
final Iterator<Blob> blobIterator = storage.list(cloudPath.bucket(), Storage.BlobListOption.prefix(prefix), Storage.BlobListOption.fields()).iterateAll();
566+
return new DirectoryStream<Path>() {
567+
@Override
568+
public Iterator<Path> iterator() {
569+
return new LazyPathIterator(cloudPath.getFileSystem(), blobIterator, filter);
570+
}
571+
572+
@Override
573+
public void close() throws IOException {
574+
// Does nothing since there's nothing to close. Commenting this method to quiet codacy.
575+
}
576+
};
537577
}
538578

539579
/**

‎gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemTest.java‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,10 @@
3333
import java.nio.file.FileSystem;
3434
import java.nio.file.FileSystems;
3535
import java.nio.file.Files;
36+
import java.nio.file.Path;
3637
import java.nio.file.Paths;
38+
import java.util.ArrayList;
39+
import java.util.List;
3740

3841
/**
3942
* Unit tests for {@link CloudStorageFileSystem}.
@@ -132,4 +135,27 @@ public void testNullness() throws IOException, NoSuchMethodException, SecurityEx
132135
tester.testAllPublicInstanceMethods(fs);
133136
}
134137
}
138+
139+
@Test
140+
public void testListFiles() throws IOException {
141+
try (FileSystem fs = CloudStorageFileSystem.forBucket("bucket")) {
142+
List<Path> goodPaths = new ArrayList<>();
143+
List<Path> paths = new ArrayList<>();
144+
goodPaths.add(fs.getPath("dir/angel"));
145+
goodPaths.add(fs.getPath("dir/alone"));
146+
paths.add(fs.getPath("dir/dir2/another_angel"));
147+
paths.add(fs.getPath("atroot"));
148+
paths.addAll(goodPaths);
149+
goodPaths.add(fs.getPath("dir/dir2/"));
150+
for (Path path : paths) {
151+
Files.write(path, ALONE.getBytes(UTF_8));
152+
}
153+
154+
List<Path> got = new ArrayList<>();
155+
for (Path path : Files.newDirectoryStream(fs.getPath("/dir/"))) {
156+
got.add(path);
157+
}
158+
assertThat(got).containsExactlyElementsIn(goodPaths);
159+
}
160+
}
135161
}

‎gcloud-java-storage/src/main/java/com/google/cloud/storage/testing/FakeStorageRpc.java‎

Lines changed: 46 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,10 @@
2828
import java.io.InputStream;
2929
import java.math.BigInteger;
3030
import java.nio.file.FileAlreadyExistsException;
31+
import java.util.ArrayList;
3132
import java.util.Arrays;
3233
import java.util.HashMap;
34+
import java.util.List;
3335
import java.util.Map;
3436

3537
import javax.annotation.concurrent.NotThreadSafe;
@@ -93,8 +95,49 @@ public Tuple<String, Iterable<Bucket>> list(Map<Option, ?> options) throws Stora
9395
@Override
9496
public Tuple<String, Iterable<StorageObject>> list(String bucket, Map<Option, ?> options)
9597
throws StorageException {
96-
potentiallyThrow(options);
97-
return null;
98+
String preprefix = "";
99+
for (Map.Entry<Option, ?> e : options.entrySet()) {
100+
switch (e.getKey()) {
101+
case PREFIX:
102+
preprefix = (String) e.getValue();
103+
if (preprefix.startsWith("/")) {
104+
preprefix = preprefix.substring(1);
105+
}
106+
break;
107+
case FIELDS:
108+
// ignore and return all the fields
109+
break;
110+
default:
111+
throw new UnsupportedOperationException("Unknown option: " + e.getKey());
112+
}
113+
}
114+
final String prefix = preprefix;
115+
116+
List<StorageObject> values = new ArrayList<>();
117+
Map<String, StorageObject> folders = new HashMap<>();
118+
for (StorageObject so : stuff.values()) {
119+
if (!so.getName().startsWith(prefix)) {
120+
continue;
121+
}
122+
int nextSlash = so.getName().indexOf("/", prefix.length());
123+
if (nextSlash >= 0) {
124+
String folderName = so.getName().substring(0, nextSlash + 1);
125+
if (folders.containsKey(folderName)) {
126+
continue;
127+
}
128+
StorageObject fakeFolder = new StorageObject();
129+
fakeFolder.setName(folderName);
130+
fakeFolder.setBucket(so.getBucket());
131+
fakeFolder.setGeneration(so.getGeneration());
132+
folders.put(folderName, fakeFolder);
133+
continue;
134+
}
135+
values.add(so);
136+
}
137+
values.addAll(folders.values());
138+
// null cursor to indicate there is no more data (empty string would cause us to be called again).
139+
// The type cast seems to be necessary to help Java's typesystem remember that collections are iterable.
140+
return Tuple.of(null, (Iterable<StorageObject>) values);
98141
}
99142

100143
/**
@@ -113,7 +156,7 @@ public Bucket get(Bucket bucket, Map<Option, ?> options) throws StorageException
113156
public StorageObject get(StorageObject object, Map<Option, ?> options) throws StorageException {
114157
// we allow the "ID" option because we need to, but then we give a whole answer anyways
115158
// because the caller won't mind the extra fields.
116-
if (throwIfOption && !options.isEmpty() && options.size()>1
159+
if (throwIfOption && !options.isEmpty() && options.size() > 1
117160
&& options.keySet().toArray()[0] != Storage.BlobGetOption.fields(Storage.BlobField.ID)) {
118161
throw new UnsupportedOperationException();
119162
}

0 commit comments

Comments
 (0)