Skip to content

Commit 19262b7

Browse files
jean-philippe-martinmichaelbausor
jean-philippe-martin
authored and
michaelbausor
committed
Add setDefaultCloudStorageConfiguration (#2159)
* Add setDefaultCloudStorageConfiguration This allows the main program to set default options that will be used by all subsequent filesystem objects. This is useful for a main program that has libraries that create their own Path objects via the recommended way (Paths.get(URI.create("gs://..."))). * Check default propagates to open channel * Test NIO reopen defaults are overridable * Improve description of default overriding
1 parent 797d665 commit 19262b7

File tree

6 files changed

+125
-5
lines changed

6 files changed

+125
-5
lines changed

‎google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageConfiguration.java‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ public abstract class CloudStorageConfiguration {
7272
* <li>Performing I/O on paths with extra slashes, e.g. {@code a//b} will throw an error.
7373
* <li>The prefix slash on absolute paths will be removed when converting to an object name.
7474
* <li>Pseudo-directories are enabled, so any path with a trailing slash is a fake directory.
75+
* <li>Channel re-opens are disabled.
7576
* </ul>
7677
*/
7778
public static Builder builder() {
@@ -159,11 +160,27 @@ public CloudStorageConfiguration build() {
159160
maxChannelReopens);
160161
}
161162

163+
Builder(CloudStorageConfiguration toModify) {
164+
workingDirectory = toModify.workingDirectory();
165+
permitEmptyPathComponents = toModify.permitEmptyPathComponents();
166+
stripPrefixSlash = toModify.stripPrefixSlash();
167+
usePseudoDirectories = toModify.usePseudoDirectories();
168+
blockSize = toModify.blockSize();
169+
maxChannelReopens = toModify.maxChannelReopens();
170+
}
171+
162172
Builder() {}
163173
}
164174

165175
static CloudStorageConfiguration fromMap(Map<String, ?> env) {
166-
Builder builder = builder();
176+
return fromMap(builder(), env);
177+
}
178+
179+
static CloudStorageConfiguration fromMap(CloudStorageConfiguration defaultValues, Map<String, ?> env) {
180+
return fromMap(new Builder(defaultValues), env);
181+
}
182+
183+
static private CloudStorageConfiguration fromMap(Builder builder, Map<String, ?> env) {
167184
for (Map.Entry<String, ?> entry : env.entrySet()) {
168185
switch (entry.getKey()) {
169186
case "workingDirectory":

‎google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystem.java‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,27 @@ public final class CloudStorageFileSystem extends FileSystem {
6262
private final String bucket;
6363
private final CloudStorageConfiguration config;
6464

65+
// Users can change this: then this affects every filesystem object created
66+
// later, including via SPI. This is meant to be done only once, at the beginning
67+
// of some main program, in order to force all libraries to use some settings we like.
68+
// Libraries should never call this. It'll cause surprise to the writers of the main
69+
// program and they'll be unhappy. Instead, create your own filesystem object with the
70+
// right configuration and pass it along.
71+
private static CloudStorageConfiguration userSpecifiedDefault = CloudStorageConfiguration.DEFAULT;
72+
73+
// Don't call this one, call the one in CloudStorageFileSystemProvider.
74+
static void setDefaultCloudStorageConfiguration(CloudStorageConfiguration config) {
75+
if (null == config) {
76+
userSpecifiedDefault = CloudStorageConfiguration.DEFAULT;
77+
} else {
78+
userSpecifiedDefault = config;
79+
}
80+
}
81+
82+
static CloudStorageConfiguration getDefaultCloudStorageConfiguration() {
83+
return userSpecifiedDefault;
84+
}
85+
6586
/**
6687
* Returns Google Cloud Storage {@link FileSystem} object for {@code bucket}.
6788
*
@@ -79,7 +100,7 @@ public final class CloudStorageFileSystem extends FileSystem {
79100
*/
80101
@CheckReturnValue
81102
public static CloudStorageFileSystem forBucket(String bucket) {
82-
return forBucket(bucket, CloudStorageConfiguration.DEFAULT);
103+
return forBucket(bucket, userSpecifiedDefault);
83104
}
84105

85106
/**

‎google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemProvider.java‎

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,30 @@ public static void setStorageOptions(StorageOptions newStorageOptions) {
137137
futureStorageOptions = newStorageOptions;
138138
}
139139

140+
/**
141+
* Changes the default configuration for every filesystem object created
142+
* from here on, including via SPI. If null then future filesystem objects
143+
* will have the factory default configuration.
144+
*
145+
* <p>If options are specified later then they override the defaults.
146+
* Methods that take a whole CloudStorageConfiguration (eg.
147+
* CloudStorageFileSystem.forBucket) will completely override the defaults.
148+
* Methods that take individual options (eg.
149+
* CloudStorageFileSystemProvier.newFileSystem) will override only these options;
150+
* the rest will be taken from the defaults specified here.
151+
*
152+
* <p>This is meant to be done only once, at the beginning of some main program,
153+
* in order to force all libraries to use some settings we like.
154+
*
155+
* <p>Libraries should never call this. If you're a library then, instead, create your own
156+
* filesystem object with the right configuration and pass it along.
157+
*
158+
* @param newDefault new default CloudStorageConfiguration
159+
*/
160+
public static void setDefaultCloudStorageConfiguration(CloudStorageConfiguration newDefault) {
161+
CloudStorageFileSystem.setDefaultCloudStorageConfiguration(newDefault);
162+
}
163+
140164
/**
141165
* Default constructor which should only be called by Java SPI.
142166
*
@@ -208,7 +232,11 @@ && isNullOrEmpty(uri.getUserInfo()),
208232
uri);
209233
CloudStorageUtil.checkBucket(uri.getHost());
210234
initStorage();
211-
return new CloudStorageFileSystem(this, uri.getHost(), CloudStorageConfiguration.fromMap(env));
235+
return new CloudStorageFileSystem(
236+
this,
237+
uri.getHost(),
238+
CloudStorageConfiguration.fromMap(
239+
CloudStorageFileSystem.getDefaultCloudStorageConfiguration(), env));
212240
}
213241

214242
@Override

‎google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageOptions.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ public static CloudStorageOption.OpenCopy withBlockSize(int size) {
9292
}
9393

9494
/**
95-
* Sets the max number of times that the channel can be reopen if reading
95+
* Sets the max number of times that the channel can be reopened if reading
9696
* fails because the channel unexpectedly closes.
9797
*
9898
* <p>The default is 0.

‎google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageReadChannel.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import com.google.cloud.storage.BlobInfo;
2222
import com.google.cloud.storage.Storage;
2323
import com.google.cloud.storage.StorageException;
24+
import com.google.common.annotations.VisibleForTesting;
2425

2526
import javax.annotation.CheckReturnValue;
2627
import javax.annotation.concurrent.ThreadSafe;
@@ -49,7 +50,8 @@ final class CloudStorageReadChannel implements SeekableByteChannel {
4950
private final Storage gcsStorage;
5051
private final BlobId file;
5152
// max # of times we may reopen the file
52-
private final int maxChannelReopens;
53+
@VisibleForTesting
54+
final int maxChannelReopens;
5355
// how many times we re-opened the file
5456
private int reopens;
5557
private ReadChannel channel;

‎google-cloud-contrib/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemTest.java‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333

3434
import java.io.IOException;
3535
import java.net.URI;
36+
import java.nio.channels.SeekableByteChannel;
3637
import java.nio.file.FileSystem;
3738
import java.nio.file.FileSystems;
3839
import java.nio.file.Files;
@@ -69,6 +70,57 @@ public void before() {
6970
CloudStorageFileSystemProvider.setStorageOptions(LocalStorageHelper.getOptions());
7071
}
7172

73+
@Test
74+
public void checkDefaultOptions() throws IOException {
75+
// 1. We get the normal default if we don't do anything special.
76+
Path path = Paths.get(URI.create("gs://bucket/file"));
77+
CloudStorageFileSystem gcs = (CloudStorageFileSystem)path.getFileSystem();
78+
assertThat(gcs.config().maxChannelReopens()).isEqualTo(0);
79+
80+
// 2(a). Override the default, and see it reflected.
81+
CloudStorageFileSystemProvider.setDefaultCloudStorageConfiguration(
82+
CloudStorageConfiguration.builder()
83+
.maxChannelReopens(123).build());
84+
Path path2 = Paths.get(URI.create("gs://newbucket/file"));
85+
CloudStorageFileSystem gcs2 = (CloudStorageFileSystem)path2.getFileSystem();
86+
assertThat(gcs2.config().maxChannelReopens()).isEqualTo(123);
87+
88+
// 2(b) ...even reflected if we try to open a file.
89+
try (FileSystem fs = CloudStorageFileSystem.forBucket("bucket")) {
90+
CloudStorageFileSystem csfs = (CloudStorageFileSystem)fs;
91+
assertThat(csfs.config().maxChannelReopens()).isEqualTo(123);
92+
Files.write(fs.getPath("/angel"), ALONE.getBytes(UTF_8));
93+
path2 = Paths.get(URI.create("gs://bucket/angel"));
94+
try (SeekableByteChannel seekableByteChannel = Files.newByteChannel(path2)) {
95+
CloudStorageReadChannel cloudChannel = (CloudStorageReadChannel) seekableByteChannel;
96+
assertThat(cloudChannel.maxChannelReopens).isEqualTo(123);
97+
}
98+
}
99+
100+
// 4. Clean up.
101+
CloudStorageFileSystemProvider.setDefaultCloudStorageConfiguration(null);
102+
Path path3 = Paths.get(URI.create("gs://newbucket/file"));
103+
CloudStorageFileSystem gcs3 = (CloudStorageFileSystem)path3.getFileSystem();
104+
assertThat(gcs3.config().maxChannelReopens()).isEqualTo(0);
105+
}
106+
107+
@Test
108+
public void canOverrideDefaultOptions() throws IOException {
109+
// Set a new default.
110+
CloudStorageFileSystemProvider.setDefaultCloudStorageConfiguration(
111+
CloudStorageConfiguration.builder()
112+
.maxChannelReopens(123).build());
113+
114+
// This code wants its own value.
115+
try (FileSystem fs = CloudStorageFileSystem.forBucket("bucket", CloudStorageConfiguration.builder().maxChannelReopens(7).build())) {
116+
CloudStorageFileSystem csfs = (CloudStorageFileSystem)fs;
117+
assertThat(csfs.config().maxChannelReopens()).isEqualTo(7);
118+
}
119+
120+
// Clean up.
121+
CloudStorageFileSystemProvider.setDefaultCloudStorageConfiguration(null);
122+
}
123+
72124
@Test
73125
public void testGetPath() throws IOException {
74126
try (FileSystem fs = CloudStorageFileSystem.forBucket("bucket")) {

0 commit comments

Comments
 (0)