Skip to content

Commit a1642ed

Browse files
jean-philippe-martinchingor13
jean-philippe-martin
authored and
chingor13
committed
---
yaml --- r: 12239 b: refs/heads/autosynth-os-login c: 324085e h: refs/heads/master i: 12237: 1f3ce20 12235: 39021e1 12231: bfccb58 12223: a0a1881
1 parent 992e273 commit a1642ed

9 files changed

Lines changed: 190 additions & 34 deletions

File tree

‎[refs]‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,7 @@ refs/heads/autosynth-firestore: 92b27fbc8855c9902168695abb0a8f1f433b750b
136136
refs/heads/autosynth-iot: 9d732be07d99843d8cb53d34ec0837328a807fce
137137
refs/heads/autosynth-kms: dcc6e15d68759010c8735cc868135bd7e6c1cc5f
138138
refs/heads/autosynth-language: 8972866b016473559702424205ce5569de47b34d
139-
refs/heads/autosynth-os-login: b9593290347854a2c793eab4490a6a05ee8d9338
139+
refs/heads/autosynth-os-login: 324085e830c35debde0b5e8d89bc629496481b27
140140
refs/heads/autosynth-redis: 1259b3a2193c6ee73b91064b5ebdedd6b51396bc
141141
refs/heads/autosynth-scheduler: 22f042c3461cee9346c46fdade5c8fad1a80f431
142142
refs/heads/autosynth-securitycenter: 1b300406bfc0f022d0ae98286c419d54d08d2ad4

‎branches/autosynth-os-login/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageConfiguration.java‎

Lines changed: 41 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,13 @@
2020
import static com.google.common.base.Preconditions.checkNotNull;
2121

2222
import com.google.auto.value.AutoValue;
23+
import com.google.common.collect.ImmutableList;
2324

2425
import javax.annotation.Nullable;
26+
import javax.net.ssl.SSLException;
27+
import java.io.EOFException;
28+
import java.net.SocketException;
29+
import java.net.SocketTimeoutException;
2530
import java.util.Map;
2631

2732
/**
@@ -89,6 +94,17 @@ public abstract class CloudStorageConfiguration {
8994
*/
9095
public abstract boolean useUserProjectOnlyForRequesterPaysBuckets();
9196

97+
/**
98+
* Returns the set of HTTP error codes that will be retried, in addition to the normally
99+
* retryable ones.
100+
*/
101+
public abstract ImmutableList<Integer> retryableHttpCodes();
102+
103+
/**
104+
* Returns the set of exceptions for which we'll try a channel reopen if maxChannelReopens
105+
* is positive.
106+
*/
107+
public abstract ImmutableList<Class<? extends Exception>> reopenableExceptions();
92108

93109
/**
94110
* Creates a new builder, initialized with the following settings:
@@ -118,6 +134,10 @@ public static final class Builder {
118134
private @Nullable String userProject = null;
119135
// This of this as "clear userProject if not RequesterPays"
120136
private boolean useUserProjectOnlyForRequesterPaysBuckets = false;
137+
private ImmutableList<Integer> retryableHttpCodes = ImmutableList.of(500, 502, 503);
138+
private ImmutableList<Class<? extends Exception>> reopenableExceptions =
139+
ImmutableList.<Class<? extends Exception>>of(
140+
SSLException.class, EOFException.class, SocketException.class, SocketTimeoutException.class);
121141

122142
/**
123143
* Changes current working directory for new filesystem. This defaults to the root directory.
@@ -186,6 +206,16 @@ public Builder autoDetectRequesterPays(boolean value) {
186206
return this;
187207
}
188208

209+
public Builder retryableHttpCodes(ImmutableList<Integer> value) {
210+
retryableHttpCodes = value;
211+
return this;
212+
}
213+
214+
public Builder reopenableExceptions(ImmutableList<Class<? extends Exception>> values) {
215+
reopenableExceptions = values;
216+
return this;
217+
}
218+
189219
/**
190220
* Creates new instance without destroying builder.
191221
*/
@@ -198,7 +228,9 @@ public CloudStorageConfiguration build() {
198228
blockSize,
199229
maxChannelReopens,
200230
userProject,
201-
useUserProjectOnlyForRequesterPaysBuckets);
231+
useUserProjectOnlyForRequesterPaysBuckets,
232+
retryableHttpCodes,
233+
reopenableExceptions);
202234
}
203235

204236
Builder(CloudStorageConfiguration toModify) {
@@ -210,6 +242,8 @@ public CloudStorageConfiguration build() {
210242
maxChannelReopens = toModify.maxChannelReopens();
211243
userProject = toModify.userProject();
212244
useUserProjectOnlyForRequesterPaysBuckets = toModify.useUserProjectOnlyForRequesterPaysBuckets();
245+
retryableHttpCodes = toModify.retryableHttpCodes();
246+
reopenableExceptions = toModify.reopenableExceptions();
213247
}
214248

215249
Builder() {}
@@ -250,6 +284,12 @@ static private CloudStorageConfiguration fromMap(Builder builder, Map<String, ?>
250284
case "useUserProjectOnlyForRequesterPaysBuckets":
251285
builder.autoDetectRequesterPays((Boolean) entry.getValue());
252286
break;
287+
case "retryableHttpCodes":
288+
builder.retryableHttpCodes((ImmutableList<Integer>) entry.getValue());
289+
break;
290+
case "reopenableExceptions":
291+
builder.reopenableExceptions((ImmutableList<Class<? extends Exception>>) entry.getValue());
292+
break;
253293
default:
254294
throw new IllegalArgumentException(entry.getKey());
255295
}

‎branches/autosynth-os-login/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemProvider.java‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -349,6 +349,7 @@ private SeekableByteChannel newReadChannel(Path path, Set<? extends OpenOption>
349349
cloudPath.getBlobId(),
350350
0,
351351
maxChannelReopens,
352+
cloudPath.getFileSystem().config(),
352353
userProject,
353354
blobSourceOptions.toArray(new BlobSourceOption[blobSourceOptions.size()]));
354355
}
@@ -461,7 +462,8 @@ public boolean deleteIfExists(Path path) throws IOException {
461462
throw new CloudStoragePseudoDirectoryException(cloudPath);
462463
}
463464

464-
final CloudStorageRetryHandler retryHandler = new CloudStorageRetryHandler(CloudStorageUtil.getMaxChannelReopensFromPath(path));
465+
final CloudStorageRetryHandler retryHandler = new CloudStorageRetryHandler(
466+
cloudPath.getFileSystem().config());
465467
// Loop will terminate via an exception if all retries are exhausted
466468
while (true) {
467469
try {
@@ -586,7 +588,8 @@ public void copy(Path source, Path target, CopyOption... options) throws IOExcep
586588
throw new CloudStoragePseudoDirectoryException(toPath);
587589
}
588590

589-
final CloudStorageRetryHandler retryHandler = new CloudStorageRetryHandler(CloudStorageUtil.getMaxChannelReopensFromPath(source));
591+
final CloudStorageRetryHandler retryHandler = new CloudStorageRetryHandler(
592+
fromPath.getFileSystem().config());
590593
// Loop will terminate via an exception if all retries are exhausted
591594
while (true) {
592595
try {
@@ -672,11 +675,12 @@ public void checkAccess(Path path, AccessMode... modes) throws IOException {
672675
}
673676
}
674677

675-
final CloudStorageRetryHandler retryHandler = new CloudStorageRetryHandler(CloudStorageUtil.getMaxChannelReopensFromPath(path));
678+
final CloudStoragePath cloudPath = CloudStorageUtil.checkPath(path);
679+
final CloudStorageRetryHandler retryHandler = new CloudStorageRetryHandler(
680+
cloudPath.getFileSystem().config());
676681
// Loop will terminate via an exception if all retries are exhausted
677682
while (true) {
678683
try {
679-
CloudStoragePath cloudPath = CloudStorageUtil.checkPath(path);
680684
boolean nullId;
681685
if (isNullOrEmpty(userProject)) {
682686
nullId = storage.get(
@@ -725,11 +729,12 @@ public <A extends BasicFileAttributes> A readAttributes(
725729
}
726730
initStorage();
727731

728-
final CloudStorageRetryHandler retryHandler = new CloudStorageRetryHandler(CloudStorageUtil.getMaxChannelReopensFromPath(path));
732+
final CloudStoragePath cloudPath = CloudStorageUtil.checkPath(path);
733+
final CloudStorageRetryHandler retryHandler = new CloudStorageRetryHandler(
734+
cloudPath.getFileSystem().config());
729735
// Loop will terminate via an exception if all retries are exhausted
730736
while (true) {
731737
try {
732-
CloudStoragePath cloudPath = CloudStorageUtil.checkPath(path);
733738
BlobInfo blobInfo = null;
734739
try {
735740
BlobId blobId = cloudPath.getBlobId();
@@ -811,7 +816,8 @@ public DirectoryStream<Path> newDirectoryStream(final Path dir, final Filter<? s
811816
checkNotNull(filter);
812817
initStorage();
813818

814-
final CloudStorageRetryHandler retryHandler = new CloudStorageRetryHandler(CloudStorageUtil.getMaxChannelReopensFromPath(dir));
819+
final CloudStorageRetryHandler retryHandler = new CloudStorageRetryHandler(
820+
cloudPath.getFileSystem().config());
815821
// Loop will terminate via an exception if all retries are exhausted
816822
while (true) {
817823
try {

‎branches/autosynth-os-login/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageReadChannel.java‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ final class CloudStorageReadChannel implements SeekableByteChannel {
5757
final int maxRetries;
5858
// open options, we keep them around for reopens.
5959
final BlobSourceOption[] blobSourceOptions;
60+
final CloudStorageConfiguration config;
6061
private ReadChannel channel;
6162
private long position;
6263
private long size;
@@ -67,6 +68,7 @@ final class CloudStorageReadChannel implements SeekableByteChannel {
6768
/**
6869
* @param maxChannelReopens max number of times to try re-opening the channel if it closes on us
6970
* unexpectedly.
71+
* @param config configuration for what to retry on.
7072
* @param blobSourceOptions BlobSourceOption.userProject if you want to pay the charges (required
7173
* for requester-pays buckets). Note:
7274
* Buckets that have Requester Pays disabled still accept requests that include a billing
@@ -79,17 +81,18 @@ final class CloudStorageReadChannel implements SeekableByteChannel {
7981
*/
8082
@CheckReturnValue
8183
@SuppressWarnings("resource")
82-
static CloudStorageReadChannel create(Storage gcsStorage, BlobId file, long position, int maxChannelReopens, @Nullable String userProject, BlobSourceOption... blobSourceOptions)
84+
static CloudStorageReadChannel create(Storage gcsStorage, BlobId file, long position, int maxChannelReopens, final CloudStorageConfiguration config, @Nullable String userProject, BlobSourceOption... blobSourceOptions)
8385
throws IOException {
84-
return new CloudStorageReadChannel(gcsStorage, file, position, maxChannelReopens, userProject, blobSourceOptions);
86+
return new CloudStorageReadChannel(gcsStorage, file, position, maxChannelReopens, config, userProject, blobSourceOptions);
8587
}
8688

87-
private CloudStorageReadChannel(Storage gcsStorage, BlobId file, long position, int maxChannelReopens, @Nullable String userProject, BlobSourceOption... blobSourceOptions) throws IOException {
89+
private CloudStorageReadChannel(Storage gcsStorage, BlobId file, long position, int maxChannelReopens, final CloudStorageConfiguration config, @Nullable String userProject, BlobSourceOption... blobSourceOptions) throws IOException {
8890
this.gcsStorage = gcsStorage;
8991
this.file = file;
9092
this.position = position;
9193
this.maxChannelReopens = maxChannelReopens;
9294
this.maxRetries = Math.max(3, maxChannelReopens);
95+
this.config = config;
9396
// get the generation, enshrine that in our options
9497
fetchSize(gcsStorage, userProject, file);
9598
List options = Lists.newArrayList(blobSourceOptions);
@@ -133,7 +136,7 @@ public int read(ByteBuffer dst) throws IOException {
133136
synchronized (this) {
134137
checkOpen();
135138
int amt;
136-
final CloudStorageRetryHandler retryHandler = new CloudStorageRetryHandler(maxRetries, maxChannelReopens);
139+
final CloudStorageRetryHandler retryHandler = new CloudStorageRetryHandler(maxRetries, maxChannelReopens, config);
137140
dst.mark();
138141
while (true) {
139142
try {
@@ -203,7 +206,7 @@ private void checkOpen() throws ClosedChannelException {
203206
}
204207

205208
private long fetchSize(Storage gcsStorage, @Nullable String userProject, BlobId file) throws IOException {
206-
final CloudStorageRetryHandler retryHandler = new CloudStorageRetryHandler(maxRetries, maxChannelReopens);
209+
final CloudStorageRetryHandler retryHandler = new CloudStorageRetryHandler(maxRetries, maxChannelReopens, config);
207210

208211
while (true) {
209212
try {

‎branches/autosynth-os-login/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageRetryHandler.java‎

Lines changed: 58 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,7 @@
1717
package com.google.cloud.storage.contrib.nio;
1818

1919
import com.google.cloud.storage.StorageException;
20-
21-
import java.io.EOFException;
22-
import java.net.SocketException;
23-
import java.net.SocketTimeoutException;
24-
import javax.net.ssl.SSLException;
20+
import com.google.common.annotations.VisibleForTesting;
2521

2622
/**
2723
* Simple counter class to keep track of retry and reopen attempts when StorageExceptions are
@@ -34,26 +30,62 @@ public class CloudStorageRetryHandler {
3430
private long totalWaitTime; // in milliseconds
3531
private final int maxRetries;
3632
private final int maxReopens;
33+
private final CloudStorageConfiguration config;
34+
3735

3836
/**
3937
* Create a CloudStorageRetryHandler with the maximum retries and reopens set to the same value.
4038
*
4139
* @param maxRetriesAndReopens value for both maxRetries and maxReopens
40+
*
41+
* @deprecated use CloudStorageRetryHandler(CloudStorageConfiguration) instead.
4242
*/
43+
@java.lang.Deprecated
4344
public CloudStorageRetryHandler(final int maxRetriesAndReopens) {
4445
this.maxRetries = maxRetriesAndReopens;
4546
this.maxReopens = maxRetriesAndReopens;
47+
// we're just using the retry parameters from the config, so it's OK to have a default.
48+
this.config = CloudStorageConfiguration.DEFAULT;
4649
}
4750

48-
/**
51+
/**
4952
* Create a CloudStorageRetryHandler with the maximum retries and reopens set to different values.
5053
*
5154
* @param maxRetries maximum number of retries
5255
* @param maxReopens maximum number of reopens
56+
*
57+
* @deprecated use CloudStorageRetryHandler(CloudStorageConfiguration) instead.
5358
*/
59+
@java.lang.Deprecated
5460
public CloudStorageRetryHandler(final int maxRetries, final int maxReopens) {
5561
this.maxRetries = maxRetries;
5662
this.maxReopens = maxReopens;
63+
// we're just using the retry parameters from the config, so it's OK to have a default.
64+
this.config = CloudStorageConfiguration.DEFAULT;
65+
}
66+
67+
/**
68+
* Create a CloudStorageRetryHandler with the maximum retries and reopens set to the same value.
69+
*
70+
* @param config - configuration for reopens and retryable codes.
71+
*/
72+
public CloudStorageRetryHandler(final CloudStorageConfiguration config) {
73+
this.maxRetries = config.maxChannelReopens();
74+
this.maxReopens = config.maxChannelReopens();
75+
this.config = config;
76+
}
77+
78+
/**
79+
* Create a CloudStorageRetryHandler with the maximum retries and reopens set to different values.
80+
*
81+
* @param maxRetries maximum number of retries
82+
* @param maxReopens maximum number of reopens (overrides what's in the config)
83+
* @param config http codes we'll retry on, and exceptions we'll reopen on.
84+
*/
85+
public CloudStorageRetryHandler(final int maxRetries, final int maxReopens, final CloudStorageConfiguration config) {
86+
this.maxRetries = maxRetries;
87+
this.maxReopens = maxReopens;
88+
this.config = config;
5789
}
5890

5991
/**
@@ -143,25 +175,36 @@ void sleepForAttempt(int attempt) {
143175
* @param exs StorageException to test
144176
* @return true if exs is a retryable error, otherwise false
145177
*/
146-
private static boolean isRetryable(final StorageException exs) {
147-
return exs.isRetryable() || exs.getCode() == 500 || exs.getCode() == 502 || exs.getCode() == 503;
178+
@VisibleForTesting
179+
boolean isRetryable(final StorageException exs) {
180+
if (exs.isRetryable()) {
181+
return true;
182+
}
183+
for (int code : config.retryableHttpCodes()) {
184+
if (exs.getCode() == code) {
185+
return true;
186+
}
187+
}
188+
return false;
148189
}
149190

150191
/**
151192
* @param exs StorageException to test
152193
* @return true if exs is an error that can be resolved via a channel reopen, otherwise false
153194
*/
154-
private static boolean isReopenable(final StorageException exs) {
195+
@VisibleForTesting
196+
boolean isReopenable(final StorageException exs) {
155197
Throwable throwable = exs;
156198
// ensures finite iteration
157199
int maxDepth = 20;
158200
while (throwable != null && maxDepth-- > 0) {
159-
if ((throwable.getMessage() != null
160-
&& throwable.getMessage().contains("Connection closed prematurely"))
161-
|| throwable instanceof SSLException
162-
|| throwable instanceof EOFException
163-
|| throwable instanceof SocketException
164-
|| throwable instanceof SocketTimeoutException) {
201+
for (Class<? extends Exception> reopenableException : config.reopenableExceptions()) {
202+
if (reopenableException.isInstance(throwable)) {
203+
return true;
204+
}
205+
}
206+
if (throwable.getMessage() != null
207+
&& throwable.getMessage().contains("Connection closed prematurely")) {
165208
return true;
166209
}
167210
throwable = throwable.getCause();

0 commit comments

Comments
 (0)