Skip to content
This repository was archived by the owner on Apr 7, 2026. It is now read-only.

Commit 76bcbec

Browse files
olavloiteolavloite
committed
fix: only retry RESOURCE_EXHAUSTED when throttling requests
1 parent 918d4af commit 76bcbec

2 files changed

Lines changed: 44 additions & 28 deletions

File tree

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java‎

Lines changed: 40 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@
6565
import com.google.cloud.spanner.admin.database.v1.stub.DatabaseAdminStub;
6666
import com.google.cloud.spanner.admin.database.v1.stub.DatabaseAdminStubSettings;
6767
import com.google.cloud.spanner.admin.database.v1.stub.GrpcDatabaseAdminCallableFactory;
68+
import com.google.cloud.spanner.admin.database.v1.stub.GrpcDatabaseAdminStub;
6869
import com.google.cloud.spanner.admin.instance.v1.stub.GrpcInstanceAdminStub;
6970
import com.google.cloud.spanner.admin.instance.v1.stub.InstanceAdminStub;
7071
import com.google.cloud.spanner.admin.instance.v1.stub.InstanceAdminStubSettings;
@@ -76,8 +77,8 @@
7677
import com.google.common.base.MoreObjects;
7778
import com.google.common.base.Preconditions;
7879
import com.google.common.collect.ImmutableList;
79-
import com.google.common.collect.ImmutableSet;
8080
import com.google.common.collect.ImmutableMap;
81+
import com.google.common.collect.ImmutableSet;
8182
import com.google.common.util.concurrent.RateLimiter;
8283
import com.google.common.util.concurrent.ThreadFactoryBuilder;
8384
import com.google.iam.v1.GetIamPolicyRequest;
@@ -450,33 +451,45 @@ public GapicSpannerRpc(final SpannerOptions options) {
450451
.setCredentialsProvider(credentialsProvider)
451452
.setStreamWatchdogProvider(watchdogProvider)
452453
.build();
453-
GrpcStubCallableFactory factory =
454-
new GrpcDatabaseAdminCallableFactory() {
455-
@Override
456-
public <RequestT, ResponseT> UnaryCallable<RequestT, ResponseT> createUnaryCallable(
457-
GrpcCallSettings<RequestT, ResponseT> grpcCallSettings,
458-
UnaryCallSettings<RequestT, ResponseT> callSettings,
459-
ClientContext clientContext) {
460-
// Make GetOperation retry on RESOURCE_EXHAUSTED to prevent polling operations from
461-
// failing with an Administrative requests limit exceeded error.
462-
if (grpcCallSettings
463-
.getMethodDescriptor()
464-
.getFullMethodName()
465-
.equals("google.longrunning.Operations/GetOperation")) {
466-
Set<StatusCode.Code> codes =
467-
ImmutableSet.<StatusCode.Code>builderWithExpectedSize(
468-
callSettings.getRetryableCodes().size() + 1)
469-
.addAll(callSettings.getRetryableCodes())
470-
.add(StatusCode.Code.RESOURCE_EXHAUSTED)
471-
.build();
472-
callSettings = callSettings.toBuilder().setRetryableCodes(codes).build();
454+
455+
// Automatically retry RESOURCE_EXHAUSTED for GetOperation if auto-throttling of
456+
// administrative requests has been set. The GetOperation RPC is called repeatedly by gax
457+
// while polling long-running operations for their progress and can also cause these errors.
458+
// The default behavior is not to retry these errors, and this option should normally only be
459+
// enabled for (integration) testing.
460+
if (options.isAutoThrottleAdministrativeRequests()) {
461+
GrpcStubCallableFactory factory =
462+
new GrpcDatabaseAdminCallableFactory() {
463+
@Override
464+
public <RequestT, ResponseT> UnaryCallable<RequestT, ResponseT> createUnaryCallable(
465+
GrpcCallSettings<RequestT, ResponseT> grpcCallSettings,
466+
UnaryCallSettings<RequestT, ResponseT> callSettings,
467+
ClientContext clientContext) {
468+
// Make GetOperation retry on RESOURCE_EXHAUSTED to prevent polling operations from
469+
// failing with an Administrative requests limit exceeded error.
470+
if (grpcCallSettings
471+
.getMethodDescriptor()
472+
.getFullMethodName()
473+
.equals("google.longrunning.Operations/GetOperation")) {
474+
Set<StatusCode.Code> codes =
475+
ImmutableSet.<StatusCode.Code>builderWithExpectedSize(
476+
callSettings.getRetryableCodes().size() + 1)
477+
.addAll(callSettings.getRetryableCodes())
478+
.add(StatusCode.Code.RESOURCE_EXHAUSTED)
479+
.build();
480+
callSettings = callSettings.toBuilder().setRetryableCodes(codes).build();
481+
}
482+
return super.createUnaryCallable(grpcCallSettings, callSettings, clientContext);
473483
}
474-
return super.createUnaryCallable(grpcCallSettings, callSettings, clientContext);
475-
}
476-
};
477-
this.databaseAdminStub =
478-
new GrpcDatabaseAdminStubWithCustomCallableFactory(
479-
databaseAdminStubSettings, ClientContext.create(databaseAdminStubSettings), factory);
484+
};
485+
this.databaseAdminStub =
486+
new GrpcDatabaseAdminStubWithCustomCallableFactory(
487+
databaseAdminStubSettings,
488+
ClientContext.create(databaseAdminStubSettings),
489+
factory);
490+
} else {
491+
this.databaseAdminStub = GrpcDatabaseAdminStub.create(databaseAdminStubSettings);
492+
}
480493

481494
// Check whether the SPANNER_EMULATOR_HOST env var has been set, and if so, if the emulator is
482495
// actually running.

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GrpcDatabaseAdminStubWithCustomCallableFactory.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,16 @@
1818

1919
import com.google.api.gax.grpc.GrpcStubCallableFactory;
2020
import com.google.api.gax.rpc.ClientContext;
21+
import com.google.api.gax.rpc.StatusCode;
2122
import com.google.cloud.spanner.admin.database.v1.stub.DatabaseAdminStubSettings;
2223
import com.google.cloud.spanner.admin.database.v1.stub.GrpcDatabaseAdminStub;
2324
import java.io.IOException;
2425

2526
/**
2627
* Wrapper around {@link GrpcDatabaseAdminStub} to make the constructor available inside this
27-
* package.
28+
* package. This makes it possible to create a {@link GrpcDatabaseAdminStub} with a custom {@link
29+
* GrpcStubCallableFactory} and custom settings. This is used by integration tests to automatically
30+
* retry {@link StatusCode.Code#RESOURCE_EXHAUSTED} errors for certain administrative requests.
2831
*/
2932
class GrpcDatabaseAdminStubWithCustomCallableFactory extends GrpcDatabaseAdminStub {
3033
GrpcDatabaseAdminStubWithCustomCallableFactory(

0 commit comments

Comments
 (0)