Skip to content

cloud_to_prod gateway_v4 tests failing - StatusCode=Unavailable, Detail="502:Bad Gateway" #18004

@apolcyn

Description

@apolcyn

https://source.cloud.google.com/results/invocations/7db13668-29fa-4f10-89be-38fae73fe974/targets/grpc%2Fcore%2Fpull_request%2Flinux%2Fgrpc_interop_toprod/log

snippet from the log:

2019-02-10 16:36:35,973 FAILED: cloud_to_prod:java:gateway_v4:large_unary:google_default_credentials [ret=1, pid=12582, time=22.2sec]
2019-02-10 16:36:35,973 START: cloud_to_prod:python:gateway_v4:empty_stream:tls
2019-02-10 16:36:35,978 WAITING: ETA 119.5 sec; 180 queued, 12 jobs running, 226 complete, 51 failed (load 12.00) next: cloud_to_prod:python:gateway_v4:client_streaming:tls @ 1.00 cpu
2019-02-10 16:36:36,327 Running test empty_unary
Exception in thread "main" io.grpc.StatusRuntimeException: UNAVAILABLE: 502:Bad Gateway
	at io.grpc.stub.ClientCalls.toStatusRuntimeException(ClientCalls.java:233)
	at io.grpc.stub.ClientCalls.getUnchecked(ClientCalls.java:214)
	at io.grpc.stub.ClientCalls.blockingUnaryCall(ClientCalls.java:139)
	at io.grpc.testing.integration.TestServiceGrpc$TestServiceBlockingStub.emptyCall(TestServiceGrpc.java:622)
	at io.grpc.testing.integration.AbstractInteropTest.emptyUnary(AbstractInteropTest.java:346)
	at io.grpc.testing.integration.TestServiceClient.runTest(TestServiceClient.java:224)
	at io.grpc.testing.integration.TestServiceClient.run(TestServiceClient.java:212)
	at io.grpc.testing.integration.TestServiceClient.main(TestServiceClient.java:70)
Shutting down
2019-02-10 16:36:36,327 FAILED: cloud_to_prod:java:gateway_v4:empty_unary:tls [ret=1, pid=12833, time=22.1sec]
2019-02-10 16:36:36,327 START: cloud_to_prod:python:gateway_v4:client_streaming:tls
2019-02-10 16:36:36,332 WAITING: ETA 118.6 sec; 179 queued, 12 jobs running, 227 complete, 52 failed (load 12.00) next: cloud_to_prod:python:gateway_v4:server_streaming:tls @ 1.00 cpu
2019-02-10 16:36:36,945 Running test empty_unary
Exception in thread "main" io.grpc.StatusRuntimeException: UNAVAILABLE: 502:Bad Gateway
	at io.grpc.stub.ClientCalls.toStatusRuntimeException(ClientCalls.java:233)
	at io.grpc.stub.ClientCalls.getUnchecked(ClientCalls.java:214)
	at io.grpc.stub.ClientCalls.blockingUnaryCall(ClientCalls.java:139)
	at io.grpc.testing.integration.TestServiceGrpc$TestServiceBlockingStub.emptyCall(TestServiceGrpc.java:622)
	at io.grpc.testing.integration.AbstractInteropTest.emptyUnary(AbstractInteropTest.java:346)
	at io.grpc.testing.integration.TestServiceClient.runTest(TestServiceClient.java:224)
	at io.grpc.testing.integration.TestServiceClient.run(TestServiceClient.java:212)
	at io.grpc.testing.integration.TestServiceClient.main(TestServiceClient.java:70)
Shutting down
2019-02-10 16:36:36,945 FAILED: cloud_to_prod:java:gateway_v4:empty_unary:google_default_credentials [ret=1, pid=12969, time=22.3sec]
2019-02-10 16:36:36,946 START: cloud_to_prod:python:gateway_v4:server_streaming:tls
2019-02-10 16:36:36,950 WAITING: ETA 118.0 sec; 178 queued, 12 jobs running, 228 complete, 53 failed (load 12.00) next: cloud_to_prod:python:gateway_v4:cancel_after_begin:tls @ 1.00 cpu
2019-02-10 16:36:38,716 Running test client_streaming
Exception in thread "main" java.lang.RuntimeException: java.util.concurrent.ExecutionException: io.grpc.StatusRuntimeException: UNAVAILABLE: 502:Bad Gateway
	at io.grpc.testing.integration.TestServiceClient.run(TestServiceClient.java:216)
	at io.grpc.testing.integration.TestServiceClient.main(TestServiceClient.java:70)
Caused by: java.util.concurrent.ExecutionException: io.grpc.StatusRuntimeException: UNAVAILABLE: 502:Bad Gateway
	at com.google.common.util.concurrent.AbstractFuture.getDoneValue(AbstractFuture.java:531)
	at com.google.common.util.concurrent.AbstractFuture.get(AbstractFuture.java:512)
	at com.google.common.util.concurrent.AbstractFuture$TrustedFuture.get(AbstractFuture.java:83)
	at io.grpc.testing.integration.AbstractInteropTest.clientStreaming(AbstractInteropTest.java:566)
	at io.grpc.testing.integration.TestServiceClient.runTest(TestServiceClient.java:249)
	at io.grpc.testing.integration.TestServiceClient.run(TestServiceClient.java:212)
	... 1 more
Caused by: io.grpc.StatusRuntimeException: UNAVAILABLE: 502:Bad Gateway
	at io.grpc.Status.asRuntimeException(Status.java:532)
	at io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onClose(ClientCalls.java:434)
	at io.grpc.PartialForwardingClientCallListener.onClose(PartialForwardingClientCallListener.java:39)
	at io.grpc.ForwardingClientCallListener.onClose(ForwardingClientCallListener.java:23)
	at io.grpc.ForwardingClientCallListener$SimpleForwardingClientCallListener.onClose(ForwardingClientCallListener.java:40)
	at io.grpc.internal.CensusStatsModule$StatsClientInterceptor$1$1.onClose(CensusStatsModule.java:699)
	at io.grpc.PartialForwardingClientCallListener.onClose(PartialForwardingClientCallListener.java:39)
	at io.grpc.ForwardingClientCallListener.onClose(ForwardingClientCallListener.java:23)
	at io.grpc.ForwardingClientCallListener$SimpleForwardingClientCallListener.onClose(ForwardingClientCallListener.java:40)
	at io.grpc.internal.CensusTracingModule$TracingClientInterceptor$1$1.onClose(CensusTracingModule.java:397)
	at io.grpc.internal.ClientCallImpl.closeObserver(ClientCallImpl.java:459)
	at io.grpc.internal.ClientCallImpl.access$300(ClientCallImpl.java:63)
	at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl.close(ClientCallImpl.java:546)
	at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl.access$600(ClientCallImpl.java:467)
	at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInContext(ClientCallImpl.java:584)
	at io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37)
	at io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)
Shutting down
2019-02-10 16:36:38,716 FAILED: cloud_to_prod:java:gateway_v4:client_streaming:tls [ret=1, pid=13696, time=22.1sec]
2019-02-10 16:36:38,716 START: cloud_to_prod:python:gateway_v4:cancel_after_begin:tls
2019-02-10 16:36:38,720 WAITING: ETA 118.3 sec; 177 queued, 12 jobs running, 229 complete, 54 failed (load 12.00) next: cloud_to_prod:python:gateway_v4:cancel_after_first_response:tls @ 1.00 cpu
2019-02-10 16:36:39,158 Running test client_streaming
Exception in thread "main" java.lang.RuntimeException: java.util.concurrent.ExecutionException: io.grpc.StatusRuntimeException: UNAVAILABLE: 502:Bad Gateway
	at io.grpc.testing.integration.TestServiceClient.run(TestServiceClient.java:216)
	at io.grpc.testing.integration.TestServiceClient.main(TestServiceClient.java:70)
Caused by: java.util.concurrent.ExecutionException: io.grpc.StatusRuntimeException: UNAVAILABLE: 502:Bad Gateway
	at com.google.common.util.concurrent.AbstractFuture.getDoneValue(AbstractFuture.java:531)
	at com.google.common.util.concurrent.AbstractFuture.get(AbstractFuture.java:512)
	at com.google.common.util.concurrent.AbstractFuture$TrustedFuture.get(AbstractFuture.java:83)
	at io.grpc.testing.integration.AbstractInteropTest.clientStreaming(AbstractInteropTest.java:566)
	at io.grpc.testing.integration.TestServiceClient.runTest(TestServiceClient.java:249)
	at io.grpc.testing.integration.TestServiceClient.run(TestServiceClient.java:212)
	... 1 more
Caused by: io.grpc.StatusRuntimeException: UNAVAILABLE: 502:Bad Gateway
	at io.grpc.Status.asRuntimeException(Status.java:532)
	at io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onClose(ClientCalls.java:434)
	at io.grpc.PartialForwardingClientCallListener.onClose(PartialForwardingClientCallListener.java:39)
	at io.grpc.ForwardingClientCallListener.onClose(ForwardingClientCallListener.java:23)
	at io.grpc.ForwardingClientCallListener$SimpleForwardingClientCallListener.onClose(ForwardingClientCallListener.java:40)
	at io.grpc.internal.CensusStatsModule$StatsClientInterceptor$1$1.onClose(CensusStatsModule.java:699)
	at io.grpc.PartialForwardingClientCallListener.onClose(PartialForwardingClientCallListener.java:39)
	at io.grpc.ForwardingClientCallListener.onClose(ForwardingClientCallListener.java:23)
	at io.grpc.ForwardingClientCallListener$SimpleForwardingClientCallListener.onClose(ForwardingClientCallListener.java:40)
	at io.grpc.internal.CensusTracingModule$TracingClientInterceptor$1$1.onClose(CensusTracingModule.java:397)
	at io.grpc.internal.ClientCallImpl.closeObserver(ClientCallImpl.java:459)
	at io.grpc.internal.ClientCallImpl.access$300(ClientCallImpl.java:63)
	at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl.close(ClientCallImpl.java:546)
	at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl.access$600(ClientCallImpl.java:467)
	at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInContext(ClientCallImpl.java:584)
	at io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37)
	at io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)
Shutting down
2019-02-10 16:36:39,158 FAILED: cloud_to_prod:java:gateway_v4:client_streaming:google_default_credentials [ret=1, pid=13856, time=22.2sec]
2019-02-10 16:36:39,158 START: cloud_to_prod:python:gateway_v4:cancel_after_first_response:tls
2019-02-10 16:36:39,163 WAITING: ETA 117.6 sec; 176 queued, 12 jobs running, 230 complete, 55 failed (load 12.00) next: cloud_to_prod:python:gateway_v4:timeout_on_sleeping_server:tls @ 1.00 cpu
2019-02-10 16:36:39,910 PASSED: cloud_to_prod:python:gateway_v4:cancel_after_begin:tls [time=1.2sec, retries=0:0]
2019-02-10 16:36:39,911 START: cloud_to_prod:python:gateway_v4:timeout_on_sleeping_server:tls
2019-02-10 16:36:39,915 WAITING: ETA 117.0 sec; 175 queued, 12 jobs running, 231 complete, 55 failed (load 12.00) next: cloud_to_prod:javaokhttp:gateway_v4:large_unary:tls @ 1.00 cpu
2019-02-10 16:36:41,072 PASSED: cloud_to_prod:python:gateway_v4:timeout_on_sleeping_server:tls [time=1.2sec, retries=0:0]
2019-02-10 16:36:41,072 START: cloud_to_prod:javaokhttp:gateway_v4:large_unary:tls
2019-02-10 16:36:41,077 WAITING: ETA 116.8 sec; 174 queued, 12 jobs running, 232 complete, 55 failed (load 12.00) next: cloud_to_prod:javaokhttp:gateway_v4:large_unary:google_default_credentials @ 1.00 cpu
2019-02-10 16:36:43,739 Running test server_streaming
Exception in thread "main" java.lang.AssertionError: io.grpc.StatusRuntimeException: UNAVAILABLE: 502:Bad Gateway
	at io.grpc.testing.integration.AbstractInteropTest.assertSuccess(AbstractInteropTest.java:1738)
	at io.grpc.testing.integration.AbstractInteropTest.serverStreaming(AbstractInteropTest.java:531)
	at io.grpc.testing.integration.TestServiceClient.runTest(TestServiceClient.java:261)
	at io.grpc.testing.integration.TestServiceClient.run(TestServiceClient.java:212)
	at io.grpc.testing.integration.TestServiceClient.main(TestServiceClient.java:70)
Caused by: io.grpc.StatusRuntimeException: UNAVAILABLE: 502:Bad Gateway
	at io.grpc.Status.asRuntimeException(Status.java:532)
	at io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onClose(ClientCalls.java:434)
	at io.grpc.PartialForwardingClientCallListener.onClose(PartialForwardingClientCallListener.java:39)
	at io.grpc.ForwardingClientCallListener.onClose(ForwardingClientCallListener.java:23)
	at io.grpc.ForwardingClientCallListener$SimpleForwardingClientCallListener.onClose(ForwardingClientCallListener.java:40)
	at io.grpc.internal.CensusStatsModule$StatsClientInterceptor$1$1.onClose(CensusStatsModule.java:699)
	at io.grpc.PartialForwardingClientCallListener.onClose(PartialForwardingClientCallListener.java:39)
	at io.grpc.ForwardingClientCallListener.onClose(ForwardingClientCallListener.java:23)
	at io.grpc.ForwardingClientCallListener$SimpleForwardingClientCallListener.onClose(ForwardingClientCallListener.java:40)
	at io.grpc.internal.CensusTracingModule$TracingClientInterceptor$1$1.onClose(CensusTracingModule.java:397)
	at io.grpc.internal.ClientCallImpl.closeObserver(ClientCallImpl.java:459)
	at io.grpc.internal.ClientCallImpl.access$300(ClientCallImpl.java:63)
	at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl.close(ClientCallImpl.java:546)
	at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl.access$600(ClientCallImpl.java:467)
	at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInContext(ClientCallImpl.java:584)
	at io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37)
	at io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)
Shutting down
2019-02-10 16:36:43,739 FAILED: cloud_to_prod:java:gateway_v4:server_streaming:tls [ret=1, pid=14158, time=21.9sec]
2019-02-10 16:36:43,739 START: cloud_to_prod:javaokhttp:gateway_v4:large_unary:google_default_credentials
2019-02-10 16:36:43,744 WAITING: ETA 117.8 sec; 173 queued, 12 jobs running, 233 complete, 56 failed (load 12.00) next: cloud_to_prod:javaokhttp:gateway_v4:empty_unary:tls @ 1.00 cpu
2019-02-10 16:36:44,690 Running test server_streaming
Exception in thread "main" java.lang.AssertionError: io.grpc.StatusRuntimeException: UNAVAILABLE: 502:Bad Gateway
	at io.grpc.testing.integration.AbstractInteropTest.assertSuccess(AbstractInteropTest.java:1738)
	at io.grpc.testing.integration.AbstractInteropTest.serverStreaming(AbstractInteropTest.java:531)
	at io.grpc.testing.integration.TestServiceClient.runTest(TestServiceClient.java:261)
	at io.grpc.testing.integration.TestServiceClient.run(TestServiceClient.java:212)
	at io.grpc.testing.integration.TestServiceClient.main(TestServiceClient.java:70)
Caused by: io.grpc.StatusRuntimeException: UNAVAILABLE: 502:Bad Gateway
	at io.grpc.Status.asRuntimeException(Status.java:532)
	at io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onClose(ClientCalls.java:434)
	at io.grpc.PartialForwardingClientCallListener.onClose(PartialForwardingClientCallListener.java:39)
	at io.grpc.ForwardingClientCallListener.onClose(ForwardingClientCallListener.java:23)
	at io.grpc.ForwardingClientCallListener$SimpleForwardingClientCallListener.onClose(ForwardingClientCallListener.java:40)
	at io.grpc.internal.CensusStatsModule$StatsClientInterceptor$1$1.onClose(CensusStatsModule.java:699)
	at io.grpc.PartialForwardingClientCallListener.onClose(PartialForwardingClientCallListener.java:39)
	at io.grpc.ForwardingClientCallListener.onClose(ForwardingClientCallListener.java:23)
	at io.grpc.ForwardingClientCallListener$SimpleForwardingClientCallListener.onClose(ForwardingClientCallListener.java:40)
	at io.grpc.internal.CensusTracingModule$TracingClientInterceptor$1$1.onClose(CensusTracingModule.java:397)
	at io.grpc.internal.ClientCallImpl.closeObserver(ClientCallImpl.java:459)
	at io.grpc.internal.ClientCallImpl.access$300(ClientCallImpl.java:63)
	at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl.close(ClientCallImpl.java:546)
	at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl.access$600(ClientCallImpl.java:467)
	at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInContext(ClientCallImpl.java:584)
	at io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37)
	at io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)
Shutting down

Seems to be affecting all clients

@billfeng327 do you know if this can be due any recent changes on the server side?

Metadata

Metadata

Assignees

Type

No type

Projects

No projects

Milestone

No milestone

Relationships

None yet

Development

No branches or pull requests

Issue actions