-
Notifications
You must be signed in to change notification settings - Fork 11.1k
cloud_to_prod gateway_v4 tests failing - StatusCode=Unavailable, Detail="502:Bad Gateway" #18004
Copy link
Copy link
Closed
Description
snippet from the log:
2019-02-10 16:36:35,973 FAILED: cloud_to_prod:java:gateway_v4:large_unary:google_default_credentials [ret=1, pid=12582, time=22.2sec]
2019-02-10 16:36:35,973 START: cloud_to_prod:python:gateway_v4:empty_stream:tls
2019-02-10 16:36:35,978 WAITING: ETA 119.5 sec; 180 queued, 12 jobs running, 226 complete, 51 failed (load 12.00) next: cloud_to_prod:python:gateway_v4:client_streaming:tls @ 1.00 cpu
2019-02-10 16:36:36,327 Running test empty_unary
Exception in thread "main" io.grpc.StatusRuntimeException: UNAVAILABLE: 502:Bad Gateway
at io.grpc.stub.ClientCalls.toStatusRuntimeException(ClientCalls.java:233)
at io.grpc.stub.ClientCalls.getUnchecked(ClientCalls.java:214)
at io.grpc.stub.ClientCalls.blockingUnaryCall(ClientCalls.java:139)
at io.grpc.testing.integration.TestServiceGrpc$TestServiceBlockingStub.emptyCall(TestServiceGrpc.java:622)
at io.grpc.testing.integration.AbstractInteropTest.emptyUnary(AbstractInteropTest.java:346)
at io.grpc.testing.integration.TestServiceClient.runTest(TestServiceClient.java:224)
at io.grpc.testing.integration.TestServiceClient.run(TestServiceClient.java:212)
at io.grpc.testing.integration.TestServiceClient.main(TestServiceClient.java:70)
Shutting down
2019-02-10 16:36:36,327 FAILED: cloud_to_prod:java:gateway_v4:empty_unary:tls [ret=1, pid=12833, time=22.1sec]
2019-02-10 16:36:36,327 START: cloud_to_prod:python:gateway_v4:client_streaming:tls
2019-02-10 16:36:36,332 WAITING: ETA 118.6 sec; 179 queued, 12 jobs running, 227 complete, 52 failed (load 12.00) next: cloud_to_prod:python:gateway_v4:server_streaming:tls @ 1.00 cpu
2019-02-10 16:36:36,945 Running test empty_unary
Exception in thread "main" io.grpc.StatusRuntimeException: UNAVAILABLE: 502:Bad Gateway
at io.grpc.stub.ClientCalls.toStatusRuntimeException(ClientCalls.java:233)
at io.grpc.stub.ClientCalls.getUnchecked(ClientCalls.java:214)
at io.grpc.stub.ClientCalls.blockingUnaryCall(ClientCalls.java:139)
at io.grpc.testing.integration.TestServiceGrpc$TestServiceBlockingStub.emptyCall(TestServiceGrpc.java:622)
at io.grpc.testing.integration.AbstractInteropTest.emptyUnary(AbstractInteropTest.java:346)
at io.grpc.testing.integration.TestServiceClient.runTest(TestServiceClient.java:224)
at io.grpc.testing.integration.TestServiceClient.run(TestServiceClient.java:212)
at io.grpc.testing.integration.TestServiceClient.main(TestServiceClient.java:70)
Shutting down
2019-02-10 16:36:36,945 FAILED: cloud_to_prod:java:gateway_v4:empty_unary:google_default_credentials [ret=1, pid=12969, time=22.3sec]
2019-02-10 16:36:36,946 START: cloud_to_prod:python:gateway_v4:server_streaming:tls
2019-02-10 16:36:36,950 WAITING: ETA 118.0 sec; 178 queued, 12 jobs running, 228 complete, 53 failed (load 12.00) next: cloud_to_prod:python:gateway_v4:cancel_after_begin:tls @ 1.00 cpu
2019-02-10 16:36:38,716 Running test client_streaming
Exception in thread "main" java.lang.RuntimeException: java.util.concurrent.ExecutionException: io.grpc.StatusRuntimeException: UNAVAILABLE: 502:Bad Gateway
at io.grpc.testing.integration.TestServiceClient.run(TestServiceClient.java:216)
at io.grpc.testing.integration.TestServiceClient.main(TestServiceClient.java:70)
Caused by: java.util.concurrent.ExecutionException: io.grpc.StatusRuntimeException: UNAVAILABLE: 502:Bad Gateway
at com.google.common.util.concurrent.AbstractFuture.getDoneValue(AbstractFuture.java:531)
at com.google.common.util.concurrent.AbstractFuture.get(AbstractFuture.java:512)
at com.google.common.util.concurrent.AbstractFuture$TrustedFuture.get(AbstractFuture.java:83)
at io.grpc.testing.integration.AbstractInteropTest.clientStreaming(AbstractInteropTest.java:566)
at io.grpc.testing.integration.TestServiceClient.runTest(TestServiceClient.java:249)
at io.grpc.testing.integration.TestServiceClient.run(TestServiceClient.java:212)
... 1 more
Caused by: io.grpc.StatusRuntimeException: UNAVAILABLE: 502:Bad Gateway
at io.grpc.Status.asRuntimeException(Status.java:532)
at io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onClose(ClientCalls.java:434)
at io.grpc.PartialForwardingClientCallListener.onClose(PartialForwardingClientCallListener.java:39)
at io.grpc.ForwardingClientCallListener.onClose(ForwardingClientCallListener.java:23)
at io.grpc.ForwardingClientCallListener$SimpleForwardingClientCallListener.onClose(ForwardingClientCallListener.java:40)
at io.grpc.internal.CensusStatsModule$StatsClientInterceptor$1$1.onClose(CensusStatsModule.java:699)
at io.grpc.PartialForwardingClientCallListener.onClose(PartialForwardingClientCallListener.java:39)
at io.grpc.ForwardingClientCallListener.onClose(ForwardingClientCallListener.java:23)
at io.grpc.ForwardingClientCallListener$SimpleForwardingClientCallListener.onClose(ForwardingClientCallListener.java:40)
at io.grpc.internal.CensusTracingModule$TracingClientInterceptor$1$1.onClose(CensusTracingModule.java:397)
at io.grpc.internal.ClientCallImpl.closeObserver(ClientCallImpl.java:459)
at io.grpc.internal.ClientCallImpl.access$300(ClientCallImpl.java:63)
at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl.close(ClientCallImpl.java:546)
at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl.access$600(ClientCallImpl.java:467)
at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInContext(ClientCallImpl.java:584)
at io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37)
at io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Shutting down
2019-02-10 16:36:38,716 FAILED: cloud_to_prod:java:gateway_v4:client_streaming:tls [ret=1, pid=13696, time=22.1sec]
2019-02-10 16:36:38,716 START: cloud_to_prod:python:gateway_v4:cancel_after_begin:tls
2019-02-10 16:36:38,720 WAITING: ETA 118.3 sec; 177 queued, 12 jobs running, 229 complete, 54 failed (load 12.00) next: cloud_to_prod:python:gateway_v4:cancel_after_first_response:tls @ 1.00 cpu
2019-02-10 16:36:39,158 Running test client_streaming
Exception in thread "main" java.lang.RuntimeException: java.util.concurrent.ExecutionException: io.grpc.StatusRuntimeException: UNAVAILABLE: 502:Bad Gateway
at io.grpc.testing.integration.TestServiceClient.run(TestServiceClient.java:216)
at io.grpc.testing.integration.TestServiceClient.main(TestServiceClient.java:70)
Caused by: java.util.concurrent.ExecutionException: io.grpc.StatusRuntimeException: UNAVAILABLE: 502:Bad Gateway
at com.google.common.util.concurrent.AbstractFuture.getDoneValue(AbstractFuture.java:531)
at com.google.common.util.concurrent.AbstractFuture.get(AbstractFuture.java:512)
at com.google.common.util.concurrent.AbstractFuture$TrustedFuture.get(AbstractFuture.java:83)
at io.grpc.testing.integration.AbstractInteropTest.clientStreaming(AbstractInteropTest.java:566)
at io.grpc.testing.integration.TestServiceClient.runTest(TestServiceClient.java:249)
at io.grpc.testing.integration.TestServiceClient.run(TestServiceClient.java:212)
... 1 more
Caused by: io.grpc.StatusRuntimeException: UNAVAILABLE: 502:Bad Gateway
at io.grpc.Status.asRuntimeException(Status.java:532)
at io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onClose(ClientCalls.java:434)
at io.grpc.PartialForwardingClientCallListener.onClose(PartialForwardingClientCallListener.java:39)
at io.grpc.ForwardingClientCallListener.onClose(ForwardingClientCallListener.java:23)
at io.grpc.ForwardingClientCallListener$SimpleForwardingClientCallListener.onClose(ForwardingClientCallListener.java:40)
at io.grpc.internal.CensusStatsModule$StatsClientInterceptor$1$1.onClose(CensusStatsModule.java:699)
at io.grpc.PartialForwardingClientCallListener.onClose(PartialForwardingClientCallListener.java:39)
at io.grpc.ForwardingClientCallListener.onClose(ForwardingClientCallListener.java:23)
at io.grpc.ForwardingClientCallListener$SimpleForwardingClientCallListener.onClose(ForwardingClientCallListener.java:40)
at io.grpc.internal.CensusTracingModule$TracingClientInterceptor$1$1.onClose(CensusTracingModule.java:397)
at io.grpc.internal.ClientCallImpl.closeObserver(ClientCallImpl.java:459)
at io.grpc.internal.ClientCallImpl.access$300(ClientCallImpl.java:63)
at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl.close(ClientCallImpl.java:546)
at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl.access$600(ClientCallImpl.java:467)
at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInContext(ClientCallImpl.java:584)
at io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37)
at io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Shutting down
2019-02-10 16:36:39,158 FAILED: cloud_to_prod:java:gateway_v4:client_streaming:google_default_credentials [ret=1, pid=13856, time=22.2sec]
2019-02-10 16:36:39,158 START: cloud_to_prod:python:gateway_v4:cancel_after_first_response:tls
2019-02-10 16:36:39,163 WAITING: ETA 117.6 sec; 176 queued, 12 jobs running, 230 complete, 55 failed (load 12.00) next: cloud_to_prod:python:gateway_v4:timeout_on_sleeping_server:tls @ 1.00 cpu
2019-02-10 16:36:39,910 PASSED: cloud_to_prod:python:gateway_v4:cancel_after_begin:tls [time=1.2sec, retries=0:0]
2019-02-10 16:36:39,911 START: cloud_to_prod:python:gateway_v4:timeout_on_sleeping_server:tls
2019-02-10 16:36:39,915 WAITING: ETA 117.0 sec; 175 queued, 12 jobs running, 231 complete, 55 failed (load 12.00) next: cloud_to_prod:javaokhttp:gateway_v4:large_unary:tls @ 1.00 cpu
2019-02-10 16:36:41,072 PASSED: cloud_to_prod:python:gateway_v4:timeout_on_sleeping_server:tls [time=1.2sec, retries=0:0]
2019-02-10 16:36:41,072 START: cloud_to_prod:javaokhttp:gateway_v4:large_unary:tls
2019-02-10 16:36:41,077 WAITING: ETA 116.8 sec; 174 queued, 12 jobs running, 232 complete, 55 failed (load 12.00) next: cloud_to_prod:javaokhttp:gateway_v4:large_unary:google_default_credentials @ 1.00 cpu
2019-02-10 16:36:43,739 Running test server_streaming
Exception in thread "main" java.lang.AssertionError: io.grpc.StatusRuntimeException: UNAVAILABLE: 502:Bad Gateway
at io.grpc.testing.integration.AbstractInteropTest.assertSuccess(AbstractInteropTest.java:1738)
at io.grpc.testing.integration.AbstractInteropTest.serverStreaming(AbstractInteropTest.java:531)
at io.grpc.testing.integration.TestServiceClient.runTest(TestServiceClient.java:261)
at io.grpc.testing.integration.TestServiceClient.run(TestServiceClient.java:212)
at io.grpc.testing.integration.TestServiceClient.main(TestServiceClient.java:70)
Caused by: io.grpc.StatusRuntimeException: UNAVAILABLE: 502:Bad Gateway
at io.grpc.Status.asRuntimeException(Status.java:532)
at io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onClose(ClientCalls.java:434)
at io.grpc.PartialForwardingClientCallListener.onClose(PartialForwardingClientCallListener.java:39)
at io.grpc.ForwardingClientCallListener.onClose(ForwardingClientCallListener.java:23)
at io.grpc.ForwardingClientCallListener$SimpleForwardingClientCallListener.onClose(ForwardingClientCallListener.java:40)
at io.grpc.internal.CensusStatsModule$StatsClientInterceptor$1$1.onClose(CensusStatsModule.java:699)
at io.grpc.PartialForwardingClientCallListener.onClose(PartialForwardingClientCallListener.java:39)
at io.grpc.ForwardingClientCallListener.onClose(ForwardingClientCallListener.java:23)
at io.grpc.ForwardingClientCallListener$SimpleForwardingClientCallListener.onClose(ForwardingClientCallListener.java:40)
at io.grpc.internal.CensusTracingModule$TracingClientInterceptor$1$1.onClose(CensusTracingModule.java:397)
at io.grpc.internal.ClientCallImpl.closeObserver(ClientCallImpl.java:459)
at io.grpc.internal.ClientCallImpl.access$300(ClientCallImpl.java:63)
at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl.close(ClientCallImpl.java:546)
at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl.access$600(ClientCallImpl.java:467)
at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInContext(ClientCallImpl.java:584)
at io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37)
at io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Shutting down
2019-02-10 16:36:43,739 FAILED: cloud_to_prod:java:gateway_v4:server_streaming:tls [ret=1, pid=14158, time=21.9sec]
2019-02-10 16:36:43,739 START: cloud_to_prod:javaokhttp:gateway_v4:large_unary:google_default_credentials
2019-02-10 16:36:43,744 WAITING: ETA 117.8 sec; 173 queued, 12 jobs running, 233 complete, 56 failed (load 12.00) next: cloud_to_prod:javaokhttp:gateway_v4:empty_unary:tls @ 1.00 cpu
2019-02-10 16:36:44,690 Running test server_streaming
Exception in thread "main" java.lang.AssertionError: io.grpc.StatusRuntimeException: UNAVAILABLE: 502:Bad Gateway
at io.grpc.testing.integration.AbstractInteropTest.assertSuccess(AbstractInteropTest.java:1738)
at io.grpc.testing.integration.AbstractInteropTest.serverStreaming(AbstractInteropTest.java:531)
at io.grpc.testing.integration.TestServiceClient.runTest(TestServiceClient.java:261)
at io.grpc.testing.integration.TestServiceClient.run(TestServiceClient.java:212)
at io.grpc.testing.integration.TestServiceClient.main(TestServiceClient.java:70)
Caused by: io.grpc.StatusRuntimeException: UNAVAILABLE: 502:Bad Gateway
at io.grpc.Status.asRuntimeException(Status.java:532)
at io.grpc.stub.ClientCalls$StreamObserverToCallListenerAdapter.onClose(ClientCalls.java:434)
at io.grpc.PartialForwardingClientCallListener.onClose(PartialForwardingClientCallListener.java:39)
at io.grpc.ForwardingClientCallListener.onClose(ForwardingClientCallListener.java:23)
at io.grpc.ForwardingClientCallListener$SimpleForwardingClientCallListener.onClose(ForwardingClientCallListener.java:40)
at io.grpc.internal.CensusStatsModule$StatsClientInterceptor$1$1.onClose(CensusStatsModule.java:699)
at io.grpc.PartialForwardingClientCallListener.onClose(PartialForwardingClientCallListener.java:39)
at io.grpc.ForwardingClientCallListener.onClose(ForwardingClientCallListener.java:23)
at io.grpc.ForwardingClientCallListener$SimpleForwardingClientCallListener.onClose(ForwardingClientCallListener.java:40)
at io.grpc.internal.CensusTracingModule$TracingClientInterceptor$1$1.onClose(CensusTracingModule.java:397)
at io.grpc.internal.ClientCallImpl.closeObserver(ClientCallImpl.java:459)
at io.grpc.internal.ClientCallImpl.access$300(ClientCallImpl.java:63)
at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl.close(ClientCallImpl.java:546)
at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl.access$600(ClientCallImpl.java:467)
at io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInContext(ClientCallImpl.java:584)
at io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37)
at io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Shutting down
Seems to be affecting all clients
@billfeng327 do you know if this can be due any recent changes on the server side?
Reactions are currently unavailable