Skip to content

Commit bd833e1

Browse files
committed
fix(dubbo): count the committed provider in gray inflight accounting (#8132)
1 parent cdcc026 commit bd833e1

3 files changed

Lines changed: 176 additions & 15 deletions

File tree

‎shenyu-plugin/shenyu-plugin-proxy/shenyu-plugin-rpc/shenyu-plugin-dubbo/shenyu-plugin-apache-dubbo/src/main/java/org/apache/shenyu/plugin/apache/dubbo/proxy/DubboGrayInflightFilter.java‎

Lines changed: 37 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
import org.apache.commons.collections4.CollectionUtils;
2121
import org.apache.commons.lang3.StringUtils;
22+
import org.apache.dubbo.common.URL;
2223
import org.apache.dubbo.common.constants.CommonConstants;
2324
import org.apache.dubbo.common.extension.Activate;
2425
import org.apache.dubbo.rpc.Filter;
@@ -35,15 +36,18 @@
3536
import java.util.concurrent.atomic.AtomicBoolean;
3637

3738
/**
38-
* Keeps the selected upstream's outstanding request count current for the dubbo gray
39+
* Keeps the accounting upstream's outstanding request count current for the dubbo gray
3940
* least-active path, without wrapping the invoker Dubbo receives.
4041
*
41-
* <p>The gray load balance marks the upstream it selected for the current attempt
42-
* ({@code DUBBO_INFLIGHT_UPSTREAM_URL}); this consumer-side filter runs after the cluster
43-
* has committed to a provider, so counting here sees every actual attempt, including ones
44-
* produced by a retry reselection. Wrapping the invoker instead would put a foreign object
45-
* into Dubbo's already-tried list, where raw-directory candidates can never compare equal
46-
* to it and a failed provider would be retried even when another one is available (#8132).
42+
* <p>The gray load balance marks the upstream it proposed for the current attempt
43+
* ({@code DUBBO_INFLIGHT_UPSTREAM_URL}). The marker only says that this attempt came from
44+
* the gray least-active path: it is not the accounting target, because the cluster may
45+
* reselect a different provider after the load balance returned without calling it again.
46+
* The filter therefore resolves the invoker the cluster actually committed to (address,
47+
* group and version) against the selector's upstreams and counts that one. Wrapping the
48+
* invoker instead would put a foreign object into Dubbo's already-tried list, where
49+
* raw-directory candidates can never compare equal to it and a failed provider would be
50+
* retried even when another one is available (#8132).
4751
*
4852
* <p>The count is increased when the RPC starts, not when the invoker is selected, and it is
4953
* released exactly once per attempt, either when the returned result completes or when the
@@ -55,13 +59,14 @@ public class DubboGrayInflightFilter implements Filter {
5559

5660
@Override
5761
public Result invoke(final Invoker<?> invoker, final Invocation invocation) throws RpcException {
58-
Object upstreamUrl = invocation.getObjectAttachment(Constants.DUBBO_INFLIGHT_UPSTREAM_URL);
59-
// the marker is local bookkeeping and must never travel to the provider
60-
invocation.getObjectAttachments().remove(Constants.DUBBO_INFLIGHT_UPSTREAM_URL);
61-
if (Objects.isNull(upstreamUrl)) {
62+
// the marker is local bookkeeping and must never travel to the provider; it only
63+
// says this attempt came from the gray least-active path
64+
boolean grayAttempt = Objects.nonNull(
65+
invocation.getObjectAttachments().remove(Constants.DUBBO_INFLIGHT_UPSTREAM_URL));
66+
if (!grayAttempt) {
6267
return invoker.invoke(invocation);
6368
}
64-
Upstream upstream = matchUpstream(invocation, String.valueOf(upstreamUrl));
69+
Upstream upstream = matchUpstream(invocation, invoker.getUrl());
6570
if (Objects.isNull(upstream)) {
6671
return invoker.invoke(invocation);
6772
}
@@ -82,21 +87,38 @@ public Result invoke(final Invoker<?> invoker, final Invocation invocation) thro
8287
return result.whenCompleteWithContext((response, throwable) -> release(released, upstream));
8388
}
8489

85-
private Upstream matchUpstream(final Invocation invocation, final String upstreamUrl) {
90+
private Upstream matchUpstream(final Invocation invocation, final URL providerUrl) {
8691
String selectorId = invocation.getAttachment(Constants.DUBBO_SELECTOR_ID);
87-
if (StringUtils.isBlank(selectorId)) {
92+
if (StringUtils.isBlank(selectorId) || Objects.isNull(providerUrl) || StringUtils.isBlank(providerUrl.getAddress())) {
8893
return null;
8994
}
9095
List<Upstream> upstreams = UpstreamCacheManager.getInstance().findUpstreamListBySelectorId(selectorId);
9196
if (CollectionUtils.isEmpty(upstreams)) {
9297
return null;
9398
}
99+
final String address = providerUrl.getAddress();
100+
final String group = StringUtils.trimToNull(providerUrl.getParameter(Constants.GROUP));
101+
final String version = StringUtils.trimToNull(providerUrl.getParameter(Constants.VERSION));
94102
return upstreams.stream()
95-
.filter(upstream -> upstreamUrl.equals(upstream.getUrl()))
103+
.filter(upstream -> address.equals(upstream.getUrl()))
104+
.filter(upstream -> matchesConfigured(upstream.getGroup(), group))
105+
.filter(upstream -> matchesConfigured(upstream.getVersion(), version))
96106
.findFirst()
97107
.orElse(null);
98108
}
99109

110+
/**
111+
* Whether the committed provider satisfies a configured group or version. A blank
112+
* configuration is a wildcard, which is how the gray load balance filters invokers.
113+
*
114+
* @param configured the configured group or version
115+
* @param actual the committed provider's group or version
116+
* @return true when the field does not restrict the provider
117+
*/
118+
private static boolean matchesConfigured(final String configured, final String actual) {
119+
return StringUtils.isBlank(configured) || StringUtils.trim(configured).equals(actual);
120+
}
121+
100122
private void release(final AtomicBoolean released, final Upstream upstream) {
101123
if (released.compareAndSet(false, true)) {
102124
upstream.getInflight().decrementAndGet();

‎shenyu-plugin/shenyu-plugin-proxy/shenyu-plugin-rpc/shenyu-plugin-dubbo/shenyu-plugin-apache-dubbo/src/test/java/org/apache/shenyu/plugin/apache/dubbo/proxy/DubboGrayInflightFilterTest.java‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import org.mockito.junit.jupiter.MockitoSettings;
3636
import org.mockito.quality.Strictness;
3737

38+
import java.util.Arrays;
3839
import java.util.Collections;
3940
import java.util.concurrent.CompletableFuture;
4041

@@ -124,6 +125,90 @@ public void releasesEveryInvocationOnce() {
124125
assertEquals(initial, upstream.getInflight().get());
125126
}
126127

128+
@Test
129+
public void countsTheCommittedProviderNotTheMarkedOne() {
130+
Upstream markedProposal = newUpstream("a:1");
131+
Upstream committed = newUpstream("b:1");
132+
cacheUpstreams(markedProposal, committed);
133+
invocation.setObjectAttachment(Constants.DUBBO_INFLIGHT_UPSTREAM_URL, "a:1");
134+
when(nextInvoker.getUrl()).thenReturn(URL.valueOf("dubbo://b:1"));
135+
CompletableFuture<AppResponse> responseFuture = new CompletableFuture<>();
136+
when(nextInvoker.invoke(invocation)).thenReturn(new AsyncRpcResult(responseFuture, invocation));
137+
138+
long markedInitial = markedProposal.getInflight().get();
139+
long committedInitial = committed.getInflight().get();
140+
filter.invoke(nextInvoker, invocation);
141+
142+
assertEquals(markedInitial, markedProposal.getInflight().get(),
143+
"the marked proposal is not necessarily the committed provider");
144+
assertEquals(committedInitial + 1, committed.getInflight().get(),
145+
"the committed provider carries the attempt");
146+
assertFalse(invocation.getObjectAttachments().containsKey(Constants.DUBBO_INFLIGHT_UPSTREAM_URL));
147+
148+
responseFuture.complete(new AppResponse("ok"));
149+
assertEquals(committedInitial, committed.getInflight().get());
150+
}
151+
152+
@Test
153+
public void countsTheUpstreamMatchingTheCommittedGroupAndVersion() {
154+
Upstream versionOne = newUpstream("shared:1", "g", "1");
155+
Upstream versionTwo = newUpstream("shared:1", "g", "2");
156+
cacheUpstreams(versionOne, versionTwo);
157+
invocation.setObjectAttachment(Constants.DUBBO_INFLIGHT_UPSTREAM_URL, "shared:1");
158+
when(nextInvoker.getUrl()).thenReturn(URL.valueOf("dubbo://shared:1?group=g&version=2"));
159+
CompletableFuture<AppResponse> responseFuture = new CompletableFuture<>();
160+
when(nextInvoker.invoke(invocation)).thenReturn(new AsyncRpcResult(responseFuture, invocation));
161+
162+
long versionOneInitial = versionOne.getInflight().get();
163+
long versionTwoInitial = versionTwo.getInflight().get();
164+
filter.invoke(nextInvoker, invocation);
165+
166+
assertEquals(versionOneInitial, versionOne.getInflight().get(),
167+
"the same-address upstream of another version must not be counted");
168+
assertEquals(versionTwoInitial + 1, versionTwo.getInflight().get(),
169+
"the committed group/version identifies the accounting upstream");
170+
171+
responseFuture.complete(new AppResponse("ok"));
172+
assertEquals(versionTwoInitial, versionTwo.getInflight().get());
173+
}
174+
175+
@Test
176+
public void doesNotCountWhenTheCommittedProviderIsUnknown() {
177+
Upstream markedProposal = newUpstream("a:1");
178+
cacheUpstreams(markedProposal);
179+
invocation.setObjectAttachment(Constants.DUBBO_INFLIGHT_UPSTREAM_URL, "a:1");
180+
when(nextInvoker.getUrl()).thenReturn(URL.valueOf("dubbo://c:1"));
181+
CompletableFuture<AppResponse> responseFuture = new CompletableFuture<>();
182+
when(nextInvoker.invoke(invocation)).thenReturn(new AsyncRpcResult(responseFuture, invocation));
183+
184+
long markedInitial = markedProposal.getInflight().get();
185+
filter.invoke(nextInvoker, invocation);
186+
187+
assertEquals(markedInitial, markedProposal.getInflight().get(),
188+
"an unknown committed provider must not fall back to the marked proposal");
189+
assertFalse(invocation.getObjectAttachments().containsKey(Constants.DUBBO_INFLIGHT_UPSTREAM_URL));
190+
responseFuture.complete(new AppResponse("ok"));
191+
}
192+
193+
@Test
194+
public void countsWhenTheConfiguredGroupIsAbsent() {
195+
Upstream addressOnly = newUpstream("a:1");
196+
cacheUpstreams(addressOnly);
197+
invocation.setObjectAttachment(Constants.DUBBO_INFLIGHT_UPSTREAM_URL, "a:1");
198+
when(nextInvoker.getUrl()).thenReturn(URL.valueOf("dubbo://a:1?group=g&version=2"));
199+
CompletableFuture<AppResponse> responseFuture = new CompletableFuture<>();
200+
when(nextInvoker.invoke(invocation)).thenReturn(new AsyncRpcResult(responseFuture, invocation));
201+
202+
long initial = addressOnly.getInflight().get();
203+
filter.invoke(nextInvoker, invocation);
204+
205+
assertEquals(initial + 1, addressOnly.getInflight().get(),
206+
"a blank configured group/version is a wildcard, matching the load balance");
207+
208+
responseFuture.complete(new AppResponse("ok"));
209+
assertEquals(initial, addressOnly.getInflight().get());
210+
}
211+
127212
@Test
128213
public void passesThroughWithoutTheMarker() {
129214
when(nextInvoker.invoke(invocation)).thenReturn(AsyncRpcResult.newDefaultAsyncResult("ok", invocation));
@@ -145,6 +230,20 @@ public void releasesEvenWhenTheUpstreamMarkerCannotBeResolved() {
145230
filter.invoke(nextInvoker, invocation);
146231
}
147232

233+
private Upstream newUpstream(final String url) {
234+
return newUpstream(url, null, null);
235+
}
236+
237+
private Upstream newUpstream(final String url, final String group, final String version) {
238+
Upstream result = Upstream.builder().url(url).group(group).version(version).build();
239+
result.getInflight().set(1);
240+
return result;
241+
}
242+
243+
private void cacheUpstreams(final Upstream... upstreams) {
244+
UpstreamCacheManager.getInstance().submit(SELECTOR_ID, Arrays.asList(upstreams));
245+
}
246+
148247
private void markUpstream(final RpcInvocation rpcInvocation) {
149248
rpcInvocation.setObjectAttachment(Constants.DUBBO_INFLIGHT_UPSTREAM_URL, "localhost");
150249
}

‎shenyu-plugin/shenyu-plugin-proxy/shenyu-plugin-rpc/shenyu-plugin-dubbo/shenyu-plugin-apache-dubbo/src/test/java/org/apache/shenyu/plugin/apache/dubbo/proxy/GrayLoadBalanceClusterIdentityTest.java‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@
1818
package org.apache.shenyu.plugin.apache.dubbo.proxy;
1919

2020
import org.apache.dubbo.common.URL;
21+
import org.apache.dubbo.rpc.AppResponse;
22+
import org.apache.dubbo.rpc.AsyncRpcResult;
2123
import org.apache.dubbo.rpc.Invocation;
2224
import org.apache.dubbo.rpc.Invoker;
2325
import org.apache.dubbo.rpc.RpcException;
@@ -28,6 +30,7 @@
2830
import org.apache.shenyu.common.constant.Constants;
2931
import org.apache.shenyu.common.dto.convert.rule.impl.DubboRuleHandle;
3032
import org.apache.shenyu.common.dto.convert.selector.DubboUpstream;
33+
import org.apache.shenyu.loadbalancer.cache.UpstreamCacheManager;
3134
import org.apache.shenyu.loadbalancer.entity.Upstream;
3235
import org.apache.shenyu.loadbalancer.factory.LoadBalancerFactory;
3336
import org.apache.shenyu.plugin.apache.dubbo.handler.ApacheDubboPluginDataHandler;
@@ -44,6 +47,7 @@
4447
import java.util.ArrayList;
4548
import java.util.List;
4649
import java.util.Arrays;
50+
import java.util.concurrent.CompletableFuture;
4751

4852
import static org.junit.jupiter.api.Assertions.assertEquals;
4953
import static org.mockito.ArgumentMatchers.any;
@@ -71,6 +75,9 @@ public class GrayLoadBalanceClusterIdentityTest {
7175
@Mock
7276
private Directory<Object> directory;
7377

78+
@Mock
79+
private Invoker<Object> committedProvider;
80+
7481
private MockedStatic<LoadBalancerFactory> loadBalancerFactoryMockedStatic;
7582

7683
private ApacheDubboGrayLoadBalance grayLoadBalance;
@@ -101,6 +108,7 @@ public void setup() {
101108
upstreamB.getInflight().set(1);
102109
providerA = new MockInvoker<>(URL.valueOf("dubbo://a:1"), Object.class);
103110
providerB = new MockInvoker<>(URL.valueOf("dubbo://b:1"), Object.class);
111+
UpstreamCacheManager.getInstance().submit(Constants.DUBBO_SELECTOR_ID, Arrays.asList(upstreamA, upstreamB));
104112

105113
when(directory.getConsumerUrl()).thenReturn(URL.valueOf("dubbo://consumer:1?loadbalance=shenyuDeterministic"));
106114
loadBalancerFactoryMockedStatic = mockStatic(LoadBalancerFactory.class);
@@ -119,6 +127,7 @@ public void tearDown() {
119127
loadBalancerFactoryMockedStatic.close();
120128
ApacheDubboPluginDataHandler.RULE_CACHED_HANDLE.get().getAllCache().clear();
121129
ApacheDubboPluginDataHandler.SELECTOR_CACHED_HANDLE.get().getAllCache().clear();
130+
UpstreamCacheManager.getInstance().removeByKey(Constants.DUBBO_SELECTOR_ID);
122131
}
123132

124133
@Test
@@ -138,6 +147,37 @@ public void reselectMustExcludeTheProviderALreadyTriedThroughTheGrayPath() throw
138147
"provider A is already tried, so the reselect must choose provider B");
139148
}
140149

150+
@Test
151+
public void reselectCountsTheCommittedProviderNotTheMarkedOne() throws Exception {
152+
TestClusterInvoker clusterInvoker = new TestClusterInvoker(directory);
153+
ArrayList<Invoker<Object>> alreadyTried = new ArrayList<>();
154+
Invoker<Object> firstPick = clusterInvoker.pick(grayLoadBalance, invocation,
155+
Arrays.asList(providerA, providerB), alreadyTried);
156+
alreadyTried.add(firstPick);
157+
Invoker<Object> committed = clusterInvoker.pick(grayLoadBalance, invocation,
158+
Arrays.asList(providerA, providerB), alreadyTried);
159+
assertEquals("b:1", committed.getUrl().getAddress(),
160+
"provider A is already tried, so the reselect must choose provider B");
161+
assertEquals("a:1", invocation.getObjectAttachment(Constants.DUBBO_INFLIGHT_UPSTREAM_URL),
162+
"reselect does not call the load balance again, so the marker still names the proposal");
163+
164+
when(committedProvider.getUrl()).thenReturn(committed.getUrl());
165+
CompletableFuture<AppResponse> responseFuture = new CompletableFuture<>();
166+
when(committedProvider.invoke(invocation)).thenReturn(new AsyncRpcResult(responseFuture, invocation));
167+
168+
long proposalInitial = upstreamA.getInflight().get();
169+
long committedInitial = upstreamB.getInflight().get();
170+
new DubboGrayInflightFilter().invoke(committedProvider, invocation);
171+
172+
assertEquals(proposalInitial, upstreamA.getInflight().get(),
173+
"the proposal is not carrying the request");
174+
assertEquals(committedInitial + 1, upstreamB.getInflight().get(),
175+
"the reselected provider carries the attempt");
176+
177+
responseFuture.complete(new AppResponse("ok"));
178+
assertEquals(committedInitial, upstreamB.getInflight().get());
179+
}
180+
141181
/**
142182
* Exposes the protected cluster selection, which is where Dubbo performs its
143183
* already-tried bookkeeping and reselection.

0 commit comments

Comments
 (0)