Skip to content

Commit 1518ad4

Browse files
committed
feat(admin,loadbalancer): add readiness admission and explicit resume for business instances (#8042)
1 parent aab453f commit 1518ad4

23 files changed

Lines changed: 511 additions & 19 deletions

File tree

‎shenyu-admin/src/main/java/org/apache/shenyu/admin/disruptor/subscriber/URIRegisterExecutorSubscriber.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,14 @@ private void dispatchByNamespace(final ShenyuClientRegisterService service, fina
9696
if (CollectionUtils.isNotEmpty(draining)) {
9797
service.draining(selectorName, draining, namespaceId);
9898
}
99+
List<URIRegisterDTO> notReady = groupByEventType.get(EventType.NOT_READY);
100+
if (CollectionUtils.isNotEmpty(notReady)) {
101+
service.notReady(selectorName, notReady, namespaceId);
102+
}
103+
List<URIRegisterDTO> ready = groupByEventType.get(EventType.READY);
104+
if (CollectionUtils.isNotEmpty(ready)) {
105+
service.ready(selectorName, ready, namespaceId);
106+
}
99107
});
100108
}
101109

‎shenyu-admin/src/main/java/org/apache/shenyu/admin/service/DiscoveryUpstreamService.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,26 @@ public interface DiscoveryUpstreamService {
152152
*/
153153
void markDrainingBySelectorIdAndUrl(String selectorId, String url, boolean draining);
154154

155+
/**
156+
* Merge a not-ready marker into the props of the upstream matching the selector
157+
* and url. Idempotent: the marker is written, never derived from status, so a
158+
* heartbeat-driven status flip cannot clear it.
159+
*
160+
* @param selectorId selectorId
161+
* @param url url
162+
* @param notReady true to mark not ready, false to release the marker
163+
*/
164+
void markNotReadyBySelectorIdAndUrl(String selectorId, String url, boolean notReady);
165+
166+
/**
167+
* Release both admission markers (draining and not-ready) from the upstream
168+
* matching the selector and url. Idempotent.
169+
*
170+
* @param selectorId selectorId
171+
* @param url url
172+
*/
173+
void markReadyBySelectorIdAndUrl(String selectorId, String url);
174+
155175
/**
156176
* Import the discoveryUpstream data list.
157177
*

‎shenyu-admin/src/main/java/org/apache/shenyu/admin/service/impl/DiscoveryUpstreamServiceImpl.java‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@
6666
import java.util.Objects;
6767
import java.util.Set;
6868
import java.util.stream.Collectors;
69+
import java.util.function.Consumer;
6970
import java.util.function.Function;
7071

7172
@Service
@@ -79,6 +80,9 @@ public class DiscoveryUpstreamServiceImpl implements DiscoveryUpstreamService {
7980
// free-form props key carrying the draining admission marker (#8042)
8081
private static final String DRAINING_PROP_KEY = "draining";
8182

83+
// free-form props key carrying the not-ready admission marker (#8042)
84+
private static final String NOT_READY_PROP_KEY = "notReady";
85+
8286
private final DiscoveryUpstreamMapper discoveryUpstreamMapper;
8387

8488
private final DiscoveryHandlerMapper discoveryHandlerMapper;
@@ -361,6 +365,25 @@ public void changeStatusBySelectorIdAndUrl(final String selectorId, final String
361365
@Override
362366
@Transactional(rollbackFor = Exception.class)
363367
public void markDrainingBySelectorIdAndUrl(final String selectorId, final String url, final boolean draining) {
368+
updateUpstreamProps(selectorId, url, props -> props.put(DRAINING_PROP_KEY, draining));
369+
}
370+
371+
@Override
372+
@Transactional(rollbackFor = Exception.class)
373+
public void markNotReadyBySelectorIdAndUrl(final String selectorId, final String url, final boolean notReady) {
374+
updateUpstreamProps(selectorId, url, props -> props.put(NOT_READY_PROP_KEY, notReady));
375+
}
376+
377+
@Override
378+
@Transactional(rollbackFor = Exception.class)
379+
public void markReadyBySelectorIdAndUrl(final String selectorId, final String url) {
380+
updateUpstreamProps(selectorId, url, props -> {
381+
props.remove(DRAINING_PROP_KEY);
382+
props.remove(NOT_READY_PROP_KEY);
383+
});
384+
}
385+
386+
private void updateUpstreamProps(final String selectorId, final String url, final Consumer<Map<String, Object>> mutation) {
364387
DiscoveryHandlerDO discoveryHandlerDO = discoveryHandlerMapper.selectBySelectorId(selectorId);
365388
if (Objects.isNull(discoveryHandlerDO)) {
366389
return;
@@ -371,7 +394,7 @@ public void markDrainingBySelectorIdAndUrl(final String selectorId, final String
371394
}
372395
Map<String, Object> props = !StringUtils.hasLength(upstream.getProps())
373396
? new LinkedHashMap<>() : GsonUtils.getInstance().convertToMap(upstream.getProps());
374-
props.put(DRAINING_PROP_KEY, draining);
397+
mutation.accept(props);
375398
discoveryUpstreamMapper.updatePropsByUrl(discoveryHandlerDO.getId(), url, GsonUtils.getInstance().toJson(props));
376399
}
377400

‎shenyu-admin/src/main/java/org/apache/shenyu/admin/service/register/AbstractShenyuClientRegisterServiceImpl.java‎

Lines changed: 27 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@
6464
import java.util.Collections;
6565
import java.util.List;
6666
import java.util.Objects;
67+
import java.util.function.BiConsumer;
6768
import java.util.stream.Collectors;
6869

6970
import static org.apache.shenyu.common.constant.Constants.SYS_DEFAULT_NAMESPACE_ID;
@@ -286,25 +287,44 @@ public String doHeartbeat(final String selectorName, final List<URIRegisterDTO>
286287

287288
@Override
288289
public String doDraining(final String selectorName, final List<URIRegisterDTO> uriList, final String namespaceId) {
290+
return updateAdmission(selectorName, uriList, namespaceId, "doDraining",
291+
(selectorId, url) -> discoveryUpstreamService.markDrainingBySelectorIdAndUrl(selectorId, url, true));
292+
}
293+
294+
@Override
295+
public String doNotReady(final String selectorName, final List<URIRegisterDTO> uriList, final String namespaceId) {
296+
return updateAdmission(selectorName, uriList, namespaceId, "doNotReady",
297+
(selectorId, url) -> discoveryUpstreamService.markNotReadyBySelectorIdAndUrl(selectorId, url, true));
298+
}
299+
300+
@Override
301+
public String doReady(final String selectorName, final List<URIRegisterDTO> uriList, final String namespaceId) {
302+
return updateAdmission(selectorName, uriList, namespaceId, "doReady",
303+
(selectorId, url) -> discoveryUpstreamService.markReadyBySelectorIdAndUrl(selectorId, url));
304+
}
305+
306+
private String updateAdmission(final String selectorName, final List<URIRegisterDTO> uriList,
307+
final String namespaceId, final String action,
308+
final BiConsumer<String, String> marker) {
289309
if (CollectionUtils.isEmpty(uriList)) {
290310
return "";
291311
}
292312
String pluginName = PluginNameAdapter.rpcTypeAdapter(rpcType());
293313
SelectorDO selectorDO = selectorService.findByNameAndPluginNameAndNamespaceId(selectorName, pluginName, namespaceId);
294314
if (Objects.isNull(selectorDO)) {
295-
throw new ShenyuException("doDraining Failed to execute, wait to retry.");
315+
throw new ShenyuException(action + " Failed to execute, wait to retry.");
296316
}
297317
List<URIRegisterDTO> validUriList = uriList.stream().filter(dto -> Objects.nonNull(dto.getPort()) && StringUtils.isNotBlank(dto.getHost())).toList();
298318
if (CollectionUtils.isEmpty(validUriList)) {
299319
return null;
300320
}
301321
String selectorId = selectorDO.getId();
302-
// the marker travels in props, not in status: heartbeats keep flipping status
303-
// back to enabled, and a draining instance must stay excluded regardless (#8042)
322+
// admission markers travel in props, not in status: heartbeats keep flipping status
323+
// back to enabled, and an excluded instance must stay excluded regardless (#8042)
304324
validUriList.forEach(uriRegisterDTO -> {
305325
String url = CommonUpstreamUtils.buildDefaultDiscoveryUpstreamDTO(uriRegisterDTO.getHost(),
306326
uriRegisterDTO.getPort(), uriRegisterDTO.getProtocol(), uriRegisterDTO.getNamespaceId()).getUrl();
307-
discoveryUpstreamService.markDrainingBySelectorIdAndUrl(selectorId, url, true);
327+
marker.accept(selectorId, url);
308328
});
309329
DiscoverySyncData discoverySyncData = fetch(selectorId, selectorDO.getSelectorName(), pluginName, namespaceId);
310330
eventPublisher.publishEvent(new DataChangedEvent(ConfigGroupEnum.DISCOVER_UPSTREAM, DataEventTypeEnum.UPDATE, Collections.singletonList(discoverySyncData)));
@@ -318,9 +338,9 @@ protected void doDiscoveryLocal(final SelectorDO selectorDO, final String plugin
318338
uriRegisterDTO.getPort(), uriRegisterDTO.getProtocol(), selectorDO.getNamespaceId());
319339
discoveryUpstreamDTO.setDiscoveryHandlerId(discoveryHandlerId);
320340
discoveryUpstreamService.nativeCreateOrUpdate(discoveryUpstreamDTO);
321-
// a fresh registration claims the identity: a restart releases the drain,
322-
// while heartbeats alone can never do so (#8042)
323-
discoveryUpstreamService.markDrainingBySelectorIdAndUrl(selectorDO.getId(), discoveryUpstreamDTO.getUrl(), false);
341+
// a fresh registration claims the identity and starts ready: it releases both
342+
// the drain and the not-ready marker, while heartbeats alone can never do so (#8042)
343+
discoveryUpstreamService.markReadyBySelectorIdAndUrl(selectorDO.getId(), discoveryUpstreamDTO.getUrl());
324344
}
325345
DiscoverySyncData discoverySyncData = fetch(selectorDO.getId(), selectorDO.getSelectorName(), pluginName, selectorDO.getNamespaceId());
326346
eventPublisher.publishEvent(new DataChangedEvent(ConfigGroupEnum.DISCOVER_UPSTREAM, DataEventTypeEnum.UPDATE, Collections.singletonList(discoverySyncData)));

‎shenyu-admin/src/main/java/org/apache/shenyu/admin/service/register/FallbackShenyuClientRegisterService.java‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,16 @@ public String heartbeat(final String selectorName, final List<URIRegisterDTO> ur
8282
public String draining(final String selectorName, final List<URIRegisterDTO> uriList, final String namespaceId) {
8383
return doDraining(selectorName, uriList, namespaceId);
8484
}
85+
86+
@Override
87+
public String notReady(final String selectorName, final List<URIRegisterDTO> uriList, final String namespaceId) {
88+
return doNotReady(selectorName, uriList, namespaceId);
89+
}
90+
91+
@Override
92+
public String ready(final String selectorName, final List<URIRegisterDTO> uriList, final String namespaceId) {
93+
return doReady(selectorName, uriList, namespaceId);
94+
}
8595

8696
private void addFallback(final String key, final FallbackHolder holder) {
8797
FallbackHolder oldObj = fallsRegisters.get(key);
@@ -141,6 +151,27 @@ private String key(final String selectorName, final String namespaceId) {
141151
*/
142152
abstract String doDraining(String selectorName, List<URIRegisterDTO> uriList, String namespaceId);
143153

154+
/**
155+
* Not ready: mark the instances so gateways stop selecting them for new work
156+
* until an explicit ready event.
157+
*
158+
* @param selectorName the selector name
159+
* @param uriList the uri list
160+
* @param namespaceId the namespace id
161+
* @return the string
162+
*/
163+
abstract String doNotReady(String selectorName, List<URIRegisterDTO> uriList, String namespaceId);
164+
165+
/**
166+
* Ready: release the not-ready and draining markers so new work is admitted again.
167+
*
168+
* @param selectorName the selector name
169+
* @param uriList the uri list
170+
* @param namespaceId the namespace id
171+
* @return the string
172+
*/
173+
abstract String doReady(String selectorName, List<URIRegisterDTO> uriList, String namespaceId);
174+
144175
/**
145176
* The type Fall holder.
146177
*/

‎shenyu-admin/src/main/java/org/apache/shenyu/admin/service/register/ShenyuClientRegisterService.java‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,30 @@ default String offline(final String selectorName, final List<URIRegisterDTO> uri
106106
default String draining(final String selectorName, final List<URIRegisterDTO> uriList, final String namespaceId) {
107107
return Constants.SUCCESS;
108108
}
109+
110+
/**
111+
* not-ready: the instances stay registered but are not ready to receive new work.
112+
*
113+
* @param selectorName the selector name
114+
* @param uriList the uri list
115+
* @param namespaceId the namespace id
116+
* @return the string
117+
*/
118+
default String notReady(final String selectorName, final List<URIRegisterDTO> uriList, final String namespaceId) {
119+
return Constants.SUCCESS;
120+
}
121+
122+
/**
123+
* ready: the instances may receive new work again, releasing a previous drain.
124+
*
125+
* @param selectorName the selector name
126+
* @param uriList the uri list
127+
* @param namespaceId the namespace id
128+
* @return the string
129+
*/
130+
default String ready(final String selectorName, final List<URIRegisterDTO> uriList, final String namespaceId) {
131+
return Constants.SUCCESS;
132+
}
109133

110134
/**
111135
* Register context path.

‎shenyu-admin/src/test/java/org/apache/shenyu/admin/disruptor/subscriber/URIRegisterExecutorSubscriberTest.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,10 @@ public void testExecutorGroupsUrisByNamespace() {
103103
verify(service).offline(eq(selectorName), argThat(uris -> belongsToNamespace(uris, secondNamespace)), eq(secondNamespace));
104104
verify(service).draining(eq(selectorName), argThat(uris -> belongsToNamespace(uris, firstNamespace)), eq(firstNamespace));
105105
verify(service).draining(eq(selectorName), argThat(uris -> belongsToNamespace(uris, secondNamespace)), eq(secondNamespace));
106+
verify(service).notReady(eq(selectorName), argThat(uris -> belongsToNamespace(uris, firstNamespace)), eq(firstNamespace));
107+
verify(service).notReady(eq(selectorName), argThat(uris -> belongsToNamespace(uris, secondNamespace)), eq(secondNamespace));
108+
verify(service).ready(eq(selectorName), argThat(uris -> belongsToNamespace(uris, firstNamespace)), eq(firstNamespace));
109+
verify(service).ready(eq(selectorName), argThat(uris -> belongsToNamespace(uris, secondNamespace)), eq(secondNamespace));
106110
}
107111

108112
@Test

‎shenyu-admin/src/test/java/org/apache/shenyu/admin/service/DiscoveryUpstreamServiceTest.java‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -204,6 +204,56 @@ public void testMarkDrainingIgnoresUnknownHandlerOrUrl() {
204204
verify(discoveryUpstreamMapper, never()).updatePropsByUrl(anyString(), anyString(), anyString());
205205
}
206206

207+
@Test
208+
public void testMarkNotReadyMergesPropsAndLeavesStatusUntouched() {
209+
final DiscoveryHandlerDO handler = new DiscoveryHandlerDO();
210+
handler.setId("handler-1");
211+
when(discoveryHandlerMapper.selectBySelectorId("selector-1")).thenReturn(handler);
212+
final DiscoveryUpstreamDO upstream = new DiscoveryUpstreamDO();
213+
upstream.setDiscoveryHandlerId("handler-1");
214+
upstream.setUpstreamUrl("127.0.0.1:8080");
215+
upstream.setProps("{\"warmup\":\"20\"}");
216+
when(discoveryUpstreamMapper.selectByDiscoveryHandlerIdAndUrl("handler-1", "127.0.0.1:8080")).thenReturn(upstream);
217+
218+
discoveryUpstreamService.markNotReadyBySelectorIdAndUrl("selector-1", "127.0.0.1:8080", true);
219+
220+
final ArgumentCaptor<String> propsCaptor = ArgumentCaptor.forClass(String.class);
221+
verify(discoveryUpstreamMapper).updatePropsByUrl(eq("handler-1"), eq("127.0.0.1:8080"), propsCaptor.capture());
222+
verify(discoveryUpstreamMapper, never()).updateStatusByUrl(anyString(), anyString(), org.mockito.ArgumentMatchers.anyInt());
223+
final Map<String, Object> merged = GsonUtils.getInstance().convertToMap(propsCaptor.getValue());
224+
assertEquals(Boolean.TRUE, merged.get("notReady"));
225+
assertEquals("20", merged.get("warmup"));
226+
}
227+
228+
@Test
229+
public void testMarkReadyReleasesBothMarkers() {
230+
final DiscoveryHandlerDO handler = new DiscoveryHandlerDO();
231+
handler.setId("handler-1");
232+
when(discoveryHandlerMapper.selectBySelectorId("selector-1")).thenReturn(handler);
233+
final DiscoveryUpstreamDO upstream = new DiscoveryUpstreamDO();
234+
upstream.setDiscoveryHandlerId("handler-1");
235+
upstream.setUpstreamUrl("127.0.0.1:8080");
236+
upstream.setProps("{\"draining\":true,\"notReady\":true,\"warmup\":\"20\"}");
237+
when(discoveryUpstreamMapper.selectByDiscoveryHandlerIdAndUrl("handler-1", "127.0.0.1:8080")).thenReturn(upstream);
238+
239+
discoveryUpstreamService.markReadyBySelectorIdAndUrl("selector-1", "127.0.0.1:8080");
240+
241+
final ArgumentCaptor<String> propsCaptor = ArgumentCaptor.forClass(String.class);
242+
verify(discoveryUpstreamMapper).updatePropsByUrl(eq("handler-1"), eq("127.0.0.1:8080"), propsCaptor.capture());
243+
final Map<String, Object> merged = GsonUtils.getInstance().convertToMap(propsCaptor.getValue());
244+
Assertions.assertNull(merged.get("draining"));
245+
Assertions.assertNull(merged.get("notReady"));
246+
assertEquals("20", merged.get("warmup"));
247+
}
248+
249+
@Test
250+
public void testReadinessMarkersIgnoreUnknownHandlerOrUrl() {
251+
when(discoveryHandlerMapper.selectBySelectorId("selector-1")).thenReturn(null);
252+
discoveryUpstreamService.markNotReadyBySelectorIdAndUrl("selector-1", "127.0.0.1:8080", true);
253+
discoveryUpstreamService.markReadyBySelectorIdAndUrl("selector-1", "127.0.0.1:8080");
254+
verify(discoveryUpstreamMapper, never()).updatePropsByUrl(anyString(), anyString(), anyString());
255+
}
256+
207257
@Test
208258
public void testHeartbeatDoesNotTouchDrainingProps() {
209259
// the drain marker lives in props; the heartbeat path only flips the status column,

‎shenyu-admin/src/test/java/org/apache/shenyu/admin/service/register/AbstractShenyuClientRegisterServiceImplTest.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import org.apache.shenyu.admin.model.vo.NamespacePluginVO;
2727
import org.apache.shenyu.admin.model.vo.TagVO;
2828
import org.apache.shenyu.admin.service.SelectorService;
29+
import org.apache.shenyu.admin.service.DiscoveryUpstreamService;
2930
import org.apache.shenyu.admin.service.RuleService;
3031
import org.apache.shenyu.admin.service.ApiService;
3132
import org.apache.shenyu.admin.service.TagService;
@@ -61,6 +62,9 @@
6162
import static org.junit.jupiter.api.Assertions.assertTrue;
6263
import static org.junit.jupiter.api.Assertions.assertThrows;
6364
import static org.mockito.ArgumentMatchers.any;
65+
import static org.mockito.ArgumentMatchers.anyString;
66+
import static org.mockito.ArgumentMatchers.eq;
67+
import static org.mockito.Mockito.verify;
6468
import static org.mockito.Mockito.when;
6569
import static org.mockito.Mockito.mock;
6670

@@ -95,6 +99,9 @@ public final class AbstractShenyuClientRegisterServiceImplTest {
9599
@Mock
96100
private UpstreamCheckService upstreamCheckService;
97101

102+
@Mock
103+
private DiscoveryUpstreamService discoveryUpstreamService;
104+
98105
@Mock
99106
private RegisterApiDocService registerApiDocService;
100107

@@ -239,6 +246,24 @@ public void testBuildContextPathDefaultRuleDTO() {
239246
assertEquals("Rule_Handler", ruleDTO.getHandle());
240247
}
241248

249+
@Test
250+
public void testDoNotReadyAndDoReady() {
251+
SelectorDO selectorDO = new SelectorDO();
252+
selectorDO.setId("selector-1");
253+
selectorDO.setSelectorName("ctx");
254+
when(selectorService.findByNameAndPluginNameAndNamespaceId(anyString(), anyString(), anyString())).thenReturn(selectorDO);
255+
List<URIRegisterDTO> uriList = Collections.singletonList(URIRegisterDTO.builder()
256+
.host("127.0.0.1").port(8080).protocol("http://").namespaceId(SYS_DEFAULT_NAMESPACE_ID).build());
257+
258+
assertEquals(ShenyuResultMessage.SUCCESS,
259+
abstractShenyuClientRegisterService.doNotReady("ctx", uriList, SYS_DEFAULT_NAMESPACE_ID));
260+
verify(discoveryUpstreamService).markNotReadyBySelectorIdAndUrl(eq("selector-1"), anyString(), eq(true));
261+
262+
assertEquals(ShenyuResultMessage.SUCCESS,
263+
abstractShenyuClientRegisterService.doReady("ctx", uriList, SYS_DEFAULT_NAMESPACE_ID));
264+
verify(discoveryUpstreamService).markReadyBySelectorIdAndUrl(eq("selector-1"), anyString());
265+
}
266+
242267
static class MockAbstractShenyuClientRegisterServiceImpl extends AbstractShenyuClientRegisterServiceImpl {
243268

244269
@Override

0 commit comments

Comments
 (0)