Skip to content

Commit d168632

Browse files
voidzcyvoidzcy
authored
xds: support multiple xDS servers in bootstrap file (#6493)
Support bootstrap file containing multiple xDS servers, with each has its own server URI and channel credential options. Multiple xDS servers are provided in case of one not reachable. For now, we would only use the first one. This change also formats JSON strings in bootstrap related tests and add several tests for parsing bootstrap JSON as completeness. Implementation of XdsClient is changed to take in a list of xDS servers. But still, we only use the first one.
1 parent 2963878 commit d168632

7 files changed

Lines changed: 351 additions & 119 deletions

File tree

‎xds/src/main/java/io/grpc/xds/Bootstrapper.java‎

Lines changed: 54 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -73,28 +73,33 @@ static BootstrapInfo parseConfig(String rawData) throws IOException {
7373
@SuppressWarnings("unchecked")
7474
Map<String, ?> rawBootstrap = (Map<String, ?>) JsonParser.parse(rawData);
7575

76-
Map<String, ?> rawServerConfig = JsonUtil.getObject(rawBootstrap, "xds_server");
77-
if (rawServerConfig == null) {
78-
throw new IOException("Invalid bootstrap: 'xds_server' does not exist.");
76+
List<ServerInfo> servers = new ArrayList<>();
77+
List<?> rawServerConfigs = JsonUtil.getList(rawBootstrap, "xds_servers");
78+
if (rawServerConfigs == null) {
79+
throw new IOException("Invalid bootstrap: 'xds_servers' does not exist.");
7980
}
80-
// Field "server_uri" is required.
81-
String serverUri = JsonUtil.getString(rawServerConfig, "server_uri");
82-
if (serverUri == null) {
83-
throw new IOException("Invalid bootstrap: 'xds_server : server_uri' does not exist.");
84-
}
85-
List<ChannelCreds> channelCredsOptions = new ArrayList<>();
86-
List<?> rawChannelCredsList = JsonUtil.getList(rawServerConfig, "channel_creds");
87-
// List of channel creds is optional.
88-
if (rawChannelCredsList != null) {
89-
List<Map<String, ?>> channelCredsList = JsonUtil.checkObjectList(rawChannelCredsList);
90-
for (Map<String, ?> channelCreds : channelCredsList) {
91-
String type = JsonUtil.getString(channelCreds, "type");
92-
if (type == null) {
93-
throw new IOException("Invalid bootstrap: 'channel_creds' contains unknown type.");
81+
List<Map<String, ?>> serverConfigList = JsonUtil.checkObjectList(rawServerConfigs);
82+
for (Map<String, ?> serverConfig : serverConfigList) {
83+
String serverUri = JsonUtil.getString(serverConfig, "server_uri");
84+
if (serverUri == null) {
85+
throw new IOException("Invalid bootstrap: 'xds_servers' contains unknown server.");
86+
}
87+
List<ChannelCreds> channelCredsOptions = new ArrayList<>();
88+
List<?> rawChannelCredsList = JsonUtil.getList(serverConfig, "channel_creds");
89+
// List of channel creds is optional.
90+
if (rawChannelCredsList != null) {
91+
List<Map<String, ?>> channelCredsList = JsonUtil.checkObjectList(rawChannelCredsList);
92+
for (Map<String, ?> channelCreds : channelCredsList) {
93+
String type = JsonUtil.getString(channelCreds, "type");
94+
if (type == null) {
95+
throw new IOException("Invalid bootstrap: 'xds_servers' contains server with "
96+
+ "unknown type 'channel_creds'.");
97+
}
98+
ChannelCreds creds = new ChannelCreds(type, JsonUtil.getObject(channelCreds, "config"));
99+
channelCredsOptions.add(creds);
94100
}
95-
ChannelCreds creds = new ChannelCreds(type, JsonUtil.getObject(channelCreds, "config"));
96-
channelCredsOptions.add(creds);
97101
}
102+
servers.add(new ServerInfo(serverUri, channelCredsOptions));
98103
}
99104

100105
Node.Builder nodeBuilder = Node.newBuilder();
@@ -133,7 +138,7 @@ static BootstrapInfo parseConfig(String rawData) throws IOException {
133138
}
134139
nodeBuilder.setBuildVersion(GrpcUtil.getGrpcBuildVersion());
135140

136-
return new BootstrapInfo(serverUri, channelCredsOptions, nodeBuilder.build());
141+
return new BootstrapInfo(servers, nodeBuilder.build());
137142
}
138143

139144
/**
@@ -203,26 +208,48 @@ String getType() {
203208
}
204209

205210
/**
206-
* Data class containing the results of reading bootstrap.
211+
* Data class containing xDS server information, such as server URI and channel credential
212+
* options to be used for communication.
207213
*/
208214
@Immutable
209-
public static class BootstrapInfo {
215+
static class ServerInfo {
210216
private final String serverUri;
211217
private final List<ChannelCreds> channelCredsList;
212-
private final Node node;
213218

214219
@VisibleForTesting
215-
BootstrapInfo(String serverUri, List<ChannelCreds> channelCredsList, Node node) {
220+
ServerInfo(String serverUri, List<ChannelCreds> channelCredsList) {
216221
this.serverUri = serverUri;
217222
this.channelCredsList = channelCredsList;
223+
}
224+
225+
String getServerUri() {
226+
return serverUri;
227+
}
228+
229+
List<ChannelCreds> getChannelCredentials() {
230+
return Collections.unmodifiableList(channelCredsList);
231+
}
232+
}
233+
234+
/**
235+
* Data class containing the results of reading bootstrap.
236+
*/
237+
@Immutable
238+
public static class BootstrapInfo {
239+
private List<ServerInfo> servers;
240+
private final Node node;
241+
242+
@VisibleForTesting
243+
BootstrapInfo(List<ServerInfo> servers, Node node) {
244+
this.servers = servers;
218245
this.node = node;
219246
}
220247

221248
/**
222-
* Returns the URI the traffic director to be connected to.
249+
* Returns the list of xDS servers to be connected to.
223250
*/
224-
String getServerUri() {
225-
return serverUri;
251+
List<ServerInfo> getServers() {
252+
return Collections.unmodifiableList(servers);
226253
}
227254

228255
/**
@@ -232,11 +259,5 @@ public Node getNode() {
232259
return node;
233260
}
234261

235-
/**
236-
* Returns the credentials to use when communicating with the xDS server.
237-
*/
238-
List<ChannelCreds> getChannelCredentials() {
239-
return Collections.unmodifiableList(channelCredsList);
240-
}
241262
}
242263
}

‎xds/src/main/java/io/grpc/xds/LookasideLb.java‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
import io.grpc.util.GracefulSwitchLoadBalancer;
4242
import io.grpc.xds.Bootstrapper.BootstrapInfo;
4343
import io.grpc.xds.Bootstrapper.ChannelCreds;
44+
import io.grpc.xds.Bootstrapper.ServerInfo;
4445
import io.grpc.xds.EnvoyProtoData.DropOverload;
4546
import io.grpc.xds.EnvoyProtoData.Locality;
4647
import io.grpc.xds.EnvoyProtoData.LocalityLbEndpoints;
@@ -204,9 +205,21 @@ XdsClient createXdsClient() {
204205
new ErrorPicker(Status.UNAVAILABLE.withCause(e)));
205206
return;
206207
}
208+
209+
List<ServerInfo> serverList = bootstrapInfo.getServers();
210+
if (serverList.isEmpty()) {
211+
lookasideLbHelper.updateBalancingState(
212+
TRANSIENT_FAILURE,
213+
new ErrorPicker(
214+
Status.UNAVAILABLE
215+
.withDescription("No traffic director provided by bootstrap")));
216+
return;
217+
}
218+
// Currently we only support using the first server from bootstrap.
219+
ServerInfo serverInfo = serverList.get(0);
207220
channel = initLbChannel(
208-
lookasideLbHelper, bootstrapInfo.getServerUri(),
209-
bootstrapInfo.getChannelCredentials());
221+
lookasideLbHelper, serverInfo.getServerUri(),
222+
serverInfo.getChannelCredentials());
210223
xdsClientRef = new RefCountedXdsClientObjectPool(new XdsClientFactory() {
211224
@Override
212225
XdsClient createXdsClient() {

‎xds/src/main/java/io/grpc/xds/XdsClientImpl.java‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
import io.grpc.internal.BackoffPolicy;
5050
import io.grpc.stub.StreamObserver;
5151
import io.grpc.xds.Bootstrapper.ChannelCreds;
52+
import io.grpc.xds.Bootstrapper.ServerInfo;
5253
import io.grpc.xds.EnvoyProtoData.DropOverload;
5354
import io.grpc.xds.EnvoyProtoData.Locality;
5455
import io.grpc.xds.EnvoyProtoData.LocalityLbEndpoints;
@@ -135,19 +136,14 @@ final class XdsClientImpl extends XdsClient {
135136
private String ldsResourceName;
136137

137138
XdsClientImpl(
138-
// URI of the management server to be connected to.
139-
String serverUri,
139+
List<ServerInfo> servers, // list of management servers
140140
Node node,
141-
// List of channel credential configurations for the channel to management server.
142-
// Should pick the first supported one.
143-
List<ChannelCreds> channelCredsList,
144141
SynchronizationContext syncContext,
145142
ScheduledExecutorService timeService,
146143
BackoffPolicy.Provider backoffPolicyProvider,
147144
Stopwatch stopwatch) {
148145
this(
149-
buildChannel(checkNotNull(serverUri, "serverUri"),
150-
checkNotNull(channelCredsList, "channelCredsList")),
146+
buildChannel(checkNotNull(servers, "servers")),
151147
node,
152148
syncContext,
153149
timeService,
@@ -319,9 +315,15 @@ void cancelEndpointDataWatch(String clusterName, EndpointWatcher watcher) {
319315
}
320316

321317
/**
322-
* Builds a channel to the given server URI with the first supported channel creds config.
318+
* Builds a channel to one of the provided management servers.
319+
*
320+
* <p>Note: currently we only support using the first server.
323321
*/
324-
private static ManagedChannel buildChannel(String serverUri,List<ChannelCreds> channelCredsList) {
322+
private static ManagedChannel buildChannel(List<ServerInfo> servers) {
323+
checkArgument(!servers.isEmpty(), "No management server provided.");
324+
ServerInfo serverInfo = servers.get(0);
325+
String serverUri = serverInfo.getServerUri();
326+
List<ChannelCreds> channelCredsList = serverInfo.getChannelCredentials();
325327
ManagedChannel ch = null;
326328
// Use the first supported channel credentials configuration.
327329
// Currently, only "google_default" is supported.

‎xds/src/main/java/io/grpc/xds/XdsNameResolver.java‎

Lines changed: 22 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import io.grpc.internal.JsonParser;
3030
import io.grpc.xds.Bootstrapper.BootstrapInfo;
3131
import io.grpc.xds.Bootstrapper.ChannelCreds;
32+
import io.grpc.xds.Bootstrapper.ServerInfo;
3233
import java.io.IOException;
3334
import java.net.URI;
3435
import java.util.Collections;
@@ -89,13 +90,26 @@ public void start(final Listener2 listener) {
8990
return;
9091
}
9192

92-
String serviceConfig = "{"
93-
+ "\"loadBalancingConfig\": ["
94-
+ "{\"xds_experimental\" : {"
95-
+ "\"balancerName\" : \"" + bootstrapInfo.getServerUri() + "\","
96-
+ "\"childPolicy\" : [{\"round_robin\" : {}}]"
97-
+ "}}"
98-
+ "]}";
93+
List<ServerInfo> serverList = bootstrapInfo.getServers();
94+
if (serverList.isEmpty()) {
95+
listener.onError(
96+
Status.UNAVAILABLE.withDescription("No traffic director provided by bootstrap"));
97+
return;
98+
}
99+
100+
// Currently we only support using the first server from bootstrap.
101+
ServerInfo serverInfo = serverList.get(0);
102+
103+
String serviceConfig = "{\n"
104+
+ " \"loadBalancingConfig\": [\n"
105+
+ " {\n"
106+
+ " \"xds_experimental\": {\n"
107+
+ " \"balancerName\": \"" + serverInfo.getServerUri() + "\",\n"
108+
+ " \"childPolicy\": [ {\"round_robin\": {} } ]\n"
109+
+ " }\n"
110+
+ " }"
111+
+ " ]\n"
112+
+ "}";
99113
Map<String, ?> config;
100114
try {
101115
config = (Map<String, ?>) JsonParser.parse(serviceConfig);
@@ -108,7 +122,7 @@ public void start(final Listener2 listener) {
108122
Attributes.newBuilder()
109123
.set(GrpcAttributes.NAME_RESOLVER_SERVICE_CONFIG, config)
110124
.set(XDS_NODE, bootstrapInfo.getNode())
111-
.set(XDS_CHANNEL_CREDS_LIST, bootstrapInfo.getChannelCredentials())
125+
.set(XDS_CHANNEL_CREDS_LIST, serverInfo.getChannelCredentials())
112126
.build();
113127
ResolutionResult result =
114128
ResolutionResult.newBuilder()

0 commit comments

Comments
 (0)