envoyproxy · dio · May 1, 2018 · May 1, 2018 · May 3, 2018 · May 3, 2018
diff --git a/DEPRECATED.md b/DEPRECATED.md
@@ -21,6 +21,9 @@ A logged warning is expected for each deprecated item that is in deprecation win
 * `SAN` is replaced by `URI` in the `x-forwarded-client-cert` header.
 * The `endpoint` field in the http health check filter is deprecated in favor of the `headers`
   field where one can specify HeaderMatch objects to match on.
+* The `hosts` field in the `Cluster` is deprecated in favor of the `load_assigment` field. This change
+  allows CDS static/DNS assignments to contain embedded EDS equivalent endpoint assignments and
+  accommodates specifying `HealthCheckConfig` for each cluster member.
 
 ## Version 1.6.0 (March 20, 2018)
 

diff --git a/api/envoy/api/v2/BUILD b/api/envoy/api/v2/BUILD
@@ -62,6 +62,7 @@ api_proto_library(
     visibility = [":friends"],
     deps = [
         ":discovery",
+        ":eds",
         "//envoy/api/v2/auth:cert",
         "//envoy/api/v2/cluster:circuit_breaker",
         "//envoy/api/v2/cluster:outlier_detection",
@@ -70,6 +71,7 @@ api_proto_library(
         "//envoy/api/v2/core:config_source",
         "//envoy/api/v2/core:health_check",
         "//envoy/api/v2/core:protocol",
+        "//envoy/api/v2/endpoint",
         "//envoy/type:percent",
     ],
 )
@@ -79,6 +81,7 @@ api_go_grpc_library(
     proto = ":cds",
     deps = [
         ":discovery_go_proto",
+        ":eds_go_grpc",
         "//envoy/api/v2/auth:cert_go_proto",
         "//envoy/api/v2/cluster:circuit_breaker_go_proto",
         "//envoy/api/v2/cluster:outlier_detection_go_proto",
@@ -87,6 +90,7 @@ api_go_grpc_library(
         "//envoy/api/v2/core:config_source_go_proto",
         "//envoy/api/v2/core:health_check_go_proto",
         "//envoy/api/v2/core:protocol_go_proto",
+        "//envoy/api/v2/endpoint:endpoint_go_proto",
         "//envoy/type:percent_go_proto",
     ],
 )

diff --git a/api/envoy/api/v2/cds.proto b/api/envoy/api/v2/cds.proto
@@ -13,6 +13,7 @@ import "envoy/api/v2/core/health_check.proto";
 import "envoy/api/v2/core/protocol.proto";
 import "envoy/api/v2/cluster/circuit_breaker.proto";
 import "envoy/api/v2/cluster/outlier_detection.proto";
+import "envoy/api/v2/eds.proto";
 import "envoy/type/percent.proto";
 
 import "google/api/annotations.proto";
@@ -41,7 +42,7 @@ service ClusterDiscoveryService {
 // [#protodoc-title: Clusters]
 
 // Configuration for a single upstream cluster.
-// [#comment:next free field: 32]
+// [#comment:next free field: 34]
 message Cluster {
   // Supplies the name of the cluster which must be unique across all clusters.
   // The cluster name is used when emitting
@@ -150,12 +151,26 @@ message Cluster {
   // when picking a host in the cluster.
   LbPolicy lb_policy = 6 [(validate.rules).enum.defined_only = true];
 
-  // If the service discovery type is
+  // Setting this is a requirement for clusters with discovery type
   // :ref:`STATIC<envoy_api_enum_value_Cluster.DiscoveryType.STATIC>`,
   // :ref:`STRICT_DNS<envoy_api_enum_value_Cluster.DiscoveryType.STRICT_DNS>`
-  // or :ref:`LOGICAL_DNS<envoy_api_enum_value_Cluster.DiscoveryType.LOGICAL_DNS>`,
-  // then hosts is required.
-  repeated core.Address hosts = 7;
+  // or :ref:`LOGICAL_DNS<envoy_api_enum_value_Cluster.DiscoveryType.LOGICAL_DNS>`.
+  // **This field is deprecated**. Set the :ref:`load_assignment<envoy_api_field_Cluster.load_assignment>`
+  // field instead.
+  repeated core.Address hosts = 7 [deprecated = true];
+
+  // Setting this is required for specifying members of
+  // :ref:`STATIC<envoy_api_enum_value_Cluster.DiscoveryType.STATIC>`,
 // The upstream host address. 
 // hostname to be resolved via DNS. If it is a hostname, :ref:`resolver_name 
 // The upstream host address. 
 // hostname to be resolved via DNS. If it is a hostname, :ref:`resolver_name 
+  // :ref:`STRICT_DNS<envoy_api_enum_value_Cluster.DiscoveryType.STRICT_DNS>`
+  // or :ref:`LOGICAL_DNS<envoy_api_enum_value_Cluster.DiscoveryType.LOGICAL_DNS>` clusters.
+  // This field supersedes :ref:`hosts<envoy_api_field_Cluster.hosts>` field.
+  //
+  // .. attention::
+  //
+  //   Setting this allows CDS static/DNS assignments to contain embedded EDS equivalent
+  //   :ref:`endpoint assignments<envoy_api_msg_ClusterLoadAssignment>`.
+  //   Setting this overrides :ref:`hosts<envoy_api_field_Cluster.hosts>` values.
+  ClusterLoadAssignment load_assignment = 33;
 
   // Optional :ref:`active health checking <arch_overview_health_checking>`
   // configuration for the cluster. If no

diff --git a/api/envoy/api/v2/endpoint/endpoint.proto b/api/envoy/api/v2/endpoint/endpoint.proto
@@ -21,7 +21,7 @@ message Endpoint {
   // The upstream host address.
   core.Address address = 1;
 
-  // [#not-implemented-hide:] The optional health check configuration.
+  // The optional health check configuration.
   message HealthCheckConfig {
     // Optional alternative health check port value.
     //
@@ -32,8 +32,8 @@ message Endpoint {
     uint32 port_value = 1;
   }
 
-  // [#not-implemented-hide:] The optional health check configuration is used as
-  // configuration for the health checker to contact the health checked host.
+  // The optional health check configuration is used as configuration for the health checker
+  // to contact the health checked host.
   //
   // .. attention::
   //

diff --git a/docs/root/intro/arch_overview/health_checking.rst b/docs/root/intro/arch_overview/health_checking.rst
@@ -24,6 +24,35 @@ unhealthy, successes required before marking a host healthy, etc.):
   maintenance by setting the specified key to any value and waiting for traffic to drain. See
   :ref:`redis_key <config_cluster_manager_cluster_hc_redis_key>`.
 
+Per cluster member health checking config
+-----------------------------------------
+
+If active health checking is configured for an upstream cluster, a specific additional configuration
+for each registered member can be specified by setting the
+:ref:`health check config<envoy_api_msg_endpoint.Endpoint.HealthCheckConfig>`
+in :ref:`endpoint message<envoy_api_msg_endpoint.Endpoint>`
+of an :ref:`LbEndpoint<envoy_api_msg_endpoint.LbEndpoint>` of each defined
+:ref:`LocalityLbEndpoints<envoy_api_msg_endpoint.LocalityLbEndpoints>`
+in a :ref:`ClusterLoadAssignment<envoy_api_msg_ClusterLoadAssignment>`.
+
+An example of setting up :ref:`health check config<envoy_api_msg_endpoint.Endpoint.HealthCheckConfig>`
+to set a :ref:`cluster member<envoy_api_msg_endpoint.Endpoint>`'s alternative health check
+:ref:`port<envoy_api_field_endpoint.Endpoint.HealthCheckConfig.port_value>` is:
+
+.. code-block:: yaml
+
+  load_assignment:
+    endpoints:
+    - lb_endpoints:
+      - endpoint:
+          health_check_config:
+            port_value: 8080
+
+          address:
+            socket_address:
+              address: localhost
+              port_value: 80
+
 Passive health checking
 -----------------------
 

diff --git a/source/common/config/utility.cc b/source/common/config/utility.cc
@@ -228,5 +228,20 @@ Grpc::AsyncClientFactoryPtr Utility::factoryForGrpcApiConfigSource(
   return async_client_manager.factoryForGrpcService(grpc_service, scope, false);
 }
 
+envoy::api::v2::ClusterLoadAssignment Utility::translateClusterHosts(
+    const Protobuf::RepeatedPtrField<envoy::api::v2::core::Address>& hosts) {
+  envoy::api::v2::ClusterLoadAssignment load_assignment;
+  envoy::api::v2::endpoint::LocalityLbEndpoints* locality_lb_endpoints =
+      load_assignment.add_endpoints();
+  // Since this LocalityLbEndpoints is built from hosts list, set the default weight to 1.
+  locality_lb_endpoints->mutable_load_balancing_weight()->set_value(1);
+  for (const envoy::api::v2::core::Address& host : hosts) {
+    envoy::api::v2::endpoint::LbEndpoint* lb_endpoint = locality_lb_endpoints->add_lb_endpoints();
+    lb_endpoint->mutable_endpoint()->mutable_address()->MergeFrom(host);
+    lb_endpoint->mutable_load_balancing_weight()->set_value(1);
+  }
+  return load_assignment;
+}
+
 } // namespace Config
 } // namespace Envoy
diff --git a/source/common/config/utility.h b/source/common/config/utility.h
@@ -284,6 +284,14 @@ class Utility {
   factoryForGrpcApiConfigSource(Grpc::AsyncClientManager& async_client_manager,
                                 const envoy::api::v2::core::ApiConfigSource& api_config_source,
                                 Stats::Scope& scope);
+
+  /**
+   * Translate a set of cluster's hosts into a load assignment configuration.
+   * @param hosts cluster's list of hosts.
+   * @return envoy::api::v2::ClusterLoadAssignment a load assignment configuration.
+   */
+  static envoy::api::v2::ClusterLoadAssignment
+  translateClusterHosts(const Protobuf::RepeatedPtrField<envoy::api::v2::core::Address>& hosts);
 };
 
 } // namespace Config

diff --git a/source/common/upstream/logical_dns_cluster.cc b/source/common/upstream/logical_dns_cluster.cc
@@ -18,6 +18,7 @@ namespace Upstream {
 LogicalDnsCluster::LogicalDnsCluster(const envoy::api::v2::Cluster& cluster,
                                      Runtime::Loader& runtime, Stats::Store& stats,
                                      Ssl::ContextManager& ssl_context_manager,
+                                     const LocalInfo::LocalInfo& local_info,
                                      Network::DnsResolverSharedPtr dns_resolver,
                                      ThreadLocal::SlotAllocator& tls, ClusterManager& cm,
                                      Event::Dispatcher& dispatcher, bool added_via_api)
@@ -26,10 +27,15 @@ LogicalDnsCluster::LogicalDnsCluster(const envoy::api::v2::Cluster& cluster,
       dns_refresh_rate_ms_(
           std::chrono::milliseconds(PROTOBUF_GET_MS_OR_DEFAULT(cluster, dns_refresh_rate, 5000))),
       tls_(tls.allocateSlot()),
-      resolve_timer_(dispatcher.createTimer([this]() -> void { startResolve(); })) {
-  const auto& hosts = cluster.hosts();
-  if (hosts.size() != 1) {
-    throw EnvoyException("logical_dns clusters must have a single host");
+      resolve_timer_(dispatcher.createTimer([this]() -> void { startResolve(); })),
+      priority_state_manager_(new PriorityStateManager(*this, local_info)),
+      load_assignment_(cluster.has_load_assignment()
+                           ? cluster.load_assignment()
+                           : Config::Utility::translateClusterHosts(cluster.hosts())) {
+  const auto& locality_lb_endpoints = load_assignment_.endpoints();
+  if (locality_lb_endpoints.size() != 1 || locality_lb_endpoints[0].lb_endpoints().size() != 1) {
+    throw EnvoyException(
+        "LOGICAL_DNS clusters must have a single locality_lb_endpoint and a single lb_endpoint");
   }
 
   switch (cluster.dns_lookup_family()) {
@@ -46,10 +52,12 @@ LogicalDnsCluster::LogicalDnsCluster(const envoy::api::v2::Cluster& cluster,
     NOT_REACHED;
   }
 
-  const auto& socket_address = hosts[0].socket_address();
+  const envoy::api::v2::core::SocketAddress& socket_address =
+      lbEndpoint().endpoint().address().socket_address();
   dns_url_ = fmt::format("tcp://{}:{}", socket_address.address(), socket_address.port_value());
   hostname_ = Network::Utility::hostFromTcpUrl(dns_url_);
   Network::Utility::portFromTcpUrl(dns_url_);
+  priority_state_manager_->initializePerPriority(localityLbEndpoint());
 
   tls_->set([](Event::Dispatcher&) -> ThreadLocal::ThreadLocalObjectSharedPtr {
     return std::make_shared<PerThreadCurrentHostData>();
@@ -87,7 +95,8 @@ void LogicalDnsCluster::startResolve() {
             current_resolved_address_ = new_address;
             // Capture URL to avoid a race with another update.
             tls_->runOnAllThreads([this, new_address]() -> void {
-              tls_->getTyped<PerThreadCurrentHostData>().current_resolved_address_ = new_address;
+              PerThreadCurrentHostData& data = tls_->getTyped<PerThreadCurrentHostData>();
+              data.current_resolved_address_ = new_address;
             });
           }
 
@@ -106,14 +115,11 @@ void LogicalDnsCluster::startResolve() {
                   new LogicalHost(info_, hostname_, Network::Utility::getIpv6AnyAddress(), *this));
               break;
             }
-            HostVectorSharedPtr new_hosts(new HostVector());
-            new_hosts->emplace_back(logical_host_);
-            // Given the current config, only EDS clusters support multiple priorities.
-            ASSERT(priority_set_.hostSetsPerPriority().size() == 1);
-            auto& first_host_set = priority_set_.getOrCreateHostSet(0);
-            first_host_set.updateHosts(new_hosts, createHealthyHostList(*new_hosts),
-                                       HostsPerLocalityImpl::empty(), HostsPerLocalityImpl::empty(),
-                                       {}, *new_hosts, {});
+
+            const uint32_t priority = localityLbEndpoint().priority();
+            priority_state_manager_->registerHostPerPriority(logical_host_, priority);
+            priority_state_manager_->updateClusterPrioritySet(priority, absl::nullopt,
+                                                              absl::nullopt, false);
           }
         }
 
@@ -130,7 +136,8 @@ Upstream::Host::CreateConnectionData LogicalDnsCluster::LogicalHost::createConne
   return {HostImpl::createConnection(dispatcher, *parent_.info_, data.current_resolved_address_,
                                      options),
           HostDescriptionConstSharedPtr{
-              new RealHostDescription(data.current_resolved_address_, shared_from_this())}};
+              new RealHostDescription(data.current_resolved_address_, parent_.localityLbEndpoint(),
+                                      parent_.lbEndpoint(), shared_from_this())}};
 }
 
 } // namespace Upstream

diff --git a/source/common/upstream/logical_dns_cluster.h b/source/common/upstream/logical_dns_cluster.h
@@ -30,6 +30,7 @@ class LogicalDnsCluster : public ClusterImplBase {
 public:
   LogicalDnsCluster(const envoy::api::v2::Cluster& cluster, Runtime::Loader& runtime,
                     Stats::Store& stats, Ssl::ContextManager& ssl_context_manager,
+                    const LocalInfo::LocalInfo& local_info,
                     Network::DnsResolverSharedPtr dns_resolver, ThreadLocal::SlotAllocator& tls,
                     ClusterManager& cm, Event::Dispatcher& dispatcher, bool added_via_api);
 
@@ -42,9 +43,10 @@ class LogicalDnsCluster : public ClusterImplBase {
   struct LogicalHost : public HostImpl {
     LogicalHost(ClusterInfoConstSharedPtr cluster, const std::string& hostname,
                 Network::Address::InstanceConstSharedPtr address, LogicalDnsCluster& parent)
-        : HostImpl(cluster, hostname, address, envoy::api::v2::core::Metadata::default_instance(),
-                   1, envoy::api::v2::core::Locality().default_instance(),
-                   envoy::api::v2::endpoint::Endpoint::HealthCheckConfig().default_instance()),
+        : HostImpl(cluster, hostname, address, parent.lbEndpoint().metadata(),
+                   parent.lbEndpoint().load_balancing_weight().value(),
+                   parent.localityLbEndpoint().locality(),
+                   parent.lbEndpoint().endpoint().health_check_config()),
           parent_(parent) {}
 
     // Upstream::Host
@@ -57,13 +59,22 @@ class LogicalDnsCluster : public ClusterImplBase {
 
   struct RealHostDescription : public HostDescription {
     RealHostDescription(Network::Address::InstanceConstSharedPtr address,
+                        const envoy::api::v2::endpoint::LocalityLbEndpoints& locality_lb_endpoint,
+                        const envoy::api::v2::endpoint::LbEndpoint& lb_endpoint,
                         HostConstSharedPtr logical_host)
-        : address_(address), logical_host_(logical_host) {}
+        : address_(address),
+          health_check_address_(
+              lb_endpoint.endpoint().health_check_config().port_value() == 0
+                  ? address
+                  : Network::Utility::getAddressWithPort(
+                        *address, lb_endpoint.endpoint().health_check_config().port_value())),
+          locality_lb_endpoint_(locality_lb_endpoint), lb_endpoint_(lb_endpoint),
+          logical_host_(logical_host) {}
 
     // Upstream:HostDescription
     bool canary() const override { return false; }
     const envoy::api::v2::core::Metadata& metadata() const override {
-      return envoy::api::v2::core::Metadata::default_instance();
+      return lb_endpoint_.metadata();
     }
     const ClusterInfo& cluster() const override { return logical_host_->cluster(); }
     HealthCheckHostMonitor& healthChecker() const override {
@@ -76,20 +87,33 @@ class LogicalDnsCluster : public ClusterImplBase {
     const std::string& hostname() const override { return logical_host_->hostname(); }
     Network::Address::InstanceConstSharedPtr address() const override { return address_; }
     const envoy::api::v2::core::Locality& locality() const override {
-      return envoy::api::v2::core::Locality().default_instance();
+      return locality_lb_endpoint_.locality();
     }
-    // TODO(dio): To support different address port.
     Network::Address::InstanceConstSharedPtr healthCheckAddress() const override {
-      return address_;
+      return health_check_address_;
     }
+    uint32_t priority() const { return locality_lb_endpoint_.priority(); }
     Network::Address::InstanceConstSharedPtr address_;
+    Network::Address::InstanceConstSharedPtr health_check_address_;
+    const envoy::api::v2::endpoint::LocalityLbEndpoints& locality_lb_endpoint_;
+    const envoy::api::v2::endpoint::LbEndpoint& lb_endpoint_;
     HostConstSharedPtr logical_host_;
   };
 
   struct PerThreadCurrentHostData : public ThreadLocal::ThreadLocalObject {
     Network::Address::InstanceConstSharedPtr current_resolved_address_;
   };
 
+  const envoy::api::v2::endpoint::LocalityLbEndpoints& localityLbEndpoint() const {
+    ASSERT(load_assignment_.endpoints().size() == 1);
+    return load_assignment_.endpoints()[0];
+  }
+
+  const envoy::api::v2::endpoint::LbEndpoint& lbEndpoint() const {
+    ASSERT(localityLbEndpoint().lb_endpoints().size() == 1);
+    return localityLbEndpoint().lb_endpoints()[0];
+  }
+
   void startResolve();
 
   // ClusterImplBase
@@ -105,6 +129,8 @@ class LogicalDnsCluster : public ClusterImplBase {
   Network::Address::InstanceConstSharedPtr current_resolved_address_;
   HostSharedPtr logical_host_;
   Network::ActiveDnsQuery* active_dns_query_{};
+  PriorityStateManagerPtr priority_state_manager_;
+  const envoy::api::v2::ClusterLoadAssignment load_assignment_;
 };
 
 } // namespace Upstream