Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
41 commits
Select commit Hold shift + click to select a range
8cbda27
support locality weighted loadbalancer
hzxuzhonghu Jan 2, 2019
14eb898
validate LocalityWeightSettings
hzxuzhonghu Jan 3, 2019
30ec747
set proxy locality
hzxuzhonghu Jan 3, 2019
106069e
fix ci
hzxuzhonghu Jan 3, 2019
6c0241d
address comments
hzxuzhonghu Jan 6, 2019
6f37049
EDS part
hzxuzhonghu Jan 6, 2019
6d0fa85
fix test
hzxuzhonghu Jan 6, 2019
d545d3d
reduce locality string split
hzxuzhonghu Jan 6, 2019
a9e60ca
perf optimize
hzxuzhonghu Jan 6, 2019
585cbfc
add ut
hzxuzhonghu Jan 6, 2019
484e2c3
fix lint
hzxuzhonghu Jan 6, 2019
ff01e4b
mark TODO
hzxuzhonghu Jan 6, 2019
70b0a7d
fix nil pointer
hzxuzhonghu Jan 7, 2019
a09f729
optimize ApplyLocalityWeightSetting
hzxuzhonghu Jan 15, 2019
16f3a35
per locality edsClusters cache
hzxuzhonghu Jan 16, 2019
e1dbb30
update validation ut
hzxuzhonghu Jan 16, 2019
ec604c1
update Locality field
hzxuzhonghu Jan 16, 2019
13f3220
set locality Lb priority
hzxuzhonghu Jan 16, 2019
5f86655
fix build
hzxuzhonghu Jan 16, 2019
2e7bbad
fix build
hzxuzhonghu Jan 16, 2019
caf852b
fix panic
hzxuzhonghu Jan 16, 2019
1b72d55
fix minor
hzxuzhonghu Jan 16, 2019
79ad5a7
add ut
hzxuzhonghu Jan 17, 2019
6e143b4
address comments
hzxuzhonghu Jan 17, 2019
0496af9
use cloned endpoints
hzxuzhonghu Jan 17, 2019
1c6bed8
fix subset locality lb
hzxuzhonghu Jan 17, 2019
71ebe14
shallow copy cluster to prevent r/w data race
hzxuzhonghu Jan 17, 2019
29d7cc3
fix frankbu comments
hzxuzhonghu Jan 18, 2019
985241a
update Validate
hzxuzhonghu Jan 20, 2019
d4eafd1
update locality lb set
hzxuzhonghu Jan 20, 2019
8e81897
fix ut
hzxuzhonghu Jan 20, 2019
a2ab1e0
bump istio api
hzxuzhonghu Jan 21, 2019
9cd4393
update
hzxuzhonghu Jan 21, 2019
61f6fc1
fix nil pointer
hzxuzhonghu Jan 21, 2019
a2f61f9
fix build
hzxuzhonghu Jan 21, 2019
6c23568
add comments
hzxuzhonghu Jan 21, 2019
cbb99d7
perf improve
hzxuzhonghu Jan 21, 2019
cac5215
add more comments
hzxuzhonghu Jan 22, 2019
31ffe5a
Merge branch 'release-1.1' into locality-lb
hzxuzhonghu Jan 22, 2019
16283e3
move validate localityLbsetting to validation.go
hzxuzhonghu Jan 23, 2019
c404c34
fix lint
hzxuzhonghu Jan 23, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions Gopkg.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 4 additions & 0 deletions istioctl/cmd/istioctl/kubeinject.go
Original file line number Diff line number Diff line change
Expand Up @@ -254,6 +254,10 @@ istioctl kube-inject -f deployment.yaml -o deployment-injected.yaml --injectConf
return err
}
}
err = model.ValidateMeshConfig(meshConfig)
if err != nil {
return err
}

var sidecarTemplate string

Expand Down
3 changes: 3 additions & 0 deletions mixer/test/client/gateway/gateway_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,9 @@ func (mock) ID(*core.Node) string {
func (mock) GetProxyServiceInstances(_ *model.Proxy) ([]*model.ServiceInstance, error) {
return nil, nil
}
func (mock) GetProxyLocality(_ *model.Proxy) string {
return ""
}
func (mock) GetService(_ model.Hostname) (*model.Service, error) { return nil, nil }
func (mock) InstancesByPort(_ model.Hostname, _ int, _ model.LabelsCollection) ([]*model.ServiceInstance, error) {
return nil, nil
Expand Down
3 changes: 3 additions & 0 deletions mixer/test/client/pilotplugin/pilotplugin_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -297,6 +297,9 @@ func (mock) ID(*core.Node) string {
func (mock) GetProxyServiceInstances(_ *model.Proxy) ([]*model.ServiceInstance, error) {
return nil, nil
}
func (mock) GetProxyLocality(_ *model.Proxy) string {
return ""
}
func (mock) GetService(_ model.Hostname) (*model.Service, error) { return nil, nil }
func (mock) InstancesByPort(_ model.Hostname, _ int, _ model.LabelsCollection) ([]*model.ServiceInstance, error) {
return nil, nil
Expand Down
3 changes: 3 additions & 0 deletions mixer/test/client/pilotplugin_mtls/pilotplugin_mtls_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -312,6 +312,9 @@ func (mock) ID(*core.Node) string {
func (mock) GetProxyServiceInstances(_ *model.Proxy) ([]*model.ServiceInstance, error) {
return nil, nil
}
func (mock) GetProxyLocality(_ *model.Proxy) string {
return ""
}
func (mock) GetService(_ model.Hostname) (*model.Service, error) { return nil, nil }
func (mock) InstancesByPort(_ model.Hostname, _ int, _ model.LabelsCollection) ([]*model.ServiceInstance, error) {
return nil, nil
Expand Down
3 changes: 3 additions & 0 deletions mixer/test/client/pilotplugin_tcp/pilotplugin_tcp_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,9 @@ func (mock) ID(*core.Node) string {
func (mock) GetProxyServiceInstances(_ *model.Proxy) ([]*model.ServiceInstance, error) {
return nil, nil
}
func (mock) GetProxyLocality(_ *model.Proxy) string {
return ""
}
func (mock) GetService(_ model.Hostname) (*model.Service, error) { return nil, nil }
func (mock) InstancesByPort(_ model.Hostname, _ int, _ model.LabelsCollection) ([]*model.ServiceInstance, error) {
return nil, nil
Expand Down
5 changes: 5 additions & 0 deletions pilot/pkg/bootstrap/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -410,6 +410,11 @@ func (s *Server) initMesh(args *PilotArgs) error {
}
}

if err = model.ValidateMeshConfig(mesh); err != nil {
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is too dangerous - invalid location-based config should not crash down the entire cluster ( which will happen since pilot will crash loop ).

Please report an error, increment a stat - but make sure pilot keeps running.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Isn't this a normal behavior for a system to stop bootstrap to prevent undefined error once a config is invalid?

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not in a production system. Configs need to be validate before submission, and ignored/dropped if they are invalid in prod - we don't want the entire production to go down because of a bad config.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is in the bootstrap procedure , for a reload procedure i think we can omit it and keep running use the previous config.

log.Errorf("invalid mesh configuration: %v", err)
return err
}

log.Infof("mesh configuration %s", spew.Sdump(mesh))
log.Infof("version %s", version.Info.String())
log.Infof("flags %s", spew.Sdump(args))
Expand Down
5 changes: 5 additions & 0 deletions pilot/pkg/kube/inject/webhook.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ import (

meshconfig "istio.io/api/mesh/v1alpha1"
"istio.io/istio/pilot/cmd"
"istio.io/istio/pilot/pkg/model"
"istio.io/istio/pkg/log"
)

Expand Down Expand Up @@ -87,6 +88,10 @@ func loadConfig(injectFile, meshFile string) (*Config, *meshconfig.MeshConfig, e
if err != nil {
return nil, nil, err
}
err = model.ValidateMeshConfig(meshConfig)
if err != nil {
return nil, nil, err
}

log.Infof("New configuration: sha256sum %x", sha256.Sum256(data))
log.Infof("Policy: %v", c.Policy)
Expand Down
17 changes: 17 additions & 0 deletions pilot/pkg/model/context.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"strings"
"time"

"github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
"github.com/gogo/protobuf/types"
multierror "github.com/hashicorp/go-multierror"

Expand Down Expand Up @@ -86,6 +87,9 @@ type Proxy struct {
// namespace.
ID string

// Locality is the location of where Envoy proxy runs.
Locality Locality

// DNSDomain defines the DNS domain suffix for short hostnames (e.g.
// "default.svc.cluster.local")
DNSDomain string
Expand Down Expand Up @@ -282,6 +286,19 @@ func GetProxyConfigNamespace(proxy *Proxy) string {
return ""
}

// GetProxyLocality returns the locality where Envoy proxy is running.
func GetProxyLocality(proxy *core.Node) *Locality {
if proxy == nil || proxy.Locality == nil {
return nil
}

return &Locality{
Region: proxy.Locality.Region,
Zone: proxy.Locality.Zone,
SubZone: proxy.Locality.SubZone,
}
}

const (
serviceNodeSeparator = "~"

Expand Down
61 changes: 61 additions & 0 deletions pilot/pkg/model/locality.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
// Copyright 2019 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package model

type LocalityInterface interface {
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What is this ? It's not java - we don't define interfaces for structs.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is to support passing both model.Locality and core.Locality as a param.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Then please add a comment to indicate this.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also - which core.Locality ? From envoy ?

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK, will do

GetRegion() string
GetZone() string
GetSubZone() string
}

// Identifies location of where either Envoy runs or where upstream hosts run.
type Locality struct {
// Region this proxy belongs to.
Region string
// Defines the local service zone where Envoy is running. Though optional, it
// should be set if discovery service routing is used and the discovery
// service exposes :ref:`zone data <envoy_api_field_endpoint.LocalityLbEndpoints.locality>`,
// either in this message or via :option:`--service-zone`. The meaning of zone
// is context dependent, e.g. `Availability Zone (AZ)
// <https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html>`_
// on AWS, `Zone <https://cloud.google.com/compute/docs/regions-zones/>`_ on
// GCP, etc.
Zone string
// When used for locality of upstream hosts, this field further splits zone
// into smaller chunks of sub-zones so they can be load balanced
// independently.
SubZone string
}

func (l *Locality) GetRegion() string {
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same - let's not introduce java-style in go, we don't use this pattern anywhere in istio.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ACK

if l != nil {
return l.Region
}
return ""
}

func (l *Locality) GetZone() string {
if l != nil {
return l.Zone
}
return ""
}

func (l *Locality) GetSubZone() string {
if l != nil {
return l.SubZone
}
return ""
}
11 changes: 11 additions & 0 deletions pilot/pkg/model/push_context.go
Original file line number Diff line number Diff line change
Expand Up @@ -480,6 +480,17 @@ func (ps *PushContext) GetAllSidecarScopes() map[string][]*SidecarScope {
return ps.sidecarsByNamespace
}

// ServicePort returns the port model for the given service and port.
func (ps *PushContext) ServicePort(hostname Hostname, port int) *Port {
portList := ps.ServicePort2Name[string(hostname)]
for i := range portList {
if portList[i] != nil && portList[i].Port == port {
return portList[i]
}
}
return nil
}

// DestinationRule returns a destination rule for a service name in a given domain.
func (ps *PushContext) DestinationRule(proxy *Proxy, hostname Hostname) *Config {
// If proxy has a sidecar scope that is user supplied, then get the destination rules from the sidecar scope
Expand Down
11 changes: 7 additions & 4 deletions pilot/pkg/model/service.go
Original file line number Diff line number Diff line change
Expand Up @@ -496,6 +496,9 @@ type ServiceDiscovery interface {
// determine the intended destination of a connection without a Host header on the request.
GetProxyServiceInstances(*Proxy) ([]*ServiceInstance, error)

// GetProxyLocality returns the locality where the proxy runs.
GetProxyLocality(*Proxy) string
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please remove this method - the locality is part of the ServiceInstance and node, it is not a separate call to make to the registry. We already had support for returning it ( and AFAIK it worked for Consul as well if the proper labels were set )

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, we can get from ServiceInstance, first it must be a service Instance. But in many case, the proxy is not colocated with a service instance. So can not depend on the InstancesByPort ot other method


// ManagementPorts lists set of management ports associated with an IPv4 address.
// These management ports are typically used by the platform for out of band management
// tasks such as health checks, etc. In a scenario where the proxy functions in the
Expand Down Expand Up @@ -535,12 +538,12 @@ func (h Hostname) Matches(o Hostname) bool {
return true
}

hWildcard := string(h[0]) == "*"
hWildcard := len(h) > 0 && string(h[0]) == "*"
if hWildcard && len(o) == 0 {
return true
}

oWildcard := string(o[0]) == "*"
oWildcard := len(o) > 0 && string(o[0]) == "*"
if !hWildcard && !oWildcard {
// both are non-wildcards, so do normal string comparison
return h == o
Expand Down Expand Up @@ -574,8 +577,8 @@ func (h Hostname) SubsetOf(o Hostname) bool {
return true
}

hWildcard := string(h[0]) == "*"
oWildcard := string(o[0]) == "*"
hWildcard := len(h) > 0 && string(h[0]) == "*"
oWildcard := len(o) > 0 && string(o[0]) == "*"
if !oWildcard {
if hWildcard {
return false
Expand Down
2 changes: 2 additions & 0 deletions pilot/pkg/model/service_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,8 @@ func TestHostnameMatches(t *testing.T) {
out bool
}{
{"empty", "", "", true},
{"first empty", "", "foo.com", false},
{"second empty", "foo.com", "", false},

{"non-wildcard domain",
"foo.com", "foo.com", true},
Expand Down
113 changes: 112 additions & 1 deletion pilot/pkg/model/validation.go
Original file line number Diff line number Diff line change
Expand Up @@ -824,7 +824,6 @@ func validateLoadBalancer(settings *networking.LoadBalancerSettings) (errs error
}
}
}

return
}

Expand Down Expand Up @@ -1044,6 +1043,10 @@ func ValidateMeshConfig(mesh *meshconfig.MeshConfig) (errs error) {
errs = multierror.Append(errs, err)
}

if err := validateLocalityLbSetting(mesh.LocalityLbSetting); err != nil {
errs = multierror.Append(errs, err)
}

return
}

Expand Down Expand Up @@ -2214,3 +2217,111 @@ func ValidateNetworkEndpointAddress(n *NetworkEndpoint) error {
}
return nil
}

// validateLocalityLbSetting checks the LocalityLbSetting of MeshConfig
func validateLocalityLbSetting(lb *meshconfig.LocalityLoadBalancerSetting) error {
if lb == nil {
return nil
}

if len(lb.GetDistribute()) > 0 && len(lb.GetFailover()) > 0 {
return fmt.Errorf("can not simultaneously specify 'distribute' and 'failover'")
}

srcLocalities := []string{}
for _, locality := range lb.GetDistribute() {
srcLocalities = append(srcLocalities, locality.From)
var totalWeight uint32
destLocalities := []string{}
for loc, weight := range locality.To {
destLocalities = append(destLocalities, loc)
if weight == 0 {
return fmt.Errorf("locality weight must not be in range [1, 100]")
}
totalWeight += weight
}
if totalWeight != 100 {
return fmt.Errorf("total locality weight %v != 100", totalWeight)
}
if err := validateLocalities(destLocalities); err != nil {
return err
}
}

if err := validateLocalities(srcLocalities); err != nil {
return err
}

for _, failover := range lb.GetFailover() {
if failover.From == failover.To {
return fmt.Errorf("locality lb failover settings must specify different regions")
}
if strings.Contains(failover.To, "*") {
return fmt.Errorf("locality lb failover region should not contain '*' wildcard")
}
}

return nil
}

func validateLocalities(localities []string) error {
regionZoneSubZoneMap := map[string]map[string]map[string]bool{}

for _, locality := range localities {
if n := strings.Count(locality, "*"); n > 0 {
if n > 1 || !strings.HasSuffix(locality, "*") {
return fmt.Errorf("locality %s wildcard '*' number can not exceed 1 and must be in the end", locality)
}
}

items := strings.SplitN(locality, "/", 3)
for _, item := range items {
if item == "" {
return fmt.Errorf("locality %s must not contain empty region/zone/subzone info", locality)
}
}
if _, ok := regionZoneSubZoneMap["*"]; ok {
return fmt.Errorf("locality %s overlap with previous specified ones", locality)
}
switch len(items) {
case 1:
if _, ok := regionZoneSubZoneMap[items[0]]; ok {
return fmt.Errorf("locality %s overlap with previous specified ones", locality)
}
regionZoneSubZoneMap[items[0]] = map[string]map[string]bool{"*": {"*": true}}
case 2:
if _, ok := regionZoneSubZoneMap[items[0]]; ok {
if _, ok := regionZoneSubZoneMap[items[0]]["*"]; ok {
return fmt.Errorf("locality %s overlap with previous specified ones", locality)
}
if _, ok := regionZoneSubZoneMap[items[0]][items[1]]; ok {
return fmt.Errorf("locality %s overlap with previous specified ones", locality)
}
regionZoneSubZoneMap[items[0]][items[1]] = map[string]bool{"*": true}
} else {
regionZoneSubZoneMap[items[0]] = map[string]map[string]bool{items[1]: {"*": true}}
}
case 3:
if _, ok := regionZoneSubZoneMap[items[0]]; ok {
if _, ok := regionZoneSubZoneMap[items[0]]["*"]; ok {
return fmt.Errorf("locality %s overlap with previous specified ones", locality)
}
if _, ok := regionZoneSubZoneMap[items[0]][items[1]]; ok {
if regionZoneSubZoneMap[items[0]][items[1]]["*"] {
return fmt.Errorf("locality %s overlap with previous specified ones", locality)
}
if regionZoneSubZoneMap[items[0]][items[1]][items[2]] {
return fmt.Errorf("locality %s overlap with previous specified ones", locality)
}
regionZoneSubZoneMap[items[0]][items[1]][items[2]] = true
} else {
regionZoneSubZoneMap[items[0]][items[1]] = map[string]bool{items[2]: true}
}
} else {
regionZoneSubZoneMap[items[0]] = map[string]map[string]bool{items[1]: {items[2]: true}}
}
}
}

return nil
}
Loading