envoyproxy · arkodg · Feb 27, 2024 · Feb 16, 2024 · Feb 17, 2024 · Feb 19, 2024
@@ -12,6 +12,7 @@ import (
 
 	autoscalingv2 "k8s.io/api/autoscaling/v2"
 	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/utils/ptr"
 )
 
@@ -120,3 +121,13 @@ func (logging *ProxyLogging) GetEnvoyProxyComponentLevel() string {
 
 	return strings.Join(args, ",")
 }
+
+// DefaultShutdownManagerContainerResourceRequirements returns a new ResourceRequirements with default settings.
+func DefaultShutdownManagerContainerResourceRequirements() *v1.ResourceRequirements {
+	return &v1.ResourceRequirements{
+		Requests: v1.ResourceList{
+			v1.ResourceCPU:    resource.MustParse(DefaultShutdownManagerCPUResourceRequests),
+			v1.ResourceMemory: resource.MustParse(DefaultShutdownManagerMemoryResourceRequests),
+		},
+	}
+}
@@ -81,6 +81,11 @@ type EnvoyProxySpec struct {
 	//
 	// +optional
 	MergeGateways *bool `json:"mergeGateways,omitempty"`
+
+	// Shutdown defines configuration for graceful envoy shutdown process.
+	//
+	// +optional
+	Shutdown *ShutdownConfig `json:"shutdown,omitempty"`
 }
 
 type ProxyTelemetry struct {
@@ -115,6 +120,20 @@ type EnvoyProxyProvider struct {
 	Kubernetes *EnvoyProxyKubernetesProvider `json:"kubernetes,omitempty"`
 }
 
+// ShutdownConfig defines configuration for graceful envoy shutdown process.
+type ShutdownConfig struct {
+	// DrainTimeout defines the graceful drain timeout. This should be less than the pod's terminationGracePeriodSeconds.
+	// If unspecified, defaults to 600 seconds.
+	//
+	// +optional
+	DrainTimeout *metav1.Duration `json:"drainTimeout,omitempty"`
+	// MinDrainDuration defines the minimum drain duration allowing time for endpoint deprogramming to complete.
+	// If unspecified, defaults to 5 seconds.
+	//
+	// +optional
+	MinDrainDuration *metav1.Duration `json:"minDrainDuration,omitempty"`
+}
+
 // EnvoyProxyKubernetesProvider defines configuration for the Kubernetes resource
 // provider.
 type EnvoyProxyKubernetesProvider struct {

@@ -21,6 +21,12 @@ const (
 	DefaultDeploymentMemoryResourceRequests = "512Mi"
 	// DefaultEnvoyProxyImage is the default image used by envoyproxy
 	DefaultEnvoyProxyImage = "envoyproxy/envoy:distroless-dev"
+	// DefaultShutdownManagerCPUResourceRequests for shutdown manager cpu resource
+	DefaultShutdownManagerCPUResourceRequests = "10m"
+	// DefaultShutdownManagerMemoryResourceRequests for shutdown manager memory resource
+	DefaultShutdownManagerMemoryResourceRequests = "32Mi"
+	// DefaultShutdownManagerImage is the default image used for the shutdown manager.
+	DefaultShutdownManagerImage = "envoyproxy/gateway-dev:latest"
 	// DefaultRateLimitImage is the default image used by ratelimit.
 	DefaultRateLimitImage = "envoyproxy/ratelimit:master"
 	// HTTPProtocol is the common-used http protocol.

@@ -6631,6 +6631,21 @@ spec:
                 required:
                 - type
                 type: object
+              shutdown:
+                description: Shutdown defines configuration for graceful envoy shutdown
+                  process.
+                properties:
+                  drainTimeout:
+                    description: DrainTimeout defines the graceful drain timeout.
+                      This should be less than the pod's terminationGracePeriodSeconds.
+                      If unspecified, defaults to 600 seconds.
+                    type: string
+                  minDrainDuration:
+                    description: MinDrainDuration defines the minimum drain duration
+                      allowing time for endpoint deprogramming to complete. If unspecified,
+                      defaults to 5 seconds.
+                    type: string
+                type: object
               telemetry:
                 description: Telemetry defines telemetry parameters for managed proxies.
                 properties:

@@ -32,6 +32,7 @@ require (
 	go.opentelemetry.io/proto/otlp v1.1.0
 	go.uber.org/zap v1.26.0
 	golang.org/x/exp v0.0.0-20231006140011-7918f672742d
+	golang.org/x/sys v0.17.0
 	google.golang.org/grpc v1.61.1
 	google.golang.org/protobuf v1.32.0
 	gopkg.in/yaml.v3 v3.0.1
@@ -109,7 +110,6 @@ require (
 	golang.org/x/net v0.20.0 // indirect
 	golang.org/x/oauth2 v0.16.0 // indirect
 	golang.org/x/sync v0.6.0 // indirect
-	golang.org/x/sys v0.17.0 // indirect
 	golang.org/x/term v0.16.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.3.0 // indirect

@@ -815,6 +815,7 @@ xds:
                   serverHeaderTransformation: PASS_THROUGH
                   statPrefix: http
                   useRemoteAddress: true
+            drainType: MODIFY_ONLY
             name: default/eg/http
             perConnectionBufferLimitBytes: 32768
       - activeState:
@@ -881,6 +882,7 @@ xds:
                   serverHeaderTransformation: PASS_THROUGH
                   statPrefix: http
                   useRemoteAddress: true
+            drainType: MODIFY_ONLY
             name: default/eg/grpc
             perConnectionBufferLimitBytes: 32768
       - activeState:
@@ -903,6 +905,7 @@ xds:
               socketAddress:
                 address: 0.0.0.0
                 portValue: 8443
+            drainType: MODIFY_ONLY
             filterChains:
             - filterChainMatch:
                 serverNames:
@@ -948,6 +951,7 @@ xds:
               socketAddress:
                 address: 0.0.0.0
                 portValue: 1234
+            drainType: MODIFY_ONLY
             filterChains:
             - filters:
               - name: envoy.filters.network.tcp_proxy

@@ -211,6 +211,7 @@ xds:
                   serverHeaderTransformation: PASS_THROUGH
                   statPrefix: http
                   useRemoteAddress: true
+            drainType: MODIFY_ONLY
             name: default/eg/http
             perConnectionBufferLimitBytes: 32768
     - '@type': type.googleapis.com/envoy.admin.v3.RoutesConfigDump

@@ -575,6 +575,7 @@
                       }
                     ]
                   },
+                  "drainType": "MODIFY_ONLY",
                   "name": "default/eg/http",
                   "perConnectionBufferLimitBytes": 32768
                 }
@@ -679,6 +680,7 @@
                       }
                     ]
                   },
+                  "drainType": "MODIFY_ONLY",
                   "name": "default/eg/grpc",
                   "perConnectionBufferLimitBytes": 32768
                 }
@@ -715,6 +717,7 @@
                       "portValue": 8443
                     }
                   },
+                  "drainType": "MODIFY_ONLY",
                   "filterChains": [
                     {
                       "filterChainMatch": {
@@ -792,6 +795,7 @@
                       "portValue": 1234
                     }
                   },
+                  "drainType": "MODIFY_ONLY",
                   "filterChains": [
                     {
                       "filters": [

@@ -341,6 +341,7 @@ xds:
                   serverHeaderTransformation: PASS_THROUGH
                   statPrefix: http
                   useRemoteAddress: true
+            drainType: MODIFY_ONLY
             name: default/eg/http
             perConnectionBufferLimitBytes: 32768
       - activeState:
@@ -407,6 +408,7 @@ xds:
                   serverHeaderTransformation: PASS_THROUGH
                   statPrefix: http
                   useRemoteAddress: true
+            drainType: MODIFY_ONLY
             name: default/eg/grpc
             perConnectionBufferLimitBytes: 32768
       - activeState:
@@ -429,6 +431,7 @@ xds:
               socketAddress:
                 address: 0.0.0.0
                 portValue: 8443
+            drainType: MODIFY_ONLY
             filterChains:
             - filterChainMatch:
                 serverNames:
@@ -474,6 +477,7 @@ xds:
               socketAddress:
                 address: 0.0.0.0
                 portValue: 1234
+            drainType: MODIFY_ONLY
             filterChains:
             - filters:
               - name: envoy.filters.network.tcp_proxy

@@ -58,6 +58,7 @@ xds:
                 serverHeaderTransformation: PASS_THROUGH
                 statPrefix: http
                 useRemoteAddress: true
+          drainType: MODIFY_ONLY
           name: default/eg/http
           perConnectionBufferLimitBytes: 32768
     - activeState:
@@ -124,6 +125,7 @@ xds:
                 serverHeaderTransformation: PASS_THROUGH
                 statPrefix: http
                 useRemoteAddress: true
+          drainType: MODIFY_ONLY
           name: default/eg/grpc
           perConnectionBufferLimitBytes: 32768
     - activeState:
@@ -146,6 +148,7 @@ xds:
             socketAddress:
               address: 0.0.0.0
               portValue: 8443
+          drainType: MODIFY_ONLY
           filterChains:
           - filterChainMatch:
               serverNames:
@@ -191,6 +194,7 @@ xds:
             socketAddress:
               address: 0.0.0.0
               portValue: 1234
+          drainType: MODIFY_ONLY
           filterChains:
           - filters:
             - name: envoy.filters.network.tcp_proxy

@@ -452,6 +452,7 @@
                       }
                     ]
                   },
+                  "drainType": "MODIFY_ONLY",
                   "name": "envoy-gateway-system/eg/http",
                   "perConnectionBufferLimitBytes": 32768
                 }

@@ -268,6 +268,7 @@ xds:
                   serverHeaderTransformation: PASS_THROUGH
                   statPrefix: http
                   useRemoteAddress: true
+            drainType: MODIFY_ONLY
             name: envoy-gateway-system/eg/http
             perConnectionBufferLimitBytes: 32768
     - '@type': type.googleapis.com/envoy.admin.v3.RoutesConfigDump

@@ -75,5 +75,6 @@ xds:
                 serverHeaderTransformation: PASS_THROUGH
                 statPrefix: http
                 useRemoteAddress: true
+          drainType: MODIFY_ONLY
           name: envoy-gateway-system/eg/http
           perConnectionBufferLimitBytes: 32768
@@ -0,0 +1,71 @@
+// Copyright Envoy Gateway Authors
+// SPDX-License-Identifier: Apache-2.0
+// The full text of the Apache license is available in the LICENSE file at
+// the root of the repo.
+
+package cmd
+
+import (
+	"time"
+
+	"github.com/spf13/cobra"
+
+	"github.com/envoyproxy/gateway/internal/cmd/envoy"
+)
+
+// getEnvoyCommand returns the envoy cobra command to be executed.
+func getEnvoyCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "envoy",
+		Short: "Envoy proxy management",
+	}
+
+	cmd.AddCommand(getShutdownCommand())
+	cmd.AddCommand(getShutdownManagerCommand())
+
+	return cmd
+}
+
+// getShutdownCommand returns the shutdown cobra command to be executed.
+func getShutdownCommand() *cobra.Command {
+	var drainTimeout time.Duration
+	var minDrainDuration time.Duration
+	var exitAtConnections int
+
+	cmd := &cobra.Command{
+		Use:   "shutdown",
+		Short: "Gracefully drain open connections prior to pod shutdown.",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return envoy.Shutdown(drainTimeout, minDrainDuration, exitAtConnections)
+		},
+	}
+
+	cmd.PersistentFlags().DurationVar(&drainTimeout, "drain-timeout", 600*time.Second,
+		"Graceful shutdown timeout. This should be less than the pod's terminationGracePeriodSeconds.")
+
+	cmd.PersistentFlags().DurationVar(&minDrainDuration, "min-drain-duration", 5*time.Second,
+		"Minimum drain duration allowing time for endpoint deprogramming to complete.")
+
+	cmd.PersistentFlags().IntVar(&exitAtConnections, "exit-at-connections", 0,
+		"Number of connections to wait for when monitoring Envoy listener drain process.")
+
+	return cmd
+}
+
+// getShutdownManagerCommand returns the shutdown manager cobra command to be executed.
+func getShutdownManagerCommand() *cobra.Command {
+	var readyTimeout time.Duration
+
+	cmd := &cobra.Command{
+		Use:   "shutdown-manager",
+		Short: "Provides HTTP endpoint used in preStop hook to block until ready for pod shutdown.",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return envoy.ShutdownManager(readyTimeout)
+		},
+	}
+
+	cmd.PersistentFlags().DurationVar(&readyTimeout, "ready-timeout", 610*time.Second,
+		"Shutdown ready timeout. This should be greater than shutdown's drain-timeout and less than the pod's terminationGracePeriodSeconds.")
+
+	return cmd
+}
-Original file line number
+Diff line change
@@ Expand Up / @@ -452,6 +452,7 @@ @@
                           }
                         ]
                       },
+                      "drainType": "MODIFY_ONLY",
                       "name": "envoy-gateway-system/eg/http",
                       "perConnectionBufferLimitBytes": 32768
                     }
@@ Expand Down @@