Skip to content

Commit 06447e3

Browse files
danehansdanehans
committed
Adds Support for Bootstrap Config
Signed-off-by: danehans <[email protected]>
1 parent 2be093c commit 06447e3

File tree

2 files changed

+134
-19
lines changed

2 files changed

+134
-19
lines changed

‎internal/infrastructure/kubernetes/deployment.go‎

Lines changed: 112 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
package kubernetes
22

33
import (
4+
"bytes"
45
"context"
56
"fmt"
67
"path/filepath"
8+
"text/template"
79

810
appsv1 "k8s.io/api/apps/v1"
911
corev1 "k8s.io/api/core/v1"
@@ -27,13 +29,102 @@ const (
2729
// envoyCfgVolMntDir is the directory name of the Envoy configuration volume.
2830
envoyCfgVolMntDir = "config"
2931
// envoyCfgFileName is the name of the Envoy configuration file.
30-
envoyCfgFileName = "envoy.json"
32+
envoyCfgFileName = "bootstrap.yaml"
3133
// envoyHTTPPort is the container port number of Envoy's HTTP endpoint.
3234
envoyHTTPPort = int32(8080)
3335
// envoyHTTPSPort is the container port number of Envoy's HTTPS endpoint.
3436
envoyHTTPSPort = int32(8443)
3537
)
3638

39+
var bootstrapTmpl = template.Must(template.New(envoyCfgFileName).Parse(`
40+
admin:
41+
access_log_path: /dev/null
42+
address:
43+
socket_address:
44+
address: 127.0.0.1
45+
port_value: 19000
46+
dynamic_resources:
47+
cds_config:
48+
resource_api_version: V3
49+
api_config_source:
50+
api_type: GRPC
51+
transport_api_version: V3
52+
grpc_services:
53+
- envoy_grpc:
54+
cluster_name: xds_cluster
55+
set_node_on_first_message_only: true
56+
lds_config:
57+
resource_api_version: V3
58+
api_config_source:
59+
api_type: GRPC
60+
transport_api_version: V3
61+
grpc_services:
62+
- envoy_grpc:
63+
cluster_name: xds_cluster
64+
set_node_on_first_message_only: true
65+
node:
66+
cluster: envoy-gateway-system
67+
id: envoy-default
68+
static_resources:
69+
clusters:
70+
- connect_timeout: 1s
71+
load_assignment:
72+
cluster_name: xds_cluster
73+
endpoints:
74+
- lb_endpoints:
75+
- endpoint:
76+
address:
77+
socket_address:
78+
address: {{ .XdsServerAddress }}
79+
port_value: 18000
80+
http2_protocol_options: {}
81+
name: xds_cluster
82+
type: STRICT_DNS
83+
layered_runtime:
84+
layers:
85+
- name: runtime-0
86+
rtds_layer:
87+
rtds_config:
88+
resource_api_version: V3
89+
api_config_source:
90+
transport_api_version: V3
91+
api_type: GRPC
92+
grpc_services:
93+
envoy_grpc:
94+
cluster_name: xds_cluster
95+
name: runtime-0
96+
`))
97+
98+
var (
99+
// envoyGatewayService is the name of the Envoy Gateway service.
100+
envoyGatewayService = "envoy-gateway"
101+
)
102+
103+
// envoyBootstrap defines the envoy Bootstrap configuration.
104+
type bootstrapConfig struct {
105+
// parameters defines configurable bootstrap configuration parameters.
106+
parameters bootstrapParameters
107+
// rendered is the rendered bootstrap configuration.
108+
rendered string
109+
}
110+
111+
// envoyBootstrap defines the envoy Bootstrap configuration.
112+
type bootstrapParameters struct {
113+
// XdsServerAddress is the address of the XDS Server that Envoy is managed by.
114+
XdsServerAddress string
115+
}
116+
117+
// render the stringified bootstrap config in yaml format.
118+
func (b *bootstrapConfig) render() error {
119+
buf := new(bytes.Buffer)
120+
if err := bootstrapTmpl.Execute(buf, b.parameters); err != nil {
121+
return fmt.Errorf("failed to render bootstrap config: %v", err)
122+
}
123+
b.rendered = buf.String()
124+
125+
return nil
126+
}
127+
37128
// createDeploymentIfNeeded creates a Deployment based on the provided infra, if
38129
// it doesn't exist in the kube api server.
39130
func (i *Infra) createDeploymentIfNeeded(ctx context.Context, infra *ir.Infra) error {
@@ -76,8 +167,11 @@ func (i *Infra) getDeployment(ctx context.Context, infra *ir.Infra) (*appsv1.Dep
76167
}
77168

78169
// expectedDeployment returns the expected Deployment based on the provided infra.
79-
func (i *Infra) expectedDeployment(infra *ir.Infra) *appsv1.Deployment {
80-
containers := expectedContainers(infra)
170+
func (i *Infra) expectedDeployment(infra *ir.Infra) (*appsv1.Deployment, error) {
171+
containers, err := expectedContainers(infra)
172+
if err != nil {
173+
return nil, err
174+
}
81175

82176
deployment := &appsv1.Deployment{
83177
TypeMeta: metav1.TypeMeta{
@@ -97,15 +191,7 @@ func (i *Infra) expectedDeployment(infra *ir.Infra) *appsv1.Deployment {
97191
Labels: EnvoyPodSelector().MatchLabels,
98192
},
99193
Spec: corev1.PodSpec{
100-
Containers: containers,
101-
Volumes: []corev1.Volume{
102-
{
103-
Name: envoyCfgVolName,
104-
VolumeSource: corev1.VolumeSource{
105-
EmptyDir: &corev1.EmptyDirVolumeSource{},
106-
},
107-
},
108-
},
194+
Containers: containers,
109195
ServiceAccountName: infra.Proxy.ObjectName(),
110196
AutomountServiceAccountToken: pointer.BoolPtr(false),
111197
TerminationGracePeriodSeconds: pointer.Int64Ptr(int64(300)),
@@ -117,10 +203,10 @@ func (i *Infra) expectedDeployment(infra *ir.Infra) *appsv1.Deployment {
117203
},
118204
}
119205

120-
return deployment
206+
return deployment, nil
121207
}
122208

123-
func expectedContainers(infra *ir.Infra) []corev1.Container {
209+
func expectedContainers(infra *ir.Infra) ([]corev1.Container, error) {
124210
ports := []corev1.ContainerPort{
125211
{
126212
Name: "http",
@@ -134,6 +220,11 @@ func expectedContainers(infra *ir.Infra) []corev1.Container {
134220
},
135221
}
136222

223+
cfg := bootstrapConfig{parameters: bootstrapParameters{XdsServerAddress: envoyGatewayService}}
224+
if err := cfg.render(); err != nil {
225+
return nil, err
226+
}
227+
137228
containers := []corev1.Container{
138229
{
139230
Name: envoyContainerName,
@@ -147,6 +238,7 @@ func expectedContainers(infra *ir.Infra) []corev1.Container {
147238
filepath.Join("/", envoyCfgVolMntDir, envoyCfgFileName),
148239
fmt.Sprintf("--service-cluster $(%s)", envoyNsEnvVar),
149240
fmt.Sprintf("--service-node $(%s)", envoyPodEnvVar),
241+
fmt.Sprintf("--config-yaml %s", cfg.rendered),
150242
"--log-level info",
151243
},
152244
Env: []corev1.EnvVar{
@@ -182,15 +274,18 @@ func expectedContainers(infra *ir.Infra) []corev1.Container {
182274
},
183275
}
184276

185-
return containers
277+
return containers, nil
186278
}
187279

188280
// createDeployment creates a Deployment in the kube api server based on the provided
189281
// infra, if it doesn't exist.
190282
func (i *Infra) createDeployment(ctx context.Context, infra *ir.Infra) (*appsv1.Deployment, error) {
191-
expected := i.expectedDeployment(infra)
192-
err := i.Client.Create(ctx, expected)
283+
expected, err := i.expectedDeployment(infra)
193284
if err != nil {
285+
return nil, err
286+
}
287+
288+
if err := i.Client.Create(ctx, expected); err != nil {
194289
if kerrors.IsAlreadyExists(err) {
195290
return expected, nil
196291
}

‎internal/infrastructure/kubernetes/deployment_test.go‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package kubernetes
22

33
import (
44
"context"
5+
"fmt"
56
"testing"
67

78
"github.com/stretchr/testify/require"
@@ -52,6 +53,17 @@ func checkContainer(t *testing.T, deploy *appsv1.Deployment, name string, expect
5253
return nil
5354
}
5455

56+
func checkContainerHasArg(t *testing.T, container *corev1.Container, arg string) {
57+
t.Helper()
58+
59+
for _, a := range container.Args {
60+
if a == arg {
61+
return
62+
}
63+
}
64+
t.Errorf("container is missing argument %q", arg)
65+
}
66+
5567
func checkLabels(t *testing.T, deploy *appsv1.Deployment, expected map[string]string) {
5668
t.Helper()
5769

@@ -88,7 +100,8 @@ func TestExpectedDeployment(t *testing.T) {
88100
cli := fakeclient.NewClientBuilder().WithScheme(envoygateway.GetScheme()).WithObjects().Build()
89101
kube := NewInfra(cli)
90102
infra := ir.NewInfra()
91-
deploy := kube.expectedDeployment(infra)
103+
deploy, err := kube.expectedDeployment(infra)
104+
require.NoError(t, err)
92105

93106
// Check container details, i.e. env vars, labels, etc. for the deployment are as expected.
94107
container := checkContainer(t, deploy, envoyContainerName, true)
@@ -97,6 +110,12 @@ func TestExpectedDeployment(t *testing.T) {
97110
checkEnvVar(t, deploy, envoyContainerName, envoyPodEnvVar)
98111
checkLabels(t, deploy, deploy.Labels)
99112

113+
// Create a bootstrap config, render it into an arg, and ensure it's as expected.
114+
cfg := &bootstrapConfig{parameters: bootstrapParameters{XdsServerAddress: envoyGatewayService}}
115+
err = cfg.render()
116+
require.NoError(t, err)
117+
checkContainerHasArg(t, container, fmt.Sprintf("--config-yaml %s", cfg.rendered))
118+
100119
// Check container ports for the deployment are as expected.
101120
ports := []int32{envoyHTTPPort, envoyHTTPSPort}
102121
for _, port := range ports {
@@ -107,7 +126,8 @@ func TestExpectedDeployment(t *testing.T) {
107126
func TestCreateDeploymentIfNeeded(t *testing.T) {
108127
kube := NewInfra(nil)
109128
infra := ir.NewInfra()
110-
deploy := kube.expectedDeployment(infra)
129+
deploy, err := kube.expectedDeployment(infra)
130+
require.NoError(t, err)
111131
deploy.ResourceVersion = "1"
112132

113133
testCases := []struct {

0 commit comments

Comments
 (0)