Skip to content

Commit 6efa0d6

Browse files
author
Jianfei Hu
committed
WIP working on modifying sidecar.Args first, then modify app container patch.
1 parent 65a2194 commit 6efa0d6

File tree

3 files changed

+145
-18
lines changed

3 files changed

+145
-18
lines changed

‎pilot/cmd/pilot-agent/status/server.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,13 @@ func NewServer(config Config) (*Server, error) {
9696
return s, nil
9797
}
9898

99+
// FormatProberURL returns a pair of HTTP URLs that pilot agent will serve to take over Kubernetes
100+
// app probers.
101+
func FormatProberURL(container string) (string, string) {
102+
return fmt.Sprintf("/app-health/%v/readyz", container),
103+
fmt.Sprintf("/app-health/%v/livez", container)
104+
}
105+
99106
// Run opens a the status port and begins accepting probes.
100107
func (s *Server) Run(ctx context.Context) {
101108
log.Infof("Opening status port %d\n", s.statusPort)

‎pilot/pkg/kube/inject/app_probe.go‎

Lines changed: 133 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -80,32 +80,59 @@ func extractStatusPort(sidecar *corev1.Container) int {
8080
return -1
8181
}
8282

83-
// calculateRewrite returns a copy of the original and the expected HTTPGetProber config after injection.
83+
// calculateRewrite returns a pointer to the HTTPGetAction.
8484
func calculateRewrite(probe *corev1.Probe, newURL string,
85-
statusPort int, portMap map[string]int32) (original, after *corev1.HTTPGetAction) {
85+
statusPort int, portMap map[string]int32, kubeProbers *status.KubeAppProbers) *corev1.HTTPGetAction {
8686
if probe == nil || probe.HTTPGet == nil {
87-
return nil, nil
87+
return nil
88+
}
89+
copyProber := func() *corev1.HTTPGetAction {
90+
c := proto.Clone(probe.HTTPGet).(*corev1.HTTPGetAction)
91+
c.Port = probe.HTTPGet.Port
92+
return c
93+
}
94+
original := copyProber()
95+
after := copyProber()
96+
// A named port, resolve by looking at port map.
97+
if original.Port.Type == intstr.String {
98+
port, exists := portMap[original.Port.StrVal]
99+
if !exists {
100+
log.Errorf("named port not found in the map skip rewriting probing %v", *probe)
101+
return after
102+
}
103+
original.Port = intstr.FromInt(int(port))
104+
}
105+
(*kubeProbers)[newURL] = original
106+
// Change the application container prober config.
107+
after.Port = intstr.FromInt(statusPort)
108+
after.Path = newURL
109+
return after
110+
}
111+
112+
func calculateRewrite2(probe *corev1.Probe, newURL string, statusPort int, portMap map[string]int32) *corev1.HTTPGetAction {
113+
if probe == nil || probe.HTTPGet == nil {
114+
return nil
88115
}
89116
copyProber := func() *corev1.HTTPGetAction {
90117
c := proto.Clone(probe.HTTPGet).(*corev1.HTTPGetAction)
91118
c.Port = probe.HTTPGet.Port
92119
return c
93120
}
94-
original = copyProber()
95-
after = copyProber()
121+
original := copyProber()
122+
after := copyProber()
96123
// A named port, resolve by looking at port map.
97124
if original.Port.Type == intstr.String {
98125
port, exists := portMap[original.Port.StrVal]
99126
if !exists {
100127
log.Errorf("named port not found in the map skip rewriting probing %v", *probe)
101-
return nil, nil
128+
return after
102129
}
103130
original.Port = intstr.FromInt(int(port))
104131
}
105132
// Change the application container prober config.
106133
after.Port = intstr.FromInt(statusPort)
107134
after.Path = newURL
108-
return original, after
135+
return after
109136
}
110137

111138
// createProbeRewritePatch generates the patch for webhook.
@@ -117,7 +144,7 @@ func createProbeRewritePatch(podSpec *corev1.PodSpec, spec *SidecarInjectionSpec
117144
return patch
118145
}
119146
var sidecar *corev1.Container
120-
for i := range podSpec.Containers {
147+
for i := range spec.Containers {
121148
if podSpec.Containers[i].Name == istioProxyContainerName {
122149
sidecar = &podSpec.Containers[i]
123150
break
@@ -132,6 +159,7 @@ func createProbeRewritePatch(podSpec *corev1.PodSpec, spec *SidecarInjectionSpec
132159
if statusPort == -1 {
133160
return nil
134161
}
162+
kubeProbers := &status.KubeAppProbers{}
135163
for i, c := range podSpec.Containers {
136164
// Skip sidecar container.
137165
if c.Name == istioProxyContainerName {
@@ -141,16 +169,16 @@ func createProbeRewritePatch(podSpec *corev1.PodSpec, spec *SidecarInjectionSpec
141169
for _, p := range c.Ports {
142170
portMap[p.Name] = p.ContainerPort
143171
}
144-
if _, after := calculateRewrite(c.ReadinessProbe, fmt.Sprintf("/app-health/%v/readyz", c.Name),
145-
statusPort, portMap); after != nil {
172+
if after := calculateRewrite(c.ReadinessProbe, fmt.Sprintf("/app-health/%v/readyz", c.Name),
173+
statusPort, portMap, kubeProbers); after != nil {
146174
patch = append(patch, rfc6902PatchOperation{
147175
Op: "replace",
148176
Path: fmt.Sprintf("/spec/containers/%v/readinessProbe/httpGet", i),
149177
Value: *after,
150178
})
151179
}
152-
if _, after := calculateRewrite(c.LivenessProbe, fmt.Sprintf("/app-health/%v/livez", c.Name),
153-
statusPort, portMap); after != nil {
180+
if after := calculateRewrite(c.LivenessProbe, fmt.Sprintf("/app-health/%v/livez", c.Name),
181+
statusPort, portMap, kubeProbers); after != nil {
154182
patch = append(patch, rfc6902PatchOperation{
155183
Op: "replace",
156184
Path: fmt.Sprintf("/spec/containers/%v/livenessProbe/httpGet", i),
@@ -159,11 +187,38 @@ func createProbeRewritePatch(podSpec *corev1.PodSpec, spec *SidecarInjectionSpec
159187
}
160188
}
161189

162-
// TODO: also here, need to add a patch to update the command line flags.
190+
// TODO: add a patch to add a args flag, [--kubeAppHTTPProbers=xxx] kubeProbers.
191+
// patch = append(patch, rfc6902PatchOperation{
192+
// Op: "append",
193+
// Path: fmt.Sprintf("/spec/containers/%v/args/-"),
194+
// Value: *kubeProbers,
195+
// })
163196
return patch
164197
}
165198

166-
func rewriteAppHTTPProbe(podSpec *corev1.PodSpec, spec *SidecarInjectionSpec) {
199+
// extractKubeAppProbers returns a pointer to the KubeAppProbers.
200+
// Also update the probers so that all the references to the named port will be resolved to integer.
201+
// TODO: reasoning as following
202+
// - we need to update istio-proxy container args.
203+
// - webhook we append that before we call calculate rewrites.
204+
func extractKubeAppProbers(podspec *corev1.PodSpec) *status.KubeAppProbers {
205+
probers := status.KubeAppProbers{}
206+
for _, c := range podspec.Containers {
207+
if c.ReadinessProbe == nil || c.ReadinessProbe.HTTPGet == nil {
208+
continue
209+
}
210+
// portMap := map[string]int{}
211+
ha := proto.Clone(c.ReadinessProbe.HTTPGet).(*corev1.HTTPGetAction)
212+
// TODO here.
213+
if c.ReadinessProbe.HTTPGet.Port.Type == intstr.String {
214+
}
215+
ha.Port = c.ReadinessProbe.HTTPGet.Port
216+
probers[fmt.Sprintf("/app-health/%v/readyz", c.Name)] = ha
217+
}
218+
return &probers
219+
}
220+
221+
func rewriteAppHTTPProbe2(podSpec *corev1.PodSpec, spec *SidecarInjectionSpec) {
167222
if spec == nil || podSpec == nil {
168223
return
169224
}
@@ -198,15 +253,13 @@ func rewriteAppHTTPProbe(podSpec *corev1.PodSpec, spec *SidecarInjectionSpec) {
198253
portMap[p.Name] = p.ContainerPort
199254
}
200255
newURL := fmt.Sprintf("/app-health/%v/readyz", c.Name)
201-
before, after := calculateRewrite(c.ReadinessProbe, newURL, statusPort, portMap)
256+
after := calculateRewrite(c.ReadinessProbe, newURL, statusPort, portMap, &appProberInfo)
202257
if after != nil {
203-
appProberInfo[newURL] = before
204258
*c.ReadinessProbe.HTTPGet = *after
205259
}
206260
newURL = fmt.Sprintf("/app-health/%v/livez", c.Name)
207-
before, after = calculateRewrite(c.LivenessProbe, newURL, statusPort, portMap)
261+
after = calculateRewrite(c.LivenessProbe, newURL, statusPort, portMap, &appProberInfo)
208262
if after != nil {
209-
appProberInfo[newURL] = before
210263
*c.LivenessProbe.HTTPGet = *after
211264
}
212265
}
@@ -220,3 +273,65 @@ func rewriteAppHTTPProbe(podSpec *corev1.PodSpec, spec *SidecarInjectionSpec) {
220273
// We don't have to escape json encoding here when using golang libraries.
221274
sidecar.Args = append(sidecar.Args, []string{fmt.Sprintf("--%v", status.KubeAppProberCmdFlagName), string(b)}...)
222275
}
276+
277+
func rewriteAppHTTPProbe(podSpec *corev1.PodSpec, spec *SidecarInjectionSpec) {
278+
if spec == nil || podSpec == nil {
279+
return
280+
}
281+
if !spec.RewriteAppHTTPProbe {
282+
return
283+
}
284+
var sidecar *corev1.Container
285+
for i := range podSpec.Containers {
286+
if podSpec.Containers[i].Name == istioProxyContainerName {
287+
sidecar = &podSpec.Containers[i]
288+
break
289+
}
290+
}
291+
if sidecar == nil {
292+
return
293+
}
294+
295+
statusPort := extractStatusPort(sidecar)
296+
// Pilot agent statusPort is not defined, skip changing application http probe.
297+
if statusPort == -1 {
298+
return
299+
}
300+
301+
appProberInfo := extractKubeAppProbers(podSpec)
302+
// Nothing to rewrite.
303+
if appProberInfo == nil {
304+
return
305+
}
306+
// Finally propagate app prober config to `istio-proxy` through command line flag.
307+
b, err := json.Marshal(appProberInfo)
308+
if err != nil {
309+
log.Errorf("failed to serialize the app prober config %v", err)
310+
return
311+
}
312+
// We don't have to escape json encoding here when using golang libraries.
313+
sidecar.Args = append(sidecar.Args, []string{fmt.Sprintf("--%v", status.KubeAppProberCmdFlagName), string(b)}...)
314+
315+
// Now time to modify the container probers.
316+
for _, c := range podSpec.Containers {
317+
// Skip sidecar container.
318+
if c.Name == istioProxyContainerName {
319+
continue
320+
}
321+
portMap := map[string]int32{}
322+
for _, p := range c.Ports {
323+
portMap[p.Name] = p.ContainerPort
324+
}
325+
newURL := fmt.Sprintf("/app-health/%v/readyz", c.Name)
326+
after := calculateRewrite2(c.ReadinessProbe, newURL, statusPort, portMap)
327+
if after != nil {
328+
*c.ReadinessProbe.HTTPGet = *after
329+
}
330+
newURL = fmt.Sprintf("/app-health/%v/livez", c.Name)
331+
after = calculateRewrite2(c.LivenessProbe, newURL, statusPort, portMap)
332+
if after != nil {
333+
*c.LivenessProbe.HTTPGet = *after
334+
}
335+
}
336+
337+
}

‎pilot/pkg/kube/inject/webhook.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -435,6 +435,11 @@ func createPatch(pod *corev1.Pod, prevStatus *SidecarInjectionStatus, annotation
435435
patch = append(patch, removeVolumes(pod.Spec.Volumes, prevStatus.Volumes, "/spec/volumes")...)
436436
patch = append(patch, removeImagePullSecrets(pod.Spec.ImagePullSecrets, prevStatus.ImagePullSecrets, "/spec/imagePullSecrets")...)
437437

438+
if appProbers := extractKubeAppProbers(&pod.Spec); appProbers != nil {
439+
// json.Unmarshal(*appProbers)
440+
// Update the sidecar container Args.
441+
}
442+
438443
patch = append(patch, addContainer(pod.Spec.InitContainers, sic.InitContainers, "/spec/initContainers")...)
439444
patch = append(patch, addContainer(pod.Spec.Containers, sic.Containers, "/spec/containers")...)
440445
patch = append(patch, addVolume(pod.Spec.Volumes, sic.Volumes, "/spec/volumes")...)

0 commit comments

Comments
 (0)