Skip to content

Commit 5bac183

Browse files
committed
fix(deploy): stabilize ingress and registry scheduling under load
1 parent 13f90b2 commit 5bac183

3 files changed

Lines changed: 98 additions & 17 deletions

File tree

‎deploy/README.md‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,42 @@ Pre-requisites:
104104
Then run `curl -H "Host: prod.registry.modelcontextprotocol.io" -k https://<EXTERNAL-IP>/v0/ping` to check that the service is up.
105105

106106

107+
## Ingress stability rollout
108+
109+
Staging and production reserve 300m CPU per ingress controller and 250m per
110+
registry replica, with no CPU limits. Each Deployment spreads its own revision
111+
across eligible nodes using `DoNotSchedule`, `maxSkew: 1`, and
112+
`matchLabelKeys: [pod-template-hash]`. Counting revisions separately prevents
113+
old pods from masking an uneven placement of the replacement replicas. Tainted
114+
nodes are excluded unless tolerated. Local development keeps soft spreading and
115+
100m CPU requests. Hosted clusters require Kubernetes 1.27 or newer with
116+
`MatchLabelKeysInPodTopologySpread` enabled.
117+
118+
Ingress readiness allows three seconds per check; liveness allows five. The
119+
chart's ten-second period and three/five failure thresholds are retained. These
120+
timeouts tolerate brief stalls; they do not establish that request latency is
121+
healthy. Both Deployments allow one surge pod and zero unavailable pods during
122+
updates. Before rollout, check that eligible nodes have enough unreserved CPU
123+
and memory for the surge pods as well as the steady-state requests. A pending
124+
replacement should be investigated, not worked around by deleting healthy pods.
125+
126+
Roll out these pod changes through staging and then the normal production
127+
promotion. Check that both Deployments become available, each pair of production
128+
replicas ends on different nodes, and no replacement remains Pending. Compare
129+
ingress probe failures/restarts, node CPU pressure, and request latency at
130+
comparable traffic over the following day, including a busy period. Application
131+
metrics omit ingress cache hits, so include ingress logs in the latency check.
132+
133+
Keep the node disk change in a subsequent rollout. The current 20 GB
134+
`pd-standard` boot disks also hold the ingress cache and container logs. The
135+
next step for the observed disk queues is larger `pd-balanced` boot disks,
136+
starting with 50 GB, using GKE surge upgrades with one extra node and zero
137+
unavailable nodes. A disk-type change recreates nodes; it is not an in-place
138+
filesystem change. Preview the actual Pulumi plan before applying it, verify a
139+
recent successful database backup, and account for a possible interruption
140+
while the single PostgreSQL instance moves. Complete the pod rollout before
141+
starting this node rollout. No node-pool change is included in the pod repair.
142+
107143
## Server-name search index
108144
109145
Migration 015 adds a GIN trigram index for substring searches on `server_name`.

‎deploy/pkg/k8s/ingress.go‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,28 @@ func SetupIngressController(ctx *pulumi.Context, cluster *providers.ProviderInfo
4848
replicaCount = 2
4949
}
5050

51+
ingressCPURequest := "100m"
52+
spreadConstraints := pulumi.Array{}
53+
if environment == "prod" || environment == "staging" {
54+
// Reserve CPU for the proxy and its health endpoint without limiting
55+
// bursts or exhausting the CPU needed for a rolling-update surge pod.
56+
ingressCPURequest = "300m"
57+
spreadConstraints = pulumi.Array{pulumi.Map{
58+
"maxSkew": pulumi.Int(1),
59+
"topologyKey": pulumi.String("kubernetes.io/hostname"),
60+
"whenUnsatisfiable": pulumi.String("DoNotSchedule"),
61+
"nodeTaintsPolicy": pulumi.String("Honor"),
62+
"matchLabelKeys": pulumi.StringArray{pulumi.String("pod-template-hash")},
63+
"labelSelector": pulumi.Map{
64+
"matchLabels": pulumi.StringMap{
65+
"app.kubernetes.io/name": pulumi.String("ingress-nginx"),
66+
"app.kubernetes.io/instance": pulumi.String("ingress-nginx"),
67+
"app.kubernetes.io/component": pulumi.String("controller"),
68+
},
69+
},
70+
}}
71+
}
72+
5173
// Install NGINX Ingress Controller
5274
ingressNginx, err := helm.NewChart(ctx, "ingress-nginx", helm.ChartArgs{
5375
Chart: pulumi.String("ingress-nginx"),
@@ -59,6 +81,25 @@ func SetupIngressController(ctx *pulumi.Context, cluster *providers.ProviderInfo
5981
Values: pulumi.Map{
6082
"controller": pulumi.Map{
6183
"replicaCount": pulumi.Int(replicaCount),
84+
"resources": pulumi.Map{
85+
"requests": pulumi.StringMap{
86+
"cpu": pulumi.String(ingressCPURequest),
87+
"memory": pulumi.String("90Mi"),
88+
},
89+
},
90+
"topologySpreadConstraints": spreadConstraints,
91+
"updateStrategy": pulumi.Map{
92+
"type": pulumi.String("RollingUpdate"),
93+
"rollingUpdate": pulumi.Map{
94+
"maxSurge": pulumi.Int(1),
95+
"maxUnavailable": pulumi.Int(0),
96+
},
97+
},
98+
// Keep readiness more responsive than liveness: a short stall should
99+
// stop new traffic before it causes a restart and discards the cache.
100+
// Helm retains the other probe defaults (10s period, 3/5 failures).
101+
"readinessProbe": pulumi.Map{"timeoutSeconds": pulumi.Int(3)},
102+
"livenessProbe": pulumi.Map{"timeoutSeconds": pulumi.Int(5)},
62103
"service": pulumi.Map{
63104
"type": serviceType,
64105
"externalTrafficPolicy": pulumi.String("Local"),

‎deploy/pkg/k8s/registry.go‎

Lines changed: 21 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,25 @@ func DeployMCPRegistry(ctx *pulumi.Context, cluster *providers.ProviderInfo, env
4141
imageTag = conf.Require("imageTag") // Explicit configuration required - no fallback
4242
}
4343

44+
registryCPURequest := "100m"
45+
spreadConstraint := &corev1.TopologySpreadConstraintArgs{
46+
MaxSkew: pulumi.Int(1),
47+
TopologyKey: pulumi.String("kubernetes.io/hostname"),
48+
WhenUnsatisfiable: pulumi.String("ScheduleAnyway"),
49+
LabelSelector: &metav1.LabelSelectorArgs{
50+
MatchLabels: pulumi.StringMap{"app": pulumi.String("mcp-registry")},
51+
},
52+
}
53+
if environment == "prod" || environment == "staging" {
54+
registryCPURequest = "250m"
55+
// Count each revision separately so removing old pods after a rolling
56+
// update cannot leave both new replicas on the same node. Unlike required
57+
// anti-affinity, maxSkew=1 still allows a surge pod on a two-node cluster.
58+
spreadConstraint.WhenUnsatisfiable = pulumi.String("DoNotSchedule")
59+
spreadConstraint.MatchLabelKeys = pulumi.StringArray{pulumi.String("pod-template-hash")}
60+
spreadConstraint.NodeTaintsPolicy = pulumi.String("Honor")
61+
}
62+
4463
// Create Secret with sensitive configuration
4564
secret, err := corev1.NewSecret(ctx, "mcp-registry-secrets", &corev1.SecretArgs{
4665
Metadata: &metav1.ObjectMetaArgs{
@@ -96,22 +115,7 @@ func DeployMCPRegistry(ctx *pulumi.Context, cluster *providers.ProviderInfo, env
96115
},
97116
},
98117
Spec: &corev1.PodSpecArgs{
99-
// Soft-spread pods across nodes so the PodDisruptionBudget below can't
100-
// deadlock a node drain when both pods happen to co-locate.
101-
// `ScheduleAnyway` keeps single-node clusters (e.g. constrained dev
102-
// envs) schedulable while still preferring spread under normal load.
103-
TopologySpreadConstraints: corev1.TopologySpreadConstraintArray{
104-
&corev1.TopologySpreadConstraintArgs{
105-
MaxSkew: pulumi.Int(1),
106-
TopologyKey: pulumi.String("kubernetes.io/hostname"),
107-
WhenUnsatisfiable: pulumi.String("ScheduleAnyway"),
108-
LabelSelector: &metav1.LabelSelectorArgs{
109-
MatchLabels: pulumi.StringMap{
110-
"app": pulumi.String("mcp-registry"),
111-
},
112-
},
113-
},
114-
},
118+
TopologySpreadConstraints: corev1.TopologySpreadConstraintArray{spreadConstraint},
115119
Containers: corev1.ContainerArray{
116120
&corev1.ContainerArgs{
117121
Name: pulumi.String("mcp-registry"),
@@ -234,7 +238,7 @@ func DeployMCPRegistry(ctx *pulumi.Context, cluster *providers.ProviderInfo, env
234238
// and limit to match.
235239
Requests: pulumi.StringMap{
236240
"memory": pulumi.String("512Mi"),
237-
"cpu": pulumi.String("100m"),
241+
"cpu": pulumi.String(registryCPURequest),
238242
},
239243
Limits: pulumi.StringMap{
240244
"memory": pulumi.String("1Gi"),

0 commit comments

Comments
 (0)