Skip to content

Commit 2131467

Browse files
kvapsclaude
andcommitted
feat: add allowICMP annotation for port-filter mode
Introduce networking.cozystack.io/allowICMP. When set to "true" on a service in port-filter mode (wholeIP: "false"), ICMP traffic toward the backend pod IP is accepted instead of being dropped by the per-port filter. Without this opt-in, port-filter mode silently drops all ICMP, which breaks ping, PMTU discovery (ICMP "fragmentation needed") and ICMP unreachable signalling. Implementation: - New nft set icmp_allowed_pods (key: ipv4 pod IP). - New rule in port_filter, between the ct established,related accept and the per-port drop: ip daddr @icmp_allowed_pods meta l4proto icmp accept. - New ProxyProcessor methods EnsureICMPAllow / DeleteICMPAllow / CleanupICMPAllow, mirroring the port-filter API. - Controller refactored to consolidate port-filter + ICMP-allow handling into reconcilePortFilter / clearPortFilter helpers. Co-Authored-By: Claude <[email protected]> Signed-off-by: Andrei Kvapil <[email protected]>
1 parent 29ea0d7 commit 2131467

6 files changed

Lines changed: 302 additions & 58 deletions

File tree

‎README.md‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,14 @@ annotation. The annotation value selects the ingress mode:
3939
In both managed modes, egress traffic from the backend pod is SNATed to the
4040
LoadBalancer IP for source-IP preservation.
4141

42+
The optional `networking.cozystack.io/allowICMP: "true"` annotation, only
43+
meaningful in port-filter mode (`wholeIP: "false"`), accepts ICMP traffic
44+
toward the backend pod IP that would otherwise be dropped by the per-port
45+
filter. Without it, all ICMP to a port-filtered pod is dropped — which also
46+
blocks `ping`, **PMTU discovery** (ICMP "fragmentation needed"), and ICMP
47+
unreachable signalling. Recommended for any service where path-MTU mismatches
48+
or observability matter.
49+
4250
## Datapath
4351

4452
The nftables ruleset placed in table `ip cozy_proxy` consists of:
@@ -55,7 +63,9 @@ The nftables ruleset placed in table `ip cozy_proxy` consists of:
5563
`(daddr, l4proto, dport)` is not in `allowed_ports`. The chain accepts
5664
packets in conntrack states `established` or `related` first, so reply
5765
packets of egress flows bypass the filter even when their dport is the VM's
58-
ephemeral source port.
66+
ephemeral source port. ICMP is dropped by default; if the
67+
`allowICMP: "true"` annotation is set, the pod IP is added to
68+
`icmp_allowed_pods` and ICMP toward it is accepted before the drop rule.
5969

6070
## Installation
6171

‎pkg/controllers/services_controller.go‎

Lines changed: 79 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -235,16 +235,7 @@ func (c *ServicesController) addServiceFunc(obj interface{}) {
235235
podIP := ep.Subsets[0].Addresses[0].IP
236236
// Ensure NAT mapping rules are set.
237237
c.Proxy.EnsureRules(svcIP, podIP)
238-
// Apply (or remove) port filtering depending on annotation value.
239-
if wholeIPPassthrough(svc) {
240-
if err := c.Proxy.DeletePortFilter(svcIP, podIP); err != nil {
241-
log.Error(err, "failed to delete port filter", "svcIP", svcIP, "podIP", podIP)
242-
}
243-
} else {
244-
if err := c.Proxy.EnsurePortFilter(svcIP, podIP, svc.Spec.Ports); err != nil {
245-
log.Error(err, "failed to ensure port filter", "svcIP", svcIP, "podIP", podIP)
246-
}
247-
}
238+
c.reconcilePortFilter(svc, svcIP, podIP, "on svc add")
248239
}
249240
}
250241

@@ -267,9 +258,7 @@ func (c *ServicesController) deleteServiceFunc(obj interface{}) {
267258

268259
svcIP := se.Service.Status.LoadBalancer.Ingress[0].IP
269260
podIP := se.Endpoint.Subsets[0].Addresses[0].IP
270-
if err := c.Proxy.DeletePortFilter(svcIP, podIP); err != nil {
271-
log.Error(err, "failed to delete port filter on svc deletion", "svcIP", svcIP, "podIP", podIP)
272-
}
261+
c.clearPortFilter(svcIP, podIP, "on svc deletion")
273262
c.Proxy.DeleteRules(svcIP, podIP)
274263
c.Services.Delete(svc.Namespace, svc.Name)
275264
}
@@ -289,9 +278,7 @@ func (c *ServicesController) updateServiceFunc(oldObj, newObj interface{}) {
289278
if hasValidServiceIP(se.Service) && hasValidEndpointIP(se.Endpoint) {
290279
svcIP := se.Service.Status.LoadBalancer.Ingress[0].IP
291280
podIP := se.Endpoint.Subsets[0].Addresses[0].IP
292-
if err := c.Proxy.DeletePortFilter(svcIP, podIP); err != nil {
293-
log.Error(err, "failed to delete port filter", "svcIP", svcIP, "podIP", podIP)
294-
}
281+
c.clearPortFilter(svcIP, podIP, "on annotation removal")
295282
c.Proxy.DeleteRules(svcIP, podIP)
296283
}
297284
c.Services.Delete(svc.Namespace, svc.Name)
@@ -305,9 +292,7 @@ func (c *ServicesController) updateServiceFunc(oldObj, newObj interface{}) {
305292
if hasValidServiceIP(se.Service) && hasValidEndpointIP(se.Endpoint) {
306293
svcIP := se.Service.Status.LoadBalancer.Ingress[0].IP
307294
podIP := se.Endpoint.Subsets[0].Addresses[0].IP
308-
if err := c.Proxy.DeletePortFilter(svcIP, podIP); err != nil {
309-
log.Error(err, "failed to delete port filter", "svcIP", svcIP, "podIP", podIP)
310-
}
295+
c.clearPortFilter(svcIP, podIP, "on svc IP loss")
311296
c.Proxy.DeleteRules(svcIP, podIP)
312297
}
313298
c.Services.Delete(svc.Namespace, svc.Name)
@@ -348,15 +333,7 @@ func (c *ServicesController) updateServiceFunc(oldObj, newObj interface{}) {
348333
svcIP := svc.Status.LoadBalancer.Ingress[0].IP
349334
podIP := ep.Subsets[0].Addresses[0].IP
350335
c.Proxy.EnsureRules(svcIP, podIP)
351-
if wholeIPPassthrough(svc) {
352-
if err := c.Proxy.DeletePortFilter(svcIP, podIP); err != nil {
353-
log.Error(err, "failed to delete port filter", "svcIP", svcIP, "podIP", podIP)
354-
}
355-
} else {
356-
if err := c.Proxy.EnsurePortFilter(svcIP, podIP, svc.Spec.Ports); err != nil {
357-
log.Error(err, "failed to ensure port filter", "svcIP", svcIP, "podIP", podIP)
358-
}
359-
}
336+
c.reconcilePortFilter(svc, svcIP, podIP, "on svc update")
360337

361338
// Update or add the service mapping with the new endpoint.
362339
c.Services.Set(svc.Namespace, svc.Name, &ServiceEndpoints{Service: svc, Endpoint: ep})
@@ -386,16 +363,7 @@ func (c *ServicesController) addEndpointFunc(obj interface{}) {
386363
svcIP := se.Service.Status.LoadBalancer.Ingress[0].IP
387364
podIP := ep.Subsets[0].Addresses[0].IP
388365
c.Proxy.EnsureRules(svcIP, podIP)
389-
// Mirror the port-filter state to the new endpoint.
390-
if wholeIPPassthrough(se.Service) {
391-
if err := c.Proxy.DeletePortFilter(svcIP, podIP); err != nil {
392-
log.Error(err, "failed to delete port filter on endpoint add", "svcIP", svcIP, "podIP", podIP)
393-
}
394-
} else {
395-
if err := c.Proxy.EnsurePortFilter(svcIP, podIP, se.Service.Spec.Ports); err != nil {
396-
log.Error(err, "failed to ensure port filter on endpoint add", "svcIP", svcIP, "podIP", podIP)
397-
}
398-
}
366+
c.reconcilePortFilter(se.Service, svcIP, podIP, "on endpoint add")
399367
}
400368
}
401369

@@ -417,9 +385,7 @@ func (c *ServicesController) deleteEndpointFunc(obj interface{}) {
417385
}
418386
svcIP := se.Service.Status.LoadBalancer.Ingress[0].IP
419387
podIP := se.Endpoint.Subsets[0].Addresses[0].IP
420-
if err := c.Proxy.DeletePortFilter(svcIP, podIP); err != nil {
421-
log.Error(err, "failed to delete port filter on endpoint delete", "svcIP", svcIP, "podIP", podIP)
422-
}
388+
c.clearPortFilter(svcIP, podIP, "on endpoint delete")
423389
c.Proxy.DeleteRules(svcIP, podIP)
424390
// Set the endpoint to nil.
425391
c.Services.SetEndpoint(ep.Namespace, ep.Name, nil)
@@ -442,9 +408,7 @@ func (c *ServicesController) updateEndpointFunc(oldObj, newObj interface{}) {
442408
if hasValidServiceIP(se.Service) && hasValidEndpointIP(se.Endpoint) {
443409
svcIP := se.Service.Status.LoadBalancer.Ingress[0].IP
444410
oldPodIP := se.Endpoint.Subsets[0].Addresses[0].IP
445-
if err := c.Proxy.DeletePortFilter(svcIP, oldPodIP); err != nil {
446-
log.Error(err, "failed to delete port filter on endpoint invalidation", "svcIP", svcIP, "podIP", oldPodIP)
447-
}
411+
c.clearPortFilter(svcIP, oldPodIP, "on endpoint invalidation")
448412
c.Proxy.DeleteRules(svcIP, oldPodIP)
449413
}
450414
c.Services.SetEndpoint(ep.Namespace, ep.Name, ep)
@@ -459,15 +423,7 @@ func (c *ServicesController) updateEndpointFunc(oldObj, newObj interface{}) {
459423
svcIP := se.Service.Status.LoadBalancer.Ingress[0].IP
460424
podIP := ep.Subsets[0].Addresses[0].IP
461425
c.Proxy.EnsureRules(svcIP, podIP)
462-
if wholeIPPassthrough(se.Service) {
463-
if err := c.Proxy.DeletePortFilter(svcIP, podIP); err != nil {
464-
log.Error(err, "failed to delete port filter on endpoint update", "svcIP", svcIP, "podIP", podIP)
465-
}
466-
} else {
467-
if err := c.Proxy.EnsurePortFilter(svcIP, podIP, se.Service.Spec.Ports); err != nil {
468-
log.Error(err, "failed to ensure port filter on endpoint update", "svcIP", svcIP, "podIP", podIP)
469-
}
470-
}
426+
c.reconcilePortFilter(se.Service, svcIP, podIP, "on endpoint update")
471427
c.Services.SetEndpoint(ep.Namespace, ep.Name, ep)
472428
}
473429

@@ -506,7 +462,10 @@ func hasValidEndpointIP(ep *v1.Endpoints) bool {
506462
return ep.Subsets[0].Addresses[0].IP != ""
507463
}
508464

509-
const wholeIPAnnotation = "networking.cozystack.io/wholeIP"
465+
const (
466+
wholeIPAnnotation = "networking.cozystack.io/wholeIP"
467+
allowICMPAnnotation = "networking.cozystack.io/allowICMP"
468+
)
510469

511470
// hasWholeIPAnnotation reports whether the service is opted into cozy-proxy
512471
// management via the wholeIP annotation. Any value triggers management — the
@@ -530,6 +489,54 @@ func wholeIPPassthrough(svc *v1.Service) bool {
530489
return svc.Annotations[wholeIPAnnotation] != "false"
531490
}
532491

492+
// allowICMP reports whether ICMP traffic should bypass the port_filter drop
493+
// rule for this service. Only meaningful in port-filter mode (wholeIP=false);
494+
// in passthrough mode the port_filter drop rule does not apply at all.
495+
func allowICMP(svc *v1.Service) bool {
496+
if svc == nil || svc.Annotations == nil {
497+
return false
498+
}
499+
return svc.Annotations[allowICMPAnnotation] == "true"
500+
}
501+
502+
// reconcilePortFilter applies the port-filter and ICMP-allow state implied by
503+
// the service's annotations. Call sites pass the resolved svcIP/podIP and a
504+
// short context string that ends up in error logs.
505+
func (c *ServicesController) reconcilePortFilter(svc *v1.Service, svcIP, podIP, ctx string) {
506+
if wholeIPPassthrough(svc) {
507+
if err := c.Proxy.DeletePortFilter(svcIP, podIP); err != nil {
508+
log.Error(err, "failed to delete port filter "+ctx, "svcIP", svcIP, "podIP", podIP)
509+
}
510+
if err := c.Proxy.DeleteICMPAllow(svcIP, podIP); err != nil {
511+
log.Error(err, "failed to delete ICMP allow "+ctx, "svcIP", svcIP, "podIP", podIP)
512+
}
513+
return
514+
}
515+
if err := c.Proxy.EnsurePortFilter(svcIP, podIP, svc.Spec.Ports); err != nil {
516+
log.Error(err, "failed to ensure port filter "+ctx, "svcIP", svcIP, "podIP", podIP)
517+
}
518+
if allowICMP(svc) {
519+
if err := c.Proxy.EnsureICMPAllow(svcIP, podIP); err != nil {
520+
log.Error(err, "failed to ensure ICMP allow "+ctx, "svcIP", svcIP, "podIP", podIP)
521+
}
522+
} else {
523+
if err := c.Proxy.DeleteICMPAllow(svcIP, podIP); err != nil {
524+
log.Error(err, "failed to delete ICMP allow "+ctx, "svcIP", svcIP, "podIP", podIP)
525+
}
526+
}
527+
}
528+
529+
// clearPortFilter unconditionally removes both port-filter and ICMP-allow
530+
// state for (svcIP, podIP). Used by delete paths.
531+
func (c *ServicesController) clearPortFilter(svcIP, podIP, ctx string) {
532+
if err := c.Proxy.DeletePortFilter(svcIP, podIP); err != nil {
533+
log.Error(err, "failed to delete port filter "+ctx, "svcIP", svcIP, "podIP", podIP)
534+
}
535+
if err := c.Proxy.DeleteICMPAllow(svcIP, podIP); err != nil {
536+
log.Error(err, "failed to delete ICMP allow "+ctx, "svcIP", svcIP, "podIP", podIP)
537+
}
538+
}
539+
533540
// cleanupRemovedServices performs an initial cleanup for removed services.
534541
func (c *ServicesController) cleanupRemovedServices() error {
535542
keepMap := make(map[string]string)
@@ -577,5 +584,23 @@ func (c *ServicesController) cleanupRemovedServices() error {
577584
if err := c.Proxy.CleanupPortFilters(keepFilters); err != nil {
578585
return fmt.Errorf("failed to perform port-filter cleanup: %w", err)
579586
}
587+
// Build ICMP-allow snapshot: services in port-filter mode that opt into
588+
// ICMP via the allowICMP annotation.
589+
keepICMP := make(map[string]string)
590+
for _, se := range allServices {
591+
if se.Service == nil || se.Endpoint == nil {
592+
continue
593+
}
594+
if !hasValidServiceIP(se.Service) || !hasValidEndpointIP(se.Endpoint) {
595+
continue
596+
}
597+
if wholeIPPassthrough(se.Service) || !allowICMP(se.Service) {
598+
continue
599+
}
600+
keepICMP[se.Service.Status.LoadBalancer.Ingress[0].IP] = se.Endpoint.Subsets[0].Addresses[0].IP
601+
}
602+
if err := c.Proxy.CleanupICMPAllow(keepICMP); err != nil {
603+
return fmt.Errorf("failed to perform ICMP-allow cleanup: %w", err)
604+
}
580605
return nil
581606
}

‎pkg/controllers/services_controller_test.go‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,3 +51,25 @@ func TestWholeIPPassthrough(t *testing.T) {
5151
})
5252
}
5353
}
54+
55+
func TestAllowICMP(t *testing.T) {
56+
cases := []struct {
57+
name string
58+
svc *v1.Service
59+
expect bool
60+
}{
61+
{"explicit true", svcWith(map[string]string{"networking.cozystack.io/allowICMP": "true"}), true},
62+
{"explicit false", svcWith(map[string]string{"networking.cozystack.io/allowICMP": "false"}), false},
63+
{"empty value", svcWith(map[string]string{"networking.cozystack.io/allowICMP": ""}), false},
64+
{"absent annotation defaults to false", svcWith(map[string]string{}), false},
65+
{"nil annotations defaults to false", &v1.Service{}, false},
66+
{"unrelated annotation", svcWith(map[string]string{"networking.cozystack.io/wholeIP": "false"}), false},
67+
}
68+
for _, c := range cases {
69+
t.Run(c.name, func(t *testing.T) {
70+
if got := allowICMP(c.svc); got != c.expect {
71+
t.Errorf("allowICMP = %v, want %v", got, c.expect)
72+
}
73+
})
74+
}
75+
}

‎pkg/proxy/dummy.go‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,5 +43,20 @@ func (d *DummyProxyProcessor) CleanupPortFilters(keep map[string]PortFilterEntry
4343
return nil
4444
}
4545

46+
func (d *DummyProxyProcessor) EnsureICMPAllow(SvcIP, PodIP string) error {
47+
fmt.Printf("EnsureICMPAllow called with SvcIP: %s, PodIP: %s\n", SvcIP, PodIP)
48+
return nil
49+
}
50+
51+
func (d *DummyProxyProcessor) DeleteICMPAllow(SvcIP, PodIP string) error {
52+
fmt.Printf("DeleteICMPAllow called with SvcIP: %s, PodIP: %s\n", SvcIP, PodIP)
53+
return nil
54+
}
55+
56+
func (d *DummyProxyProcessor) CleanupICMPAllow(keep map[string]string) error {
57+
fmt.Printf("CleanupICMPAllow called with %d entries\n", len(keep))
58+
return nil
59+
}
60+
4661
// Compile-time assertion that DummyProxyProcessor satisfies ProxyProcessor.
4762
var _ ProxyProcessor = (*DummyProxyProcessor)(nil)

‎pkg/proxy/interface.go‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,20 @@ type ProxyProcessor interface {
2424
// keepFilters. Any stale entries are removed. The PortFilterEntry struct
2525
// carries both the pod IP (used as the actual nft key) and the ports.
2626
CleanupPortFilters(keepFilters map[string]PortFilterEntry) error
27+
28+
// EnsureICMPAllow adds the pod IP to the ICMP allowlist consulted by the
29+
// port_filter chain. With this in place, ICMP traffic to a pod that is
30+
// otherwise port-filtered is accepted instead of dropped (preserves ping,
31+
// PMTU discovery, ICMP unreachable signalling). Idempotent.
32+
EnsureICMPAllow(SvcIP, PodIP string) error
33+
34+
// DeleteICMPAllow removes the pod IP from the ICMP allowlist. No-op if
35+
// not present.
36+
DeleteICMPAllow(SvcIP, PodIP string) error
37+
38+
// CleanupICMPAllow keeps only the entries listed in keepICMP (svcIP →
39+
// podIP) in the ICMP allowlist; everything else is removed.
40+
CleanupICMPAllow(keepICMP map[string]string) error
2741
}
2842

2943
// PortFilterEntry describes a port-filter desired state in the controller's

0 commit comments

Comments
 (0)