Skip to content

Commit 4a8d111

Browse files
authored
Merge pull request #11 from mattia-eleuteri/feat/port-filtering
feat: per-service ingress port filtering
2 parents 5655195 + 29ea0d7 commit 4a8d111

7 files changed

Lines changed: 723 additions & 38 deletions

File tree

‎README.md‎

Lines changed: 35 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,14 +17,46 @@ The last option is the simplest and most flexible, but it has a limitation: Kube
1717
but only traffic on specific ports (see: [Kubernetes Issue #23864](https://github.com/kubernetes/kubernetes/issues/23864)).
1818
Additionally, kube-proxy does not perform SNAT, which causes outgoing traffic from the pod to use the default gateway of the host where it is running.
1919

20-
To address these issues, we have added an additional controller that performs 1:1 NAT for services annotated with `networking.cozystack.io/wholeIP=true`.
20+
To address these issues, we have added an additional controller that performs 1:1 NAT for services annotated with `networking.cozystack.io/wholeIP`.
2121

2222
## How It Works
2323

24-
cozy-proxy is a simple Kubernetes controller that watches for services with the `networking.cozystack.io/wholeIP=true` annotation. When it finds such a service, it creates an NFT rule that forwards all traffic from the service's external IP to the pod's IP and vice versa. It also disables connection tracking (conntrack) for traffic between the service and the pod, offloading that work to NFTables.
24+
cozy-proxy is a simple Kubernetes controller that watches for services with the `networking.cozystack.io/wholeIP` annotation. When it finds such a service, it creates an NFT rule that forwards traffic from the service's external IP to the pod's IP and vice versa, performing source-IP preservation for egress traffic.
2525

2626
This controller can be used together with kube-proxy and Cilium in kube-proxy replacement mode.
2727

28+
## Service annotations
29+
30+
cozy-proxy reacts to Services that carry the `networking.cozystack.io/wholeIP`
31+
annotation. The annotation value selects the ingress mode:
32+
33+
| Value | Behavior |
34+
|-----------|-----------------------------------------------------------------------------------------------------------------|
35+
| `"true"` | **Whole-IP passthrough.** All TCP/UDP traffic to the LoadBalancer IP is forwarded to the backend pod. |
36+
| `"false"` | **Per-port filtering.** Only TCP/UDP traffic to ports listed in `Service.spec.ports` is forwarded; rest dropped.|
37+
| absent | Service is not managed by cozy-proxy. |
38+
39+
In both managed modes, egress traffic from the backend pod is SNATed to the
40+
LoadBalancer IP for source-IP preservation.
41+
42+
## Datapath
43+
44+
The nftables ruleset placed in table `ip cozy_proxy` consists of:
45+
46+
- Chain `egress_snat` at priority `raw` (-300): rewrites packet source IP via
47+
the `pod_svc` map for outbound traffic from managed pods. Runs before
48+
conntrack so the recorded tuple has `saddr=LB_IP`.
49+
- Chain `ingress_dnat` at priority `mangle` (-150): rewrites packet
50+
destination IP via the `svc_pod` map for inbound traffic to a LoadBalancer
51+
IP. Runs after conntrack so reply packets of egress flows are matched
52+
correctly.
53+
- Chain `port_filter` at priority `filter` (0): for Services in port-filter
54+
mode (`wholeIP: "false"`), drops ingress packets whose
55+
`(daddr, l4proto, dport)` is not in `allowed_ports`. The chain accepts
56+
packets in conntrack states `established` or `related` first, so reply
57+
packets of egress flows bypass the filter even when their dport is the VM's
58+
ephemeral source port.
59+
2860
## Installation
2961

3062
Install controller using Helm-chart:
@@ -35,7 +67,7 @@ helm install cozy-proxy charts/cozy-proxy -n kube-system
3567

3668
## Usage
3769

38-
Create a LoadBalancer service with `networking.cozystack.io/wholeIP=true` annotation:
70+
Create a LoadBalancer service with `networking.cozystack.io/wholeIP: "true"` annotation:
3971

4072
```yaml
4173
apiVersion: v1

‎pkg/controllers/services_controller.go‎

Lines changed: 123 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -231,8 +231,20 @@ func (c *ServicesController) addServiceFunc(obj interface{}) {
231231
if err == nil && ep != nil && hasValidEndpointIP(ep) && hasValidServiceIP(svc) {
232232
se.Endpoint = ep
233233
c.Services.Set(svc.Namespace, svc.Name, se)
234+
svcIP := svc.Status.LoadBalancer.Ingress[0].IP
235+
podIP := ep.Subsets[0].Addresses[0].IP
234236
// Ensure NAT mapping rules are set.
235-
c.Proxy.EnsureRules(svc.Status.LoadBalancer.Ingress[0].IP, ep.Subsets[0].Addresses[0].IP)
237+
c.Proxy.EnsureRules(svcIP, podIP)
238+
// Apply (or remove) port filtering depending on annotation value.
239+
if wholeIPPassthrough(svc) {
240+
if err := c.Proxy.DeletePortFilter(svcIP, podIP); err != nil {
241+
log.Error(err, "failed to delete port filter", "svcIP", svcIP, "podIP", podIP)
242+
}
243+
} else {
244+
if err := c.Proxy.EnsurePortFilter(svcIP, podIP, svc.Spec.Ports); err != nil {
245+
log.Error(err, "failed to ensure port filter", "svcIP", svcIP, "podIP", podIP)
246+
}
247+
}
236248
}
237249
}
238250

@@ -253,7 +265,12 @@ func (c *ServicesController) deleteServiceFunc(obj interface{}) {
253265
return
254266
}
255267

256-
c.Proxy.DeleteRules(se.Service.Status.LoadBalancer.Ingress[0].IP, se.Endpoint.Subsets[0].Addresses[0].IP)
268+
svcIP := se.Service.Status.LoadBalancer.Ingress[0].IP
269+
podIP := se.Endpoint.Subsets[0].Addresses[0].IP
270+
if err := c.Proxy.DeletePortFilter(svcIP, podIP); err != nil {
271+
log.Error(err, "failed to delete port filter on svc deletion", "svcIP", svcIP, "podIP", podIP)
272+
}
273+
c.Proxy.DeleteRules(svcIP, podIP)
257274
c.Services.Delete(svc.Namespace, svc.Name)
258275
}
259276

@@ -270,10 +287,12 @@ func (c *ServicesController) updateServiceFunc(oldObj, newObj interface{}) {
270287
if !hasWholeIPAnnotation(svc) {
271288
if se, exists := c.Services.Get(svc.Namespace, svc.Name); exists {
272289
if hasValidServiceIP(se.Service) && hasValidEndpointIP(se.Endpoint) {
273-
c.Proxy.DeleteRules(
274-
se.Service.Status.LoadBalancer.Ingress[0].IP,
275-
se.Endpoint.Subsets[0].Addresses[0].IP,
276-
)
290+
svcIP := se.Service.Status.LoadBalancer.Ingress[0].IP
291+
podIP := se.Endpoint.Subsets[0].Addresses[0].IP
292+
if err := c.Proxy.DeletePortFilter(svcIP, podIP); err != nil {
293+
log.Error(err, "failed to delete port filter", "svcIP", svcIP, "podIP", podIP)
294+
}
295+
c.Proxy.DeleteRules(svcIP, podIP)
277296
}
278297
c.Services.Delete(svc.Namespace, svc.Name)
279298
}
@@ -284,10 +303,12 @@ func (c *ServicesController) updateServiceFunc(oldObj, newObj interface{}) {
284303
if !hasValidServiceIP(svc) {
285304
if se, exists := c.Services.Get(svc.Namespace, svc.Name); exists {
286305
if hasValidServiceIP(se.Service) && hasValidEndpointIP(se.Endpoint) {
287-
c.Proxy.DeleteRules(
288-
se.Service.Status.LoadBalancer.Ingress[0].IP,
289-
se.Endpoint.Subsets[0].Addresses[0].IP,
290-
)
306+
svcIP := se.Service.Status.LoadBalancer.Ingress[0].IP
307+
podIP := se.Endpoint.Subsets[0].Addresses[0].IP
308+
if err := c.Proxy.DeletePortFilter(svcIP, podIP); err != nil {
309+
log.Error(err, "failed to delete port filter", "svcIP", svcIP, "podIP", podIP)
310+
}
311+
c.Proxy.DeleteRules(svcIP, podIP)
291312
}
292313
c.Services.Delete(svc.Namespace, svc.Name)
293314
}
@@ -324,10 +345,18 @@ func (c *ServicesController) updateServiceFunc(oldObj, newObj interface{}) {
324345

325346
// At this point, both the Service and Endpoint have valid IPs.
326347
// Ensure NAT mapping is up-to-date.
327-
c.Proxy.EnsureRules(
328-
svc.Status.LoadBalancer.Ingress[0].IP,
329-
ep.Subsets[0].Addresses[0].IP,
330-
)
348+
svcIP := svc.Status.LoadBalancer.Ingress[0].IP
349+
podIP := ep.Subsets[0].Addresses[0].IP
350+
c.Proxy.EnsureRules(svcIP, podIP)
351+
if wholeIPPassthrough(svc) {
352+
if err := c.Proxy.DeletePortFilter(svcIP, podIP); err != nil {
353+
log.Error(err, "failed to delete port filter", "svcIP", svcIP, "podIP", podIP)
354+
}
355+
} else {
356+
if err := c.Proxy.EnsurePortFilter(svcIP, podIP, svc.Spec.Ports); err != nil {
357+
log.Error(err, "failed to ensure port filter", "svcIP", svcIP, "podIP", podIP)
358+
}
359+
}
331360

332361
// Update or add the service mapping with the new endpoint.
333362
c.Services.Set(svc.Namespace, svc.Name, &ServiceEndpoints{Service: svc, Endpoint: ep})
@@ -354,10 +383,19 @@ func (c *ServicesController) addEndpointFunc(obj interface{}) {
354383

355384
// If both the Service and the Endpoint have valid IPs, ensure NAT mapping rules.
356385
if hasValidServiceIP(se.Service) && hasValidEndpointIP(ep) {
357-
c.Proxy.EnsureRules(
358-
se.Service.Status.LoadBalancer.Ingress[0].IP,
359-
ep.Subsets[0].Addresses[0].IP,
360-
)
386+
svcIP := se.Service.Status.LoadBalancer.Ingress[0].IP
387+
podIP := ep.Subsets[0].Addresses[0].IP
388+
c.Proxy.EnsureRules(svcIP, podIP)
389+
// Mirror the port-filter state to the new endpoint.
390+
if wholeIPPassthrough(se.Service) {
391+
if err := c.Proxy.DeletePortFilter(svcIP, podIP); err != nil {
392+
log.Error(err, "failed to delete port filter on endpoint add", "svcIP", svcIP, "podIP", podIP)
393+
}
394+
} else {
395+
if err := c.Proxy.EnsurePortFilter(svcIP, podIP, se.Service.Spec.Ports); err != nil {
396+
log.Error(err, "failed to ensure port filter on endpoint add", "svcIP", svcIP, "podIP", podIP)
397+
}
398+
}
361399
}
362400
}
363401

@@ -377,7 +415,12 @@ func (c *ServicesController) deleteEndpointFunc(obj interface{}) {
377415
if !hasValidServiceIP(se.Service) || !hasValidEndpointIP(se.Endpoint) {
378416
return
379417
}
380-
c.Proxy.DeleteRules(se.Service.Status.LoadBalancer.Ingress[0].IP, se.Endpoint.Subsets[0].Addresses[0].IP)
418+
svcIP := se.Service.Status.LoadBalancer.Ingress[0].IP
419+
podIP := se.Endpoint.Subsets[0].Addresses[0].IP
420+
if err := c.Proxy.DeletePortFilter(svcIP, podIP); err != nil {
421+
log.Error(err, "failed to delete port filter on endpoint delete", "svcIP", svcIP, "podIP", podIP)
422+
}
423+
c.Proxy.DeleteRules(svcIP, podIP)
381424
// Set the endpoint to nil.
382425
c.Services.SetEndpoint(ep.Namespace, ep.Name, nil)
383426
}
@@ -397,7 +440,12 @@ func (c *ServicesController) updateEndpointFunc(oldObj, newObj interface{}) {
397440
}
398441
if !hasValidEndpointIP(ep) {
399442
if hasValidServiceIP(se.Service) && hasValidEndpointIP(se.Endpoint) {
400-
c.Proxy.DeleteRules(se.Service.Status.LoadBalancer.Ingress[0].IP, se.Endpoint.Subsets[0].Addresses[0].IP)
443+
svcIP := se.Service.Status.LoadBalancer.Ingress[0].IP
444+
oldPodIP := se.Endpoint.Subsets[0].Addresses[0].IP
445+
if err := c.Proxy.DeletePortFilter(svcIP, oldPodIP); err != nil {
446+
log.Error(err, "failed to delete port filter on endpoint invalidation", "svcIP", svcIP, "podIP", oldPodIP)
447+
}
448+
c.Proxy.DeleteRules(svcIP, oldPodIP)
401449
}
402450
c.Services.SetEndpoint(ep.Namespace, ep.Name, ep)
403451
return
@@ -408,7 +456,18 @@ func (c *ServicesController) updateEndpointFunc(oldObj, newObj interface{}) {
408456
if !hasValidEndpointIP(ep) {
409457
return
410458
}
411-
c.Proxy.EnsureRules(se.Service.Status.LoadBalancer.Ingress[0].IP, ep.Subsets[0].Addresses[0].IP)
459+
svcIP := se.Service.Status.LoadBalancer.Ingress[0].IP
460+
podIP := ep.Subsets[0].Addresses[0].IP
461+
c.Proxy.EnsureRules(svcIP, podIP)
462+
if wholeIPPassthrough(se.Service) {
463+
if err := c.Proxy.DeletePortFilter(svcIP, podIP); err != nil {
464+
log.Error(err, "failed to delete port filter on endpoint update", "svcIP", svcIP, "podIP", podIP)
465+
}
466+
} else {
467+
if err := c.Proxy.EnsurePortFilter(svcIP, podIP, se.Service.Spec.Ports); err != nil {
468+
log.Error(err, "failed to ensure port filter on endpoint update", "svcIP", svcIP, "podIP", podIP)
469+
}
470+
}
412471
c.Services.SetEndpoint(ep.Namespace, ep.Name, ep)
413472
}
414473

@@ -447,10 +506,28 @@ func hasValidEndpointIP(ep *v1.Endpoints) bool {
447506
return ep.Subsets[0].Addresses[0].IP != ""
448507
}
449508

450-
// hasWholeIPAnnotation checks if the service has the wholeIP annotation set to true.
509+
const wholeIPAnnotation = "networking.cozystack.io/wholeIP"
510+
511+
// hasWholeIPAnnotation reports whether the service is opted into cozy-proxy
512+
// management via the wholeIP annotation. Any value triggers management — the
513+
// value's meaning (passthrough vs port-filter) is determined by wholeIPPassthrough.
451514
func hasWholeIPAnnotation(svc *v1.Service) bool {
452-
val, ok := svc.Annotations["networking.cozystack.io/wholeIP"]
453-
return ok && val == "true"
515+
if svc == nil {
516+
return false
517+
}
518+
_, ok := svc.Annotations[wholeIPAnnotation]
519+
return ok
520+
}
521+
522+
// wholeIPPassthrough reports whether ingress traffic should bypass port
523+
// filtering. Defaults to true (current behavior) for any value other than
524+
// the explicit string "false", preserving backward compatibility for services
525+
// rendered by older charts (annotation: "true") and services with no value.
526+
func wholeIPPassthrough(svc *v1.Service) bool {
527+
if svc == nil || svc.Annotations == nil {
528+
return true
529+
}
530+
return svc.Annotations[wholeIPAnnotation] != "false"
454531
}
455532

456533
// cleanupRemovedServices performs an initial cleanup for removed services.
@@ -478,5 +555,27 @@ func (c *ServicesController) cleanupRemovedServices() error {
478555
if err := c.Proxy.CleanupRules(keepMap); err != nil {
479556
return fmt.Errorf("failed to perform initial cleanup: %w", err)
480557
}
558+
// Build per-svc port filter snapshot for services in non-passthrough mode.
559+
// Keyed by svcIP (caller convenience); the entry carries podIP and ports
560+
// because the actual nft set keys are pod IPs (post-DNAT match).
561+
keepFilters := make(map[string]nat.PortFilterEntry)
562+
for _, se := range allServices {
563+
if se.Service == nil || se.Endpoint == nil {
564+
continue
565+
}
566+
if !hasValidServiceIP(se.Service) || !hasValidEndpointIP(se.Endpoint) {
567+
continue
568+
}
569+
if wholeIPPassthrough(se.Service) {
570+
continue
571+
}
572+
keepFilters[se.Service.Status.LoadBalancer.Ingress[0].IP] = nat.PortFilterEntry{
573+
PodIP: se.Endpoint.Subsets[0].Addresses[0].IP,
574+
Ports: se.Service.Spec.Ports,
575+
}
576+
}
577+
if err := c.Proxy.CleanupPortFilters(keepFilters); err != nil {
578+
return fmt.Errorf("failed to perform port-filter cleanup: %w", err)
579+
}
481580
return nil
482581
}
Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
package controllers
2+
3+
import (
4+
"testing"
5+
6+
v1 "k8s.io/api/core/v1"
7+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
8+
)
9+
10+
func svcWith(annot map[string]string) *v1.Service {
11+
return &v1.Service{ObjectMeta: metav1.ObjectMeta{Annotations: annot}}
12+
}
13+
14+
func TestHasWholeIPAnnotation(t *testing.T) {
15+
cases := []struct {
16+
name string
17+
svc *v1.Service
18+
expect bool
19+
}{
20+
{"true value", svcWith(map[string]string{"networking.cozystack.io/wholeIP": "true"}), true},
21+
{"false value", svcWith(map[string]string{"networking.cozystack.io/wholeIP": "false"}), true},
22+
{"absent annotation", svcWith(map[string]string{}), false},
23+
{"nil annotations", &v1.Service{}, false},
24+
{"unrelated annotation", svcWith(map[string]string{"foo": "bar"}), false},
25+
}
26+
for _, c := range cases {
27+
t.Run(c.name, func(t *testing.T) {
28+
if got := hasWholeIPAnnotation(c.svc); got != c.expect {
29+
t.Errorf("hasWholeIPAnnotation = %v, want %v", got, c.expect)
30+
}
31+
})
32+
}
33+
}
34+
35+
func TestWholeIPPassthrough(t *testing.T) {
36+
cases := []struct {
37+
name string
38+
svc *v1.Service
39+
expect bool
40+
}{
41+
{"true value", svcWith(map[string]string{"networking.cozystack.io/wholeIP": "true"}), true},
42+
{"false value", svcWith(map[string]string{"networking.cozystack.io/wholeIP": "false"}), false},
43+
{"absent annotation defaults to passthrough", svcWith(map[string]string{}), true},
44+
{"nil annotations defaults to passthrough", &v1.Service{}, true},
45+
}
46+
for _, c := range cases {
47+
t.Run(c.name, func(t *testing.T) {
48+
if got := wholeIPPassthrough(c.svc); got != c.expect {
49+
t.Errorf("wholeIPPassthrough = %v, want %v", got, c.expect)
50+
}
51+
})
52+
}
53+
}

‎pkg/proxy/dummy.go‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
package proxy
22

3-
import "fmt"
3+
import (
4+
"fmt"
5+
6+
corev1 "k8s.io/api/core/v1"
7+
)
48

59
type DummyProxyProcessor struct{}
610

@@ -23,3 +27,21 @@ func (d *DummyProxyProcessor) CleanupRules(KeepMap map[string]string) error {
2327
fmt.Println("CleanupRules called with KeepMap:", KeepMap)
2428
return nil
2529
}
30+
31+
func (d *DummyProxyProcessor) EnsurePortFilter(SvcIP, PodIP string, Ports []corev1.ServicePort) error {
32+
fmt.Printf("EnsurePortFilter called with SvcIP: %s, PodIP: %s, Ports: %+v\n", SvcIP, PodIP, Ports)
33+
return nil
34+
}
35+
36+
func (d *DummyProxyProcessor) DeletePortFilter(SvcIP, PodIP string) error {
37+
fmt.Printf("DeletePortFilter called with SvcIP: %s, PodIP: %s\n", SvcIP, PodIP)
38+
return nil
39+
}
40+
41+
func (d *DummyProxyProcessor) CleanupPortFilters(keep map[string]PortFilterEntry) error {
42+
fmt.Printf("CleanupPortFilters called with %d entries\n", len(keep))
43+
return nil
44+
}
45+
46+
// Compile-time assertion that DummyProxyProcessor satisfies ProxyProcessor.
47+
var _ ProxyProcessor = (*DummyProxyProcessor)(nil)

0 commit comments

Comments
 (0)