Skip to content

Commit 6acf34a

Browse files
test(gateway): pin full port-443 kinds contract across all cert modes (cilium#45559)
Signed-off-by: shreyaabaranwal <[email protected]>
1 parent 786c098 commit 6acf34a

2 files changed

Lines changed: 230 additions & 74 deletions

File tree

‎internal/controller/tenantgateway/reconciler.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -865,6 +865,11 @@ func (r *Reconciler) renderGateway(tgw *gatewayv1alpha1.TenantGateway, dynHostna
865865
// (cilium#45559). In practice only TLSRoute attaches to a Passthrough
866866
// listener, but listing HTTPRoute here is harmless — Gateway API
867867
// rejects any HTTPRoute that references a Passthrough sectionName.
868+
// NOTE: on Cilium 1.19.x each tls-<svc> listener will surface
869+
// ResolvedRefs=False/InvalidRouteKinds on the raw Gateway object
870+
// (cosmetic — Accepted, Programmed, traffic, and TenantGateway
871+
// readiness are all unaffected); removable once Cilium 1.20 /
872+
// cilium#45693 ships.
868873
// The corresponding TLSRoute templates (cozystack-api, vm-exportproxy,
869874
// cdi-uploadproxy) attach to these listeners by sectionName.
870875
for _, svc := range tgw.Spec.TLSPassthroughServices {

‎internal/controller/tenantgateway/reconciler_test.go‎

Lines changed: 225 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -3639,66 +3639,49 @@ func TestReconcile_ExistingSecretModeKeepsHTTPRedirectAndPassthrough(t *testing.
36393639
}
36403640

36413641
// TestReconcile_Port443ListenersShareKinds is a regression test for
3642-
// cilium#45559: when both HTTPS-terminate and TLS-passthrough listeners
3643-
// land on port 443, every one of them must carry the SAME
3644-
// allowedRoutes.kinds set. Divergent kinds cause Cilium to collapse all
3645-
// port-443 listeners into one, which then drops the HTTPRoutes that were
3646-
// accepted by the original HTTPS listener.
3642+
// cilium#45559 / cozystack#3070. It pins two distinct security contracts
3643+
// that must hold simultaneously across all three cert modes:
36473644
//
3648-
// The fix (Option B): each port-443 listener advertises both HTTPRoute
3649-
// and TLSRoute. Gateway API rejects any HTTPRoute that targets a
3650-
// Passthrough sectionName at route-attach time, so the expanded kinds do
3651-
// not weaken the security boundary — GRPCRoute/TCPRoute/UDPRoute are
3652-
// still excluded.
3645+
// 1. ANTI-COLLAPSE (cilium#45559): every port-443 listener must carry
3646+
// IDENTICAL allowedRoutes.kinds. Divergent kinds cause Cilium to
3647+
// merge all port-443 listeners into one, silently dropping the
3648+
// HTTPRoutes that were accepted by the HTTPS-terminate listeners.
3649+
//
3650+
// 2. FORBIDDEN KINDS: GRPCRoute, TCPRoute, and UDPRoute must NEVER
3651+
// appear in any port-443 listener's kinds set. These route types are
3652+
// not gated by the cozystack-route-hostname-policy VAP; admitting them
3653+
// would let a tenant serve arbitrary traffic under the apex cert
3654+
// without admission control.
3655+
//
3656+
// 3. NON-EMPTY: no port-443 listener may have nil or empty
3657+
// allowedRoutes.kinds. An empty set means Gateway API defaults to all
3658+
// route kinds — the same security hole as explicitly listing
3659+
// GRPCRoute/TCPRoute/UDPRoute.
3660+
//
3661+
// The canonical kinds set is exactly [HTTPRoute, TLSRoute] (both in the
3662+
// gateway.networking.k8s.io group). Gateway API rejects any HTTPRoute
3663+
// that targets a Passthrough sectionName at route-attach time, so listing
3664+
// HTTPRoute on TLS-passthrough listeners does not widen the actual attach
3665+
// surface — it only satisfies the Cilium same-port same-kinds invariant.
3666+
//
3667+
// All three cert modes are exercised because they produce different sets of
3668+
// port-443 listeners:
3669+
// - HTTP-01: per-app HTTPS listener (from HTTPRoute) + TLS-passthrough.
3670+
// - DNS-01: wildcard + apex + per-child-apex HTTPS listeners + TLS-passthrough.
3671+
// - existingSecret: same listener topology as DNS-01, operator-supplied cert.
36533672
func TestReconcile_Port443ListenersShareKinds(t *testing.T) {
3654-
s := newScheme(t)
3655-
tgw := &gatewayv1alpha1.TenantGateway{
3656-
ObjectMeta: metav1.ObjectMeta{Name: "cozystack", Namespace: "tenant-foo"},
3657-
Spec: gatewayv1alpha1.TenantGatewaySpec{
3658-
Apex: "foo.example.com",
3659-
CertMode: gatewayv1alpha1.CertModeHTTP01,
3660-
GatewayClassName: "cilium",
3661-
AttachedNamespaces: []string{"cozy-harbor"},
3662-
TLSPassthroughServices: []string{"api"},
3663-
},
3664-
}
3665-
// Attach an HTTPRoute so the controller renders an HTTPS listener for
3666-
// harbor.foo.example.com in addition to the TLS-passthrough listener
3667-
// for api.foo.example.com.
3668-
route := httpRouteAttached("harbor", "cozy-harbor", "harbor.foo.example.com")
3669-
3670-
c := fake.NewClientBuilder().
3671-
WithScheme(s).
3672-
WithObjects(tgw, route).
3673-
WithStatusSubresource(tgw).
3674-
Build()
3675-
3676-
r := &Reconciler{Client: c, Scheme: s}
3677-
if _, err := r.Reconcile(context.TODO(), ctrl.Request{
3678-
NamespacedName: types.NamespacedName{Name: "cozystack", Namespace: "tenant-foo"},
3679-
}); err != nil {
3680-
t.Fatalf("unexpected error: %v", err)
3681-
}
3682-
3683-
gw := &gatewayv1.Gateway{}
3684-
if err := c.Get(context.TODO(), types.NamespacedName{Name: "cozystack", Namespace: "tenant-foo"}, gw); err != nil {
3685-
t.Fatalf("get Gateway: %v", err)
3686-
}
3687-
3688-
// Collect every listener on port 443.
3689-
var port443 []gatewayv1.Listener
3690-
for _, l := range gw.Spec.Listeners {
3691-
if l.Port == 443 {
3692-
port443 = append(port443, l)
3693-
}
3694-
}
3695-
if len(port443) < 2 {
3696-
t.Fatalf("expected at least 2 port-443 listeners (one HTTPS, one TLS-passthrough), got %d: %+v", len(port443), gw.Spec.Listeners)
3673+
// canonicalPort443Kinds is the expected sorted "group/kind" key for
3674+
// every port-443 listener. Sorted so reflect.DeepEqual is order-independent.
3675+
canonicalPort443Kinds := []string{
3676+
gatewayv1.GroupName + "/HTTPRoute",
3677+
gatewayv1.GroupName + "/TLSRoute",
36973678
}
3679+
sort.Strings(canonicalPort443Kinds)
36983680

3699-
// Canonical kinds key: sorted "Group/Kind" strings for comparison.
3681+
// kindsKey normalises a RouteGroupKind slice to a sorted []string so
3682+
// all subsequent comparisons are order-independent.
37003683
kindsKey := func(kinds []gatewayv1.RouteGroupKind) []string {
3701-
var out []string
3684+
out := make([]string, 0, len(kinds))
37023685
for _, k := range kinds {
37033686
g := ""
37043687
if k.Group != nil {
@@ -3710,29 +3693,197 @@ func TestReconcile_Port443ListenersShareKinds(t *testing.T) {
37103693
return out
37113694
}
37123695

3713-
allowedKinds := []string{
3714-
gatewayv1.GroupName + "/HTTPRoute",
3715-
gatewayv1.GroupName + "/TLSRoute",
3716-
}
3717-
sort.Strings(allowedKinds)
3718-
3719-
var referenceKinds []string
3720-
for i, l := range port443 {
3721-
if l.AllowedRoutes == nil || len(l.AllowedRoutes.Kinds) == 0 {
3722-
t.Errorf("listener[%d] %q (port 443): allowedRoutes.kinds is empty", i, l.Name)
3723-
continue
3696+
// assertPort443Contract sweeps every port-443 listener in gw and
3697+
// verifies the three contracts above, reporting all failures via t.
3698+
assertPort443Contract := func(t *testing.T, gw *gatewayv1.Gateway) {
3699+
t.Helper()
3700+
var port443 []gatewayv1.Listener
3701+
for _, l := range gw.Spec.Listeners {
3702+
if l.Port == 443 {
3703+
port443 = append(port443, l)
3704+
}
37243705
}
3725-
got := kindsKey(l.AllowedRoutes.Kinds)
3726-
if i == 0 {
3727-
referenceKinds = got
3728-
} else if !reflect.DeepEqual(got, referenceKinds) {
3729-
t.Errorf("listener[%d] %q kinds %v differ from listener[0] kinds %v (cilium#45559: divergent kinds collapse listeners)", i, l.Name, got, referenceKinds)
3706+
if len(port443) < 2 {
3707+
t.Fatalf("expected at least 2 port-443 listeners (terminate + passthrough), got %d: %+v", len(port443), gw.Spec.Listeners)
37303708
}
3731-
// No kind outside HTTPRoute/TLSRoute should appear.
3732-
if !reflect.DeepEqual(got, allowedKinds) {
3733-
t.Errorf("listener[%d] %q: expected kinds %v, got %v", i, l.Name, allowedKinds, got)
3709+
3710+
var referenceKey []string
3711+
for i, l := range port443 {
3712+
// Contract 3: non-empty/non-nil.
3713+
if l.AllowedRoutes == nil || len(l.AllowedRoutes.Kinds) == 0 {
3714+
t.Errorf("listener[%d] %q: allowedRoutes.kinds is nil/empty — Gateway API defaults to all kinds, which bypasses the route-hostname VAP", i, l.Name)
3715+
continue
3716+
}
3717+
3718+
got := kindsKey(l.AllowedRoutes.Kinds)
3719+
3720+
// Contract 1: identical across all port-443 listeners.
3721+
if i == 0 {
3722+
referenceKey = got
3723+
} else if !reflect.DeepEqual(got, referenceKey) {
3724+
t.Errorf("listener[%d] %q kinds %v differ from listener[0] kinds %v — divergent kinds re-trigger cilium#45559 listener collapse", i, l.Name, got, referenceKey)
3725+
}
3726+
3727+
// Contract 1+2: must equal the canonical set exactly.
3728+
if !reflect.DeepEqual(got, canonicalPort443Kinds) {
3729+
t.Errorf("listener[%d] %q: got kinds %v, want canonical %v", i, l.Name, got, canonicalPort443Kinds)
3730+
}
3731+
3732+
// Contract 2: forbidden kinds must never appear by name.
3733+
kindSet := map[string]bool{}
3734+
for _, k := range l.AllowedRoutes.Kinds {
3735+
kindSet[string(k.Kind)] = true
3736+
}
3737+
for _, forbidden := range []string{"GRPCRoute", "TCPRoute", "UDPRoute"} {
3738+
if kindSet[forbidden] {
3739+
t.Errorf("listener[%d] %q: forbidden kind %q present — would bypass cozystack-route-hostname-policy VAP", i, l.Name, forbidden)
3740+
}
3741+
}
37343742
}
37353743
}
3744+
3745+
cases := []struct {
3746+
name string
3747+
tgw *gatewayv1alpha1.TenantGateway
3748+
objects []client.Object
3749+
}{
3750+
{
3751+
// HTTP-01: per-app HTTPS listener driven by attached HTTPRoute,
3752+
// plus TLS-passthrough for "api". Two port-443 listeners total.
3753+
name: "HTTP-01",
3754+
tgw: &gatewayv1alpha1.TenantGateway{
3755+
ObjectMeta: metav1.ObjectMeta{Name: "cozystack", Namespace: "tenant-foo"},
3756+
Spec: gatewayv1alpha1.TenantGatewaySpec{
3757+
Apex: "foo.example.com",
3758+
CertMode: gatewayv1alpha1.CertModeHTTP01,
3759+
GatewayClassName: "cilium",
3760+
AttachedNamespaces: []string{"cozy-harbor"},
3761+
TLSPassthroughServices: []string{"api"},
3762+
},
3763+
},
3764+
objects: []client.Object{
3765+
httpRouteAttached("harbor", "cozy-harbor", "harbor.foo.example.com"),
3766+
},
3767+
},
3768+
{
3769+
// DNS-01: wildcard + apex + per-child-apex HTTPS listeners rendered
3770+
// from the wildcard cert, plus TLS-passthrough for "api". The child
3771+
// namespace (tenant-foo-alice, with namespace.cozystack.io/gateway
3772+
// and namespace.cozystack.io/host labels) causes collectInheritingChildApexes
3773+
// to emit a *.alice.foo.example.com listener, giving us 4 port-443 listeners
3774+
// in total — the broadest surface for this regression.
3775+
name: "DNS-01",
3776+
tgw: &gatewayv1alpha1.TenantGateway{
3777+
ObjectMeta: metav1.ObjectMeta{Name: "cozystack", Namespace: "tenant-foo"},
3778+
Spec: gatewayv1alpha1.TenantGatewaySpec{
3779+
Apex: "foo.example.com",
3780+
CertMode: gatewayv1alpha1.CertModeDNS01,
3781+
GatewayClassName: "cilium",
3782+
TLSPassthroughServices: []string{"api"},
3783+
DNS01: &gatewayv1alpha1.DNS01Config{
3784+
Provider: "cloudflare",
3785+
Cloudflare: &gatewayv1alpha1.CloudflareDNS01{
3786+
APITokenSecretRef: corev1.SecretKeySelector{
3787+
LocalObjectReference: corev1.LocalObjectReference{Name: "cf-token"},
3788+
Key: "api-token",
3789+
},
3790+
},
3791+
},
3792+
},
3793+
},
3794+
objects: []client.Object{
3795+
// Helm-owned labels on own + child namespaces so
3796+
// collectInheritingChildApexes picks up the child apex.
3797+
&corev1.Namespace{
3798+
ObjectMeta: metav1.ObjectMeta{
3799+
Name: "tenant-foo",
3800+
Labels: map[string]string{
3801+
"namespace.cozystack.io/host": "foo.example.com",
3802+
"namespace.cozystack.io/gateway": "tenant-foo",
3803+
},
3804+
},
3805+
},
3806+
&corev1.Namespace{
3807+
ObjectMeta: metav1.ObjectMeta{
3808+
Name: "tenant-foo-alice",
3809+
Labels: map[string]string{
3810+
"namespace.cozystack.io/host": "alice.foo.example.com",
3811+
"namespace.cozystack.io/gateway": "tenant-foo",
3812+
},
3813+
},
3814+
},
3815+
// Route in child namespace — seeds a realistic scenario even
3816+
// though DNS-01 collectHostnameClaims returns nil (wildcard
3817+
// handles all hostnames).
3818+
httpRouteAttached("harbor", "tenant-foo-alice", "harbor.alice.foo.example.com"),
3819+
},
3820+
},
3821+
{
3822+
// existingSecret: same listener topology as DNS-01 but referencing
3823+
// the operator-supplied wildcard Secret. The child namespace produces
3824+
// the same *.alice.foo.example.com listener for 4 port-443 listeners.
3825+
name: "existingSecret",
3826+
tgw: &gatewayv1alpha1.TenantGateway{
3827+
ObjectMeta: metav1.ObjectMeta{Name: "cozystack", Namespace: "tenant-foo"},
3828+
Spec: gatewayv1alpha1.TenantGatewaySpec{
3829+
Apex: "foo.example.com",
3830+
CertMode: gatewayv1alpha1.CertModeExistingSecret,
3831+
GatewayClassName: "cilium",
3832+
TLSPassthroughServices: []string{"api"},
3833+
WildcardSecretRef: &corev1.LocalObjectReference{Name: "wildcard-tls"},
3834+
},
3835+
},
3836+
objects: []client.Object{
3837+
&corev1.Namespace{
3838+
ObjectMeta: metav1.ObjectMeta{
3839+
Name: "tenant-foo",
3840+
Labels: map[string]string{
3841+
"namespace.cozystack.io/host": "foo.example.com",
3842+
"namespace.cozystack.io/gateway": "tenant-foo",
3843+
},
3844+
},
3845+
},
3846+
&corev1.Namespace{
3847+
ObjectMeta: metav1.ObjectMeta{
3848+
Name: "tenant-foo-alice",
3849+
Labels: map[string]string{
3850+
"namespace.cozystack.io/host": "alice.foo.example.com",
3851+
"namespace.cozystack.io/gateway": "tenant-foo",
3852+
},
3853+
},
3854+
},
3855+
httpRouteAttached("harbor", "tenant-foo-alice", "harbor.alice.foo.example.com"),
3856+
},
3857+
},
3858+
}
3859+
3860+
for _, tc := range cases {
3861+
t.Run(tc.name, func(t *testing.T) {
3862+
s := newScheme(t)
3863+
builder := fake.NewClientBuilder().
3864+
WithScheme(s).
3865+
WithObjects(tc.tgw).
3866+
WithStatusSubresource(tc.tgw)
3867+
if len(tc.objects) > 0 {
3868+
builder = builder.WithObjects(tc.objects...)
3869+
}
3870+
c := builder.Build()
3871+
3872+
r := &Reconciler{Client: c, Scheme: s}
3873+
if _, err := r.Reconcile(context.TODO(), ctrl.Request{
3874+
NamespacedName: types.NamespacedName{Name: "cozystack", Namespace: "tenant-foo"},
3875+
}); err != nil {
3876+
t.Fatalf("unexpected error: %v", err)
3877+
}
3878+
3879+
gw := &gatewayv1.Gateway{}
3880+
if err := c.Get(context.TODO(), types.NamespacedName{Name: "cozystack", Namespace: "tenant-foo"}, gw); err != nil {
3881+
t.Fatalf("get Gateway: %v", err)
3882+
}
3883+
3884+
assertPort443Contract(t, gw)
3885+
})
3886+
}
37363887
}
37373888

37383889
// TestReconcile_ExistingSecretModeRendersChildApexListenerWithOperatorSecret

0 commit comments

Comments
 (0)