Skip to content

Commit f200630

Browse files
authored
Add E2E HTTPS egress tests and reject WebSocket upgrades (#1634)
Addresses #1017 by adding E2E tests for HTTPS and WebSocket egress. Added scope after the egress policy was finalized: [review comment](#1634 (comment)). Updates the gateways to reject HTTP/1.1 WebSocket upgrades with 403. PRs merge order/stack: #1945 #1944 and then this PR
1 parent fd7a9cd commit f200630

31 files changed

Lines changed: 2566 additions & 111 deletions

‎.github/workflows/pr-workflow.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -252,6 +252,8 @@ jobs:
252252
run: |
253253
hack/install-ate-kind.sh --deploy-demo-egress
254254
hack/install-ate-kind.sh --deploy-demo-egress-microvm
255+
- name: Trust local TLS origins in egress gateway
256+
run: hack/setup-e2e-egress-tls-kind.sh
255257
# scale the idle sandbox class's demo pools to zero so each lane has the node
256258
- name: Focus the node on gVisor
257259
run: hack/e2e-focus-sandbox-class.sh gvisor

‎cmd/ate-setup/internal/steps/overlay_test.go‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -651,9 +651,20 @@ func TestRenderAgentgatewayCredentialProvider(t *testing.T) {
651651
t.Fatal(err)
652652
}
653653
for _, protocol := range []string{"HTTP", "HTTPS"} {
654-
policy, err := gateway.Pipe(kyaml.Lookup("binds", "[mode=internal]", "listeners", "[protocol="+protocol+"]", "routes", "0", "policies", "substrateEgress"))
655-
if err != nil || policy == nil {
656-
t.Fatalf("%s egress policy missing: %v", protocol, err)
654+
routes, err := gateway.Pipe(kyaml.Lookup("binds", "[mode=internal]", "listeners", "[protocol="+protocol+"]", "routes"))
655+
if err != nil || routes == nil {
656+
t.Fatalf("%s routes missing: %v", protocol, err)
657+
}
658+
var policy *kyaml.RNode
659+
for _, route := range routes.YNode().Content {
660+
candidate, err := kyaml.NewRNode(route).Pipe(kyaml.Lookup("policies", "substrateEgress"))
661+
if err == nil && candidate != nil {
662+
policy = candidate
663+
break
664+
}
665+
}
666+
if policy == nil {
667+
t.Fatalf("%s egress policy missing", protocol)
657668
}
658669
var got struct {
659670
Providers []map[string]any `yaml:"credentialProviders"`

‎cmd/atenet/internal/router/egress/request.go‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import (
1919
"errors"
2020
"fmt"
2121
"log/slog"
22+
"strings"
2223

2324
extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3"
2425
envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3"
@@ -28,6 +29,8 @@ import (
2829
"github.com/agent-substrate/substrate/internal/resources"
2930
)
3031

32+
const websocketDeniedBody = "WebSocket egress is not supported"
33+
3134
// handleRequest authorizes one request the gateway can read: cleartext HTTP,
3235
// or HTTPS the gateway terminated. It runs per request, because the
3336
// Host can change between requests on one connection.
@@ -38,6 +41,11 @@ import (
3841
// request is dialed by the name it was decided on, on the port the actor
3942
// dialed.
4043
func (h *Handler) handleRequest(ctx context.Context, md *extproc.RequestMetadata, leg string) (extproc.Result, error) {
44+
if isWebSocketUpgrade(md) {
45+
slog.WarnContext(ctx, "egress denied: WebSocket upgrades are not supported", slog.String("leg", leg), slog.String("host", md.Host))
46+
return extproc.Result{}, extproc.NewReqError(envoy_type.StatusCode_Forbidden, websocketDeniedBody)
47+
}
48+
4149
ref, err := actorFromFilterState(md)
4250
if err != nil {
4351
slog.WarnContext(ctx, "egress denied: request carries no actor identity", slog.String("leg", leg), slog.Any("err", err))
@@ -121,6 +129,19 @@ func dialHost(dest egresspolicy.Destination) string {
121129
}
122130
}
123131

132+
func isWebSocketUpgrade(md *extproc.RequestMetadata) bool {
133+
return md.Method == "GET" && hasHTTPToken(md.Header("connection"), "upgrade") && hasHTTPToken(md.Header("upgrade"), "websocket")
134+
}
135+
136+
func hasHTTPToken(value, want string) bool {
137+
for _, token := range strings.Split(value, ",") {
138+
if strings.EqualFold(strings.TrimSpace(token), want) {
139+
return true
140+
}
141+
}
142+
return false
143+
}
144+
124145
// connectionDestination is the SNI a decrypted connection presented, on the
125146
// port the actor dialed, in the form the https rules are matched against. An
126147
// SNI that is not a DNS name, or none at all, matches nothing.

‎cmd/atenet/internal/router/egress/request_test.go‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -328,6 +328,45 @@ func TestRequestLegPicksRulesByLeg(t *testing.T) {
328328
}
329329
}
330330

331+
func TestRequestLegDeniesHTTP1WebSocketUpgrade(t *testing.T) {
332+
h := policyHandler(allowAllPolicy())
333+
for _, leg := range []string{extproc.EgressCleartextFilterChainName, extproc.EgressTLSMITMFilterChainName} {
334+
t.Run(leg, func(t *testing.T) {
335+
md := innerMetadata(leg, "GET", "api.example.com", nil)
336+
md.Headers["connection"] = "keep-alive, UpGrAdE"
337+
md.Headers["upgrade"] = "WebSocket"
338+
_, err := h.HandleRequestHeaders(context.Background(), md)
339+
wantStatus(t, err, envoy_type.StatusCode_Forbidden)
340+
if err.Error() != websocketDeniedBody {
341+
t.Errorf("denial body = %q, want %q", err, websocketDeniedBody)
342+
}
343+
})
344+
}
345+
}
346+
347+
func TestRequestLegAllowsOrdinaryHTTPSRequest(t *testing.T) {
348+
h := policyHandler(allowAllPolicy())
349+
for _, tc := range []struct {
350+
name string
351+
method string
352+
connection string
353+
upgrade string
354+
}{
355+
{name: "ordinary GET", method: "GET"},
356+
{name: "other upgrade", method: "GET", connection: "Upgrade", upgrade: "h2c"},
357+
{name: "upgrade without connection token", method: "GET", connection: "keep-alive", upgrade: "websocket"},
358+
{name: "non-GET", method: "POST", connection: "Upgrade", upgrade: "websocket"},
359+
} {
360+
t.Run(tc.name, func(t *testing.T) {
361+
md := innerMetadata(extproc.EgressTLSMITMFilterChainName, tc.method, "api.example.com", nil)
362+
md.Headers["connection"] = tc.connection
363+
md.Headers["upgrade"] = tc.upgrade
364+
res, err := h.HandleRequestHeaders(context.Background(), md)
365+
wantDial(t, res, err, extproc.EgressDialName)
366+
})
367+
}
368+
}
369+
331370
// Without the identity the outer chain shares, a request cannot be attributed
332371
// to any actor and is refused.
333372
func TestRequestLegRequiresIdentity(t *testing.T) {

‎cmd/atenet/internal/router/egresspolicymanifest_test.go‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -192,6 +192,67 @@ func TestEgressManifestsNameEveryExtProcChain(t *testing.T) {
192192
}
193193
}
194194

195+
func TestEgressManifestsDisableWebSocketUpgrades(t *testing.T) {
196+
want := map[string][]string{
197+
egressManifests[0]: {extproc.EgressCleartextFilterChainName, extproc.EgressTLSMITMFilterChainName},
198+
}
199+
for _, path := range egressManifests {
200+
t.Run(path, func(t *testing.T) {
201+
tree := bootstrapTree(t, path)
202+
chains := map[string]node{}
203+
for _, lc := range allChains(tree) {
204+
name := str(lc.chain, "name")
205+
if slices.Contains(requestLegs, name) {
206+
chains[name] = lc.chain
207+
}
208+
}
209+
for _, name := range want[path] {
210+
chain, present := chains[name]
211+
if !present {
212+
t.Errorf("expected request-leg chain %q is absent", name)
213+
continue
214+
}
215+
var disabled bool
216+
for _, upgrade := range list(hcm(chain), "upgrade_configs") {
217+
if str(upgrade, "upgrade_type") == "websocket" {
218+
enabled, ok := upgrade["enabled"].(bool)
219+
disabled = ok && !enabled
220+
break
221+
}
222+
}
223+
if !disabled {
224+
t.Errorf("chain %q must explicitly disable websocket upgrades", name)
225+
}
226+
var routes []node
227+
for _, virtualHost := range list(child(hcm(chain), "route_config"), "virtual_hosts") {
228+
routes = append(routes, list(virtualHost, "routes")...)
229+
}
230+
if len(routes) == 0 {
231+
t.Errorf("chain %q has no routes", name)
232+
}
233+
for i, route := range routes {
234+
action := child(route, "route")
235+
if action == nil {
236+
t.Errorf("chain %q route %d has no route action", name, i)
237+
continue
238+
}
239+
disabled := false
240+
for _, upgrade := range list(action, "upgrade_configs") {
241+
if str(upgrade, "upgrade_type") == "websocket" {
242+
enabled, ok := upgrade["enabled"].(bool)
243+
disabled = ok && !enabled
244+
break
245+
}
246+
}
247+
if !disabled {
248+
t.Errorf("chain %q route %d must explicitly disable websocket upgrades", name, i)
249+
}
250+
}
251+
}
252+
})
253+
}
254+
}
255+
195256
// extProcOf returns the ext_proc filter config of a chain's HCM and its index
196257
// among the http_filters, or nil and -1.
197258
func extProcOf(chain node) (node, int, []node) {
Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
// Copyright 2026 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package extproc_test
16+
17+
import (
18+
"context"
19+
"io"
20+
"testing"
21+
22+
corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
23+
extprocv3 "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3"
24+
"google.golang.org/protobuf/types/known/structpb"
25+
26+
"github.com/agent-substrate/substrate/cmd/atenet/internal/router/egress"
27+
"github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc"
28+
)
29+
30+
const websocketDenialBody = "WebSocket egress is not supported"
31+
32+
type websocketProcessStream struct {
33+
extprocv3.ExternalProcessor_ProcessServer
34+
ctx context.Context
35+
requests []*extprocv3.ProcessingRequest
36+
responses []*extprocv3.ProcessingResponse
37+
}
38+
39+
func (s *websocketProcessStream) Context() context.Context { return s.ctx }
40+
41+
func (s *websocketProcessStream) Recv() (*extprocv3.ProcessingRequest, error) {
42+
if len(s.requests) == 0 {
43+
return nil, io.EOF
44+
}
45+
req := s.requests[0]
46+
s.requests = s.requests[1:]
47+
return req, nil
48+
}
49+
50+
func (s *websocketProcessStream) Send(resp *extprocv3.ProcessingResponse) error {
51+
s.responses = append(s.responses, resp)
52+
return nil
53+
}
54+
55+
func TestProcessReturnsWebSocketDenialAsImmediateResponse(t *testing.T) {
56+
for _, leg := range []string{extproc.EgressCleartextFilterChainName, extproc.EgressTLSMITMFilterChainName} {
57+
t.Run(leg, func(t *testing.T) {
58+
request := &extprocv3.ProcessingRequest{
59+
Request: &extprocv3.ProcessingRequest_RequestHeaders{
60+
RequestHeaders: &extprocv3.HttpHeaders{
61+
Headers: &corev3.HeaderMap{Headers: []*corev3.HeaderValue{
62+
{Key: ":method", RawValue: []byte("GET")},
63+
{Key: ":authority", RawValue: []byte("api.example.com")},
64+
{Key: "connection", RawValue: []byte("keep-alive, Upgrade")},
65+
{Key: "upgrade", RawValue: []byte("websocket")},
66+
}},
67+
},
68+
},
69+
Attributes: map[string]*structpb.Struct{
70+
"envoy.filters.http.ext_proc": {Fields: map[string]*structpb.Value{
71+
extproc.FilterChainNameAttribute: structpb.NewStringValue(leg),
72+
}},
73+
},
74+
}
75+
stream := &websocketProcessStream{ctx: context.Background(), requests: []*extprocv3.ProcessingRequest{request}}
76+
server := extproc.NewServer(0, nil, extproc.Handlers{
77+
extproc.DirectionEgress: egress.New(nil, nil, 0, nil, ""),
78+
})
79+
80+
if err := server.Process(stream); err != nil {
81+
t.Fatalf("Process() error = %v", err)
82+
}
83+
if len(stream.responses) != 1 {
84+
t.Fatalf("responses = %d, want 1", len(stream.responses))
85+
}
86+
response := stream.responses[0].GetImmediateResponse()
87+
if response == nil {
88+
t.Fatalf("response = %v, want immediate denial", stream.responses[0])
89+
}
90+
if got := response.GetStatus().GetCode(); got != 403 {
91+
t.Errorf("status = %d, want 403", got)
92+
}
93+
if got := string(response.GetBody()); got != websocketDenialBody {
94+
t.Errorf("body = %q, want %q", got, websocketDenialBody)
95+
}
96+
})
97+
}
98+
}

‎demos/egress/README.md‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -77,8 +77,9 @@ rejects that option with agentgateway rather than silently omitting it.
7777

7878
## Components
7979

80-
- **Egress app (`main.go`)** — the Actor: `POST /` with `{"url":"..."}` → fetches it → returns
81-
status + body. It also serves `POST /grpc`, described below.
80+
- **Egress app (`main.go`)** — the Actor: `POST /` fetches an HTTP(S) URL and returns the status,
81+
body, and observed protocol. `POST /websocket` probes a `ws` or `wss` origin; `POST /grpc` makes
82+
unary and streaming gRPC calls, described below.
8283
- **Egress gateway** — the `atenet-egress` Deployment. Envoy uses a co-located atenet `ext_proc`
8384
container started with `--mode=egress`; agentgateway uses its built-in `substrateEgress` policy
8485
and does not need that sidecar. The installer renders the matching configuration and container.
@@ -132,6 +133,14 @@ then asserts:
132133

133134
Add `--cleanup` to remove everything the script created.
134135

136+
### HTTPS and WebSocket E2E tests
137+
138+
Both supported egress dataplanes intercept HTTPS policy traffic. The Actor
139+
trusts the gateway through its projected CA bundle; the gateway verifies each
140+
local test origin against the service-DNS CA. The [egress protocol test
141+
runbook](../../docs/dev/mitm-egress-test-runbook.md) sets up this trust on Kind
142+
and runs the HTTP, HTTPS, WebSocket, and delayed-restore tests.
143+
135144
## Manual walkthrough
136145

137146
```bash

0 commit comments

Comments
 (0)