Skip to content

Commit 62efb70

Browse files
authored
atenet-router no longer stalls 15s on every start (agent-substrate#3346)
<!-- The PR title becomes the release note. Write it for users. --> Fixes agent-substrate#3268 ### Problem Envoy asks for config in a fixed order: clusters, then the 2 secrets the clusters need, then listeners, then the 2 secrets the listeners need. It won't ask for listeners until every cluster is ready. Over state-of-the-world xDS, the router's ADS cache [answers a request that lists names only if it names every resource](https://github.com/envoyproxy/go-control-plane/blob/71637ad69bbc5f51fbb2562e612a4365292804a5/pkg/cache/v3/simple.go#L491). Envoy's first secret request names only the 2 cluster secrets, so it was ignored. Envoy waited out its 15s `initial_fetch_timeout`, marked the cluster ready anyway, then asked for listeners and all 4 secrets. With one replica, that meant 15s with no ingress on every start. ### Solution The router Envoy now uses delta xDS (`api_type: DELTA_GRPC`). The delta path in go-control-plane has no such check, and with `ads=true` it still orders updates across resource types, which we'll want if the router ever serves dynamic config. ### Result With real Envoy v1.39.5, startup goes from 15.3s to 0.35s, with no timeout. `TestXdsServer_AnswersClusterSecretsBeforeListeners` checks the manifest uses delta and replays the startup over a delta stream. It fails on main. AI use: AI-assisted. I reviewed the change and ran it against real Envoy. - [x] Tests pass - [x] Docs updated (none needed) Signed-off-by: Chuang Wang <[email protected]>
1 parent 5d226a5 commit 62efb70

2 files changed

Lines changed: 92 additions & 1 deletion

File tree

‎cmd/atenet/internal/router/xds_test.go‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,12 +27,14 @@ import (
2727
"net"
2828
"os"
2929
"path/filepath"
30+
"slices"
3031
"strings"
3132
"testing"
3233
"time"
3334

3435
"google.golang.org/grpc"
3536
"google.golang.org/grpc/credentials/insecure"
37+
"sigs.k8s.io/yaml"
3638

3739
clusterv3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
3840
corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
@@ -618,6 +620,95 @@ func TestXdsServer_ServesSecretOverSds(t *testing.T) {
618620
}
619621
}
620622

623+
// TestXdsServer_AnswersClusterSecretsBeforeListeners replays Envoy's startup
624+
// over delta xDS, which the router manifest must configure. While warming
625+
// clusters, Envoy subscribes only to the secrets the clusters reference, and
626+
// asks for listeners (and their secrets) only after. The server has to answer
627+
// that partial secret request, or Envoy stalls until its 15s
628+
// initial_fetch_timeout.
629+
func TestXdsServer_AnswersClusterSecretsBeforeListeners(t *testing.T) {
630+
var bootstrap struct {
631+
DynamicResources struct {
632+
AdsConfig struct {
633+
APIType string `json:"api_type"`
634+
} `json:"ads_config"`
635+
} `json:"dynamic_resources"`
636+
}
637+
if err := yaml.Unmarshal([]byte(envoyConfigFrom(t, routerManifestPath, "atenet-router-envoy-config")), &bootstrap); err != nil {
638+
t.Fatalf("parsing envoy.yaml: %v", err)
639+
}
640+
if got := bootstrap.DynamicResources.AdsConfig.APIType; got != "DELTA_GRPC" {
641+
t.Fatalf("Router Envoy ads_config.api_type is %q; it must be DELTA_GRPC, since the state-of-the-world ADS cache ignores a partial secret request", got)
642+
}
643+
644+
server := NewXdsServer(18000)
645+
server.SetConfig(50053, "127.0.0.1")
646+
server.SetTlsConfig(8443, "/run/servicedns.podcert.ate.dev/credential-bundle.pem")
647+
server.SetDownstreamClientAuth("/run/podidentity.podcert.ate.dev/trust-bundle.pem", []string{"spiffe://cluster.local/ns/demo/sa/client"})
648+
server.SetUpstreamTls("/run/podidentity.podcert.ate.dev/credential-bundle.pem", "/run/podidentity.podcert.ate.dev/trust-bundle.pem")
649+
server.SetUpstreamConnect(true)
650+
651+
lis, err := net.Listen("tcp", "127.0.0.1:0")
652+
if err != nil {
653+
t.Fatalf("Failed to create tcp listener: %v", err)
654+
}
655+
ctx, cancel := context.WithCancel(context.Background())
656+
t.Cleanup(cancel)
657+
go server.Serve(ctx, lis)
658+
659+
conn, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
660+
if err != nil {
661+
t.Fatalf("Failed to dial xDS server: %v", err)
662+
}
663+
t.Cleanup(func() { conn.Close() })
664+
665+
streamCtx, streamCancel := context.WithTimeout(ctx, 10*time.Second)
666+
t.Cleanup(streamCancel)
667+
stream, err := discoverygrpc.NewAggregatedDiscoveryServiceClient(conn).DeltaAggregatedResources(streamCtx)
668+
if err != nil {
669+
t.Fatalf("Failed to open delta ADS stream: %v", err)
670+
}
671+
672+
if err := stream.Send(&discoverygrpc.DeltaDiscoveryRequest{
673+
Node: &corev3.Node{Id: NodeID},
674+
TypeUrl: resourcev3.ClusterType,
675+
}); err != nil {
676+
t.Fatalf("Failed to send CDS request: %v", err)
677+
}
678+
cds, err := stream.Recv()
679+
if err != nil {
680+
t.Fatalf("Failed to receive CDS response: %v", err)
681+
}
682+
if got := cds.GetTypeUrl(); got != resourcev3.ClusterType {
683+
t.Fatalf("Expected a CDS response, got %s", got)
684+
}
685+
686+
clusterSecrets := []string{UpstreamCertSecretName, UpstreamTrustSecretName}
687+
if err := stream.Send(&discoverygrpc.DeltaDiscoveryRequest{
688+
TypeUrl: resourcev3.SecretType,
689+
ResourceNamesSubscribe: clusterSecrets,
690+
}); err != nil {
691+
t.Fatalf("Failed to send SDS request: %v", err)
692+
}
693+
// A server that ignores the request leaves Recv blocked until streamCtx
694+
// expires.
695+
sds, err := stream.Recv()
696+
if err != nil {
697+
t.Fatalf("No SDS response for the cluster secrets: %v", err)
698+
}
699+
if got := sds.GetTypeUrl(); got != resourcev3.SecretType {
700+
t.Fatalf("Expected an SDS response, got %s", got)
701+
}
702+
var names []string
703+
for _, r := range sds.GetResources() {
704+
names = append(names, r.GetName())
705+
}
706+
slices.Sort(names)
707+
if want := slices.Sorted(slices.Values(clusterSecrets)); !slices.Equal(names, want) {
708+
t.Errorf("Expected SDS response with secrets %v, got %v", want, names)
709+
}
710+
}
711+
621712
// Symlink names used by kubelet's AtomicWriter in projected volumes.
622713
const (
623714
dataDirName = "..data"

‎manifests/ate-install/atenet-router.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ data:
8989
resource_api_version: V3
9090
ads: {}
9191
ads_config:
92-
api_type: GRPC
92+
api_type: DELTA_GRPC
9393
transport_api_version: V3
9494
grpc_services:
9595
- envoy_grpc:

0 commit comments

Comments
 (0)