Skip to content

Commit e60a76b

Browse files
committed
fix(controller): fail opted-out RemoteMCPServer on bad CA or store error
With kagent.dev/discovery=disabled the reconciler now rejects what discovery would have rejected before dialing: a CA bundle without a valid certificate fails the server instead of looking healthy. A failure to persist the empty inventory sets Accepted=False and clears the stale discoveredTools, as the discovery error path does. The status message and the label's documentation state that DiscoveryDisabled does not validate runtime authentication or reachability, and that a spec.headersFrom Authorization entry overrides the propagated caller credential and may need an explicit `headersFrom: []` under GitOps tooling. Signed-off-by: Timo Derstappen <[email protected]>
1 parent db23850 commit e60a76b

3 files changed

Lines changed: 122 additions & 14 deletions

File tree

‎go/core/internal/controller/reconciler/reconciler.go‎

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -692,14 +692,21 @@ func (a *kagentReconciler) ReconcileKagentRemoteMCPServer(ctx context.Context, r
692692
// Accept it without connecting: the status carries no tools, the DB keeps
693693
// the server with an empty inventory, and agents resolve the tool list at
694694
// run time with the credentials they carry. The TLS Secret hash is still
695-
// published (agents fold it into their rollout hash), and a broken
696-
// spec.tls Secret reference still fails the server.
695+
// published (agents fold it into their rollout hash), and the server
696+
// still fails on what discovery would have rejected before dialing: a
697+
// broken spec.tls Secret reference or a CA bundle without a valid
698+
// certificate. A failure to persist the empty inventory fails it too,
699+
// with the stale tools cleared from the status.
697700
l.Info("skipping tool discovery for remote MCP server", "url", server.Spec.URL, "label", consts.DiscoveryLabel)
698-
if err := a.storeToolServerWithoutTools(ctx, dbServer); err != nil {
699-
return err
701+
err := secretErr
702+
if err == nil {
703+
_, err = a.buildRemoteMCPServerTLSConfig(ctx, server)
704+
}
705+
if storeErr := a.storeToolServerWithoutTools(ctx, dbServer); storeErr != nil {
706+
err = multierror.Append(err, storeErr)
700707
}
701-
if secretErr != nil {
702-
if err := a.reconcileRemoteMCPServerStatus(ctx, server, nil, secretHash, secretErr); err != nil {
708+
if err != nil {
709+
if err := a.reconcileRemoteMCPServerStatus(ctx, server, nil, secretHash, err); err != nil {
703710
return fmt.Errorf("failed to reconcile remote mcp server status %s: %w", req.NamespacedName, err)
704711
}
705712
return nil
@@ -776,7 +783,7 @@ func (a *kagentReconciler) computeRemoteMCPServerSecretHash(ctx context.Context,
776783
// remoteMCPServerDiscoveryDisabledMessage explains an Accepted RemoteMCPServer
777784
// that publishes no discovered tools because its operator opted out of discovery.
778785
const remoteMCPServerDiscoveryDisabledMessage = "Tool discovery is disabled by the " + consts.DiscoveryLabel + "=" + consts.DiscoveryDisabled +
779-
" label; agents resolve the tool list at run time"
786+
" label; agents resolve the tool list at run time. The controller did not connect: runtime authentication and reachability are not validated"
780787

781788
// remoteMCPServerDiscoveryDisabled reports whether the operator opted the server
782789
// out of tool discovery with the kagent.dev/discovery=disabled label.

‎go/core/internal/controller/reconciler/remote_mcp_server_discovery_test.go‎

Lines changed: 100 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,16 @@ package reconciler
22

33
import (
44
"context"
5+
"crypto/ecdsa"
6+
"crypto/elliptic"
7+
"crypto/rand"
8+
"crypto/x509"
9+
"crypto/x509/pkix"
10+
"encoding/pem"
11+
"errors"
12+
"math/big"
513
"testing"
14+
"time"
615

716
"github.com/stretchr/testify/assert"
817
"github.com/stretchr/testify/require"
@@ -49,6 +58,34 @@ func (s *recordingToolServerStore) RefreshToolsForServer(_ context.Context, serv
4958
return nil
5059
}
5160

61+
// failingToolServerStore fails the first catalog write the opt-out performs.
62+
type failingToolServerStore struct {
63+
database.Client
64+
}
65+
66+
func (failingToolServerStore) StoreToolServer(context.Context, *database.ToolServer) (*database.ToolServer, error) {
67+
return nil, errors.New("database unavailable")
68+
}
69+
70+
// selfSignedCAPEM returns a freshly generated, PEM-encoded CA certificate.
71+
func selfSignedCAPEM(t *testing.T) []byte {
72+
t.Helper()
73+
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
74+
require.NoError(t, err)
75+
template := &x509.Certificate{
76+
SerialNumber: big.NewInt(1),
77+
Subject: pkix.Name{CommonName: "test-ca"},
78+
NotBefore: time.Now().Add(-time.Hour),
79+
NotAfter: time.Now().Add(time.Hour),
80+
IsCA: true,
81+
BasicConstraintsValid: true,
82+
KeyUsage: x509.KeyUsageCertSign,
83+
}
84+
der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key)
85+
require.NoError(t, err)
86+
return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
87+
}
88+
5289
// TestReconcileKagentRemoteMCPServer_DiscoveryDisabled verifies that a
5390
// RemoteMCPServer labeled kagent.dev/discovery=disabled is Accepted without the
5491
// controller connecting to it: the URL below is unroutable, so any dial would
@@ -128,7 +165,8 @@ func TestReconcileKagentRemoteMCPServer_DiscoveryDisabled(t *testing.T) {
128165

129166
// TestReconcileKagentRemoteMCPServer_DiscoveryDisabledKeepsSecretHash verifies
130167
// that the opt-out still publishes the TLS Secret hash agents fold into their
131-
// rollout hash, and still fails the server on a broken spec.tls reference.
168+
// rollout hash, and still fails the server on a broken spec.tls reference or a
169+
// CA bundle discovery would reject.
132170
func TestReconcileKagentRemoteMCPServer_DiscoveryDisabledKeepsSecretHash(t *testing.T) {
133171
newServer := func() *v1alpha2.RemoteMCPServer {
134172
return &v1alpha2.RemoteMCPServer{
@@ -146,22 +184,29 @@ func TestReconcileKagentRemoteMCPServer_DiscoveryDisabledKeepsSecretHash(t *test
146184
},
147185
}
148186
}
149-
caSecret := &corev1.Secret{
150-
ObjectMeta: metav1.ObjectMeta{Name: "ca", Namespace: "default"},
151-
Data: map[string][]byte{"ca.crt": []byte("-----BEGIN CERTIFICATE-----")},
187+
caSecret := func(data []byte) *corev1.Secret {
188+
return &corev1.Secret{
189+
ObjectMeta: metav1.ObjectMeta{Name: "ca", Namespace: "default"},
190+
Data: map[string][]byte{"ca.crt": data},
191+
}
152192
}
153193

154194
for name, tc := range map[string]struct {
155195
objects []client.Object
156196
wantAccepted metav1.ConditionStatus
157197
wantReason string
158198
wantHash bool
199+
wantMessage string
159200
}{
160201
"CA Secret present: Accepted, hash published": {
161-
objects: []client.Object{caSecret}, wantAccepted: metav1.ConditionTrue, wantReason: "DiscoveryDisabled", wantHash: true,
202+
objects: []client.Object{caSecret(selfSignedCAPEM(t))}, wantAccepted: metav1.ConditionTrue, wantReason: "DiscoveryDisabled", wantHash: true,
162203
},
163204
"CA Secret missing: the broken reference still fails the server": {
164-
wantAccepted: metav1.ConditionFalse, wantReason: "ReconcileFailed",
205+
wantAccepted: metav1.ConditionFalse, wantReason: "ReconcileFailed", wantMessage: "failed to get TLS secret",
206+
},
207+
"CA bundle without a valid certificate: rejected as discovery would": {
208+
objects: []client.Object{caSecret([]byte("-----BEGIN CERTIFICATE-----"))}, wantAccepted: metav1.ConditionFalse, wantReason: "ReconcileFailed",
209+
wantHash: true, wantMessage: "does not contain valid PEM certificates",
165210
},
166211
} {
167212
t.Run(name, func(t *testing.T) {
@@ -191,13 +236,61 @@ func TestReconcileKagentRemoteMCPServer_DiscoveryDisabledKeepsSecretHash(t *test
191236
assert.NotEmpty(t, updated.Status.SecretHash, "the TLS Secret hash is published although discovery is disabled")
192237
} else {
193238
assert.Empty(t, updated.Status.SecretHash)
194-
assert.Contains(t, accepted.Message, "failed to get TLS secret")
239+
}
240+
if tc.wantMessage != "" {
241+
assert.Contains(t, accepted.Message, tc.wantMessage)
195242
}
196243
require.NotNil(t, store.stored, "the server is stored even when its TLS reference is broken")
197244
})
198245
}
199246
}
200247

248+
// TestReconcileKagentRemoteMCPServer_DiscoveryDisabledStoreFailure verifies
249+
// that a server switched to discovery disabled is not left Accepted with its
250+
// old inventory when the empty catalog cannot be persisted.
251+
func TestReconcileKagentRemoteMCPServer_DiscoveryDisabledStoreFailure(t *testing.T) {
252+
scheme := runtime.NewScheme()
253+
require.NoError(t, clientgoscheme.AddToScheme(scheme))
254+
require.NoError(t, v1alpha2.AddToScheme(scheme))
255+
256+
server := &v1alpha2.RemoteMCPServer{
257+
ObjectMeta: metav1.ObjectMeta{
258+
Name: "switched-off",
259+
Namespace: "default",
260+
Generation: 3,
261+
Labels: map[string]string{consts.DiscoveryLabel: consts.DiscoveryDisabled},
262+
},
263+
Spec: v1alpha2.RemoteMCPServerSpec{
264+
URL: "http://192.0.2.1:1/mcp",
265+
Protocol: v1alpha2.RemoteMCPServerProtocolStreamableHttp,
266+
},
267+
Status: v1alpha2.RemoteMCPServerStatus{
268+
DiscoveredTools: []*v1alpha2.MCPTool{{Name: "discovered-before-the-switch"}},
269+
Conditions: []metav1.Condition{{
270+
Type: v1alpha2.AgentConditionTypeAccepted, Status: metav1.ConditionTrue, Reason: "Reconciled", Message: "Remote MCP server configuration accepted",
271+
}},
272+
},
273+
}
274+
kube := fake.NewClientBuilder().
275+
WithScheme(scheme).
276+
WithStatusSubresource(server).
277+
WithObjects(server).
278+
Build()
279+
reconciler := &kagentReconciler{kube: kube, dbClient: failingToolServerStore{}}
280+
281+
require.NoError(t, reconciler.ReconcileKagentRemoteMCPServer(context.Background(),
282+
reconcile.Request{NamespacedName: client.ObjectKeyFromObject(server)}))
283+
284+
updated := &v1alpha2.RemoteMCPServer{}
285+
require.NoError(t, kube.Get(context.Background(), client.ObjectKeyFromObject(server), updated))
286+
accepted := meta.FindStatusCondition(updated.Status.Conditions, v1alpha2.AgentConditionTypeAccepted)
287+
require.NotNil(t, accepted)
288+
assert.Equal(t, metav1.ConditionFalse, accepted.Status)
289+
assert.Equal(t, "ReconcileFailed", accepted.Reason)
290+
assert.Contains(t, accepted.Message, "database unavailable")
291+
assert.Empty(t, updated.Status.DiscoveredTools, "the old inventory is cleared")
292+
}
293+
201294
func TestRemoteMCPServerDiscoveryDisabled(t *testing.T) {
202295
for name, tc := range map[string]struct {
203296
labels map[string]string

‎go/core/pkg/consts/labels.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,14 @@ package consts
99
// otherwise fail and mark the server as not Accepted forever. Any other value, or
1010
// no label, keeps discovery on. The same label excludes a Service or MCPServer
1111
// from automatic discovery (predicates.DiscoveryDisabledPredicate).
12+
//
13+
// Accepted=True with reason DiscoveryDisabled means only that the controller
14+
// skipped discovery on purpose and the spec.tls configuration is usable; it
15+
// does not validate runtime authentication or endpoint reachability. When
16+
// switching a server to the propagated caller credential, clear any
17+
// spec.headersFrom entry that sets Authorization: it overrides the caller's
18+
// token. Under server-side apply or GitOps tooling that only adds fields,
19+
// omitting the entry may leave it in place, so set `headersFrom: []` explicitly.
1220
const DiscoveryLabel = "kagent.dev/discovery"
1321

1422
// DiscoveryDisabled is the DiscoveryLabel value that turns discovery off.

0 commit comments

Comments
 (0)