Skip to content

Commit bb5d1df

Browse files
AustinAbro321AustinAbro321
authored
feat(init): enable switching between nodeport and proxy mode (#4608)
Signed-off-by: Austin Abro <[email protected]>
1 parent eeb5f37 commit bb5d1df

8 files changed

Lines changed: 339 additions & 29 deletions

File tree

‎.github/workflows/test-proxy.yml‎

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,8 +53,7 @@ jobs:
5353
path: build/
5454
retention-days: 1
5555

56-
# Run the tests on kind
57-
validate-proxy:
56+
validate-proxy-kind:
5857
runs-on: ubuntu-latest
5958
needs: build-zarf
6059
strategy:
@@ -97,6 +96,41 @@ jobs:
9796
run: |
9897
chmod +x build/zarf
9998
99+
- name: Run proxy tests
100+
run: |
101+
make test-proxy
102+
env:
103+
NODEPORT_INCOMPATIBLE: "true"
104+
105+
- name: get cluster info
106+
uses: ./.github/actions/debug-cluster
107+
if: failure()
108+
109+
validate-proxy-k3d:
110+
runs-on: ubuntu-latest
111+
needs: build-zarf
112+
steps:
113+
- name: Checkout
114+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
115+
116+
- name: Download build artifacts
117+
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
118+
with:
119+
name: build-artifacts
120+
path: build/
121+
122+
- name: Setup K3d
123+
uses: ./.github/actions/k3d
124+
125+
- name: Setup golang
126+
uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0
127+
with:
128+
go-version-file: go.mod
129+
130+
- name: Make Zarf executable
131+
run: |
132+
chmod +x build/zarf
133+
100134
- name: Run proxy tests
101135
run: |
102136
make test-proxy

‎site/src/content/docs/commands/zarf_init.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,7 @@ $ zarf init --artifact-push-password={PASSWORD} --artifact-push-username={USERNA
7474
-k, --key string Path to public key file for validating signed packages
7575
--nodeport int Nodeport to access a registry internal to the k8s cluster. Between [30000-32767]
7676
--oci-concurrency int Number of concurrent layer operations when pulling or pushing images or packages to/from OCI registries. (default 6)
77+
--registry-mode string how to access the registry (valid values: nodeport, proxy, external). Proxy mode is an alpha feature
7778
--registry-pull-password string Password for the pull-only user to access the registry
7879
--registry-pull-username string Username for pull-only access to the registry
7980
--registry-push-password string Password for the push-user to connect to the registry

‎src/cmd/initialize.go‎

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -79,8 +79,6 @@ func newInitCommand() *cobra.Command {
7979
fmt.Sprintf("how to access the registry (valid values: %s, %s, %s). Proxy mode is an alpha feature", state.RegistryModeNodePort, state.RegistryModeProxy, state.RegistryModeExternal))
8080
cmd.Flags().IntVar(&o.injectorPort, "injector-port", v.GetInt(InjectorPort),
8181
"the port that the injector will be exposed through. Affects the service nodeport in nodeport mode and pod hostport in proxy mode")
82-
// While this feature is in early alpha we will hide the flags
83-
cmd.Flags().MarkHidden("registry-mode")
8482

8583
// Flags for using an external Git server
8684
cmd.Flags().StringVar(&o.gitServer.Address, "git-url", v.GetString(VInitGitURL), lang.CmdInitFlagGitURL)
@@ -155,12 +153,8 @@ func (o *initOptions) run(cmd *cobra.Command, args []string) error {
155153
return err
156154
}
157155

158-
if o.registryInfo.RegistryMode == "" {
159-
if o.registryInfo.Address == "" {
160-
o.registryInfo.RegistryMode = state.RegistryModeNodePort
161-
} else {
162-
o.registryInfo.RegistryMode = state.RegistryModeExternal
163-
}
156+
if o.registryInfo.RegistryMode == "" && o.registryInfo.Address != "" {
157+
o.registryInfo.RegistryMode = state.RegistryModeExternal
164158
}
165159

166160
packageSource := ""
@@ -337,7 +331,7 @@ func validateExistingStateMatchesInput(ctx context.Context, registryInfo state.R
337331
if helpers.IsNotZeroAndNotEqual(gitServer, s.GitServer) {
338332
return fmt.Errorf("cannot change git server information after initial init, to update run `zarf tools update-creds git`")
339333
}
340-
if helpers.IsNotZeroAndNotEqual(registryInfo, s.RegistryInfo) {
334+
if state.CheckIfRegistryAddressOrCredsChanged(s.RegistryInfo, registryInfo) {
341335
return fmt.Errorf("cannot change registry information after initial init, to update run `zarf tools update-creds registry`")
342336
}
343337
if helpers.IsNotZeroAndNotEqual(artifactServer, s.ArtifactServer) {

‎src/pkg/cluster/cluster.go‎

Lines changed: 46 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -188,8 +188,19 @@ func (c *Cluster) InitState(ctx context.Context, opts InitStateOptions) (*state.
188188
return nil, fmt.Errorf("failed to check for existing state: %w", err)
189189
}
190190

191+
l.Debug("applying the Zarf namespace")
192+
zarfNamespace := NewZarfManagedApplyNamespace(state.ZarfNamespaceName)
193+
_, err = c.Clientset.CoreV1().Namespaces().Apply(ctx, zarfNamespace, metav1.ApplyOptions{FieldManager: FieldManagerName, Force: true})
194+
if err != nil {
195+
return nil, fmt.Errorf("unable to apply the Zarf namespace: %w", err)
196+
}
197+
198+
ipFamily, err := c.GetIPFamily(ctx)
199+
if err != nil {
200+
return nil, fmt.Errorf("unable to get the Kubernetes IP family: %w", err)
201+
}
202+
191203
// If state is nil, this is a new cluster.
192-
// TODO(mkcp): Simplify nesting with early returns closer to the top of the function.
193204
if s == nil {
194205
s = &state.State{}
195206
l.Debug("new cluster, no prior Zarf deployments found")
@@ -248,20 +259,6 @@ func (c *Cluster) InitState(ctx context.Context, opts InitStateOptions) (*state.
248259
}
249260
}
250261

251-
// Try to create the zarf namespace.
252-
l.Debug("creating the Zarf namespace")
253-
zarfNamespace := NewZarfManagedApplyNamespace(state.ZarfNamespaceName)
254-
_, err = c.Clientset.CoreV1().Namespaces().Apply(ctx, zarfNamespace, metav1.ApplyOptions{FieldManager: FieldManagerName, Force: true})
255-
if err != nil {
256-
return nil, fmt.Errorf("unable to apply the Zarf namespace: %w", err)
257-
}
258-
259-
ipFamily, err := c.GetIPFamily(ctx)
260-
if err != nil {
261-
return nil, fmt.Errorf("unable to get the Kubernetes IP family: %w", err)
262-
}
263-
s.IPFamily = ipFamily
264-
265262
// Wait up to 2 minutes for the default service account to be created.
266263
// Some clusters seem to take a while to create this, see https://github.com/kubernetes/kubernetes/issues/66689.
267264
// The default SA is required for pods to start properly.
@@ -283,7 +280,7 @@ func (c *Cluster) InitState(ctx context.Context, opts InitStateOptions) (*state.
283280
return nil, err
284281
}
285282
s.GitServer = opts.GitServer
286-
err = opts.RegistryInfo.FillInEmptyValues(s.IPFamily)
283+
err = opts.RegistryInfo.FillInEmptyValues(ipFamily)
287284
if err != nil {
288285
return nil, err
289286
}
@@ -292,6 +289,39 @@ func (c *Cluster) InitState(ctx context.Context, opts InitStateOptions) (*state.
292289
s.ArtifactServer = opts.ArtifactServer
293290
}
294291

292+
s.IPFamily = ipFamily
293+
294+
previousMode := s.RegistryInfo.RegistryMode
295+
if opts.RegistryInfo.RegistryMode != "" {
296+
s.RegistryInfo.RegistryMode = opts.RegistryInfo.RegistryMode
297+
}
298+
modeChanged := opts.RegistryInfo.RegistryMode != "" && opts.RegistryInfo.RegistryMode != previousMode
299+
300+
// If the registry mode is changing the injector will be re-made so the port should be reset
301+
if modeChanged {
302+
s.InjectorInfo.Port = 0
303+
}
304+
305+
switch s.RegistryInfo.RegistryMode {
306+
case state.RegistryModeNodePort:
307+
switch {
308+
case opts.RegistryInfo.NodePort != 0:
309+
s.RegistryInfo.NodePort = opts.RegistryInfo.NodePort
310+
case modeChanged:
311+
s.RegistryInfo.NodePort = state.ZarfInClusterContainerRegistryNodePort
312+
}
313+
s.RegistryInfo.MTLSStrategy = state.MTLSStrategyNone
314+
s.RegistryInfo.Address = state.LocalhostRegistryAddress(ipFamily, s.RegistryInfo.NodePort)
315+
case state.RegistryModeProxy:
316+
switch {
317+
case opts.RegistryInfo.NodePort != 0:
318+
s.RegistryInfo.NodePort = opts.RegistryInfo.NodePort
319+
case modeChanged:
320+
s.RegistryInfo.NodePort = state.ZarfRegistryHostPort
321+
}
322+
s.RegistryInfo.Address = state.LocalhostRegistryAddress(ipFamily, s.RegistryInfo.NodePort)
323+
}
324+
295325
if opts.RegistryInfo.RegistryMode == state.RegistryModeProxy {
296326
err = c.InitRegistryCerts(ctx)
297327
if err != nil {

‎src/pkg/cluster/cluster_test.go‎

Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ package cluster
77
import (
88
"context"
99
"encoding/json"
10+
"fmt"
1011
"testing"
1112
"time"
1213

@@ -310,3 +311,197 @@ func TestInit(t *testing.T) {
310311
})
311312
}
312313
}
314+
315+
func TestInitStateRegistryModeSwitch(t *testing.T) {
316+
tests := []struct {
317+
name string
318+
current state.State
319+
opts InitStateOptions
320+
expected state.State
321+
}{
322+
{
323+
name: "nodeport to proxy resets injector port, port defaults to 5000, and enables mTLS",
324+
current: state.State{
325+
RegistryInfo: state.RegistryInfo{
326+
RegistryMode: state.RegistryModeNodePort,
327+
MTLSStrategy: state.MTLSStrategyNone,
328+
},
329+
InjectorInfo: state.InjectorInfo{Port: 31999},
330+
},
331+
opts: InitStateOptions{
332+
RegistryInfo: state.RegistryInfo{RegistryMode: state.RegistryModeProxy},
333+
},
334+
expected: state.State{
335+
RegistryInfo: state.RegistryInfo{
336+
RegistryMode: state.RegistryModeProxy,
337+
MTLSStrategy: state.MTLSStrategyZarfManaged,
338+
NodePort: state.ZarfRegistryHostPort,
339+
},
340+
InjectorInfo: state.InjectorInfo{Port: 0},
341+
},
342+
},
343+
{
344+
name: "proxy to nodeport resets injector port and corrects out-of-range port",
345+
current: state.State{
346+
RegistryInfo: state.RegistryInfo{
347+
RegistryMode: state.RegistryModeProxy,
348+
MTLSStrategy: state.MTLSStrategyZarfManaged,
349+
},
350+
InjectorInfo: state.InjectorInfo{Port: 5000},
351+
},
352+
opts: InitStateOptions{
353+
RegistryInfo: state.RegistryInfo{RegistryMode: state.RegistryModeNodePort},
354+
},
355+
expected: state.State{
356+
RegistryInfo: state.RegistryInfo{
357+
RegistryMode: state.RegistryModeNodePort,
358+
MTLSStrategy: state.MTLSStrategyNone,
359+
NodePort: state.ZarfInClusterContainerRegistryNodePort,
360+
},
361+
InjectorInfo: state.InjectorInfo{Port: 0},
362+
},
363+
},
364+
{
365+
name: "proxy to nodeport with explicit valid port uses provided port",
366+
current: state.State{
367+
RegistryInfo: state.RegistryInfo{
368+
RegistryMode: state.RegistryModeProxy,
369+
MTLSStrategy: state.MTLSStrategyZarfManaged,
370+
},
371+
InjectorInfo: state.InjectorInfo{Port: 5000},
372+
},
373+
opts: InitStateOptions{
374+
RegistryInfo: state.RegistryInfo{
375+
RegistryMode: state.RegistryModeNodePort,
376+
NodePort: 30500,
377+
},
378+
},
379+
expected: state.State{
380+
RegistryInfo: state.RegistryInfo{
381+
RegistryMode: state.RegistryModeNodePort,
382+
MTLSStrategy: state.MTLSStrategyNone,
383+
NodePort: 30500,
384+
},
385+
InjectorInfo: state.InjectorInfo{Port: 0},
386+
},
387+
},
388+
{
389+
name: "nodeport to proxy with explicit port uses provided port",
390+
current: state.State{
391+
RegistryInfo: state.RegistryInfo{
392+
RegistryMode: state.RegistryModeNodePort,
393+
MTLSStrategy: state.MTLSStrategyNone,
394+
},
395+
InjectorInfo: state.InjectorInfo{Port: 31999},
396+
},
397+
opts: InitStateOptions{
398+
RegistryInfo: state.RegistryInfo{
399+
RegistryMode: state.RegistryModeProxy,
400+
NodePort: 8080,
401+
},
402+
},
403+
expected: state.State{
404+
RegistryInfo: state.RegistryInfo{
405+
RegistryMode: state.RegistryModeProxy,
406+
MTLSStrategy: state.MTLSStrategyZarfManaged,
407+
NodePort: 8080,
408+
},
409+
InjectorInfo: state.InjectorInfo{Port: 0},
410+
},
411+
},
412+
{
413+
name: "nodeport to nodeport preserves existing port and injector port",
414+
current: state.State{
415+
RegistryInfo: state.RegistryInfo{
416+
RegistryMode: state.RegistryModeNodePort,
417+
MTLSStrategy: state.MTLSStrategyNone,
418+
NodePort: 30500,
419+
},
420+
InjectorInfo: state.InjectorInfo{Port: 31999},
421+
},
422+
opts: InitStateOptions{
423+
RegistryInfo: state.RegistryInfo{RegistryMode: state.RegistryModeNodePort},
424+
},
425+
expected: state.State{
426+
RegistryInfo: state.RegistryInfo{
427+
RegistryMode: state.RegistryModeNodePort,
428+
MTLSStrategy: state.MTLSStrategyNone,
429+
NodePort: 30500,
430+
},
431+
InjectorInfo: state.InjectorInfo{Port: 31999},
432+
},
433+
},
434+
{
435+
name: "proxy to proxy preserves injector port and refreshes mTLS",
436+
current: state.State{
437+
RegistryInfo: state.RegistryInfo{
438+
RegistryMode: state.RegistryModeProxy,
439+
MTLSStrategy: state.MTLSStrategyZarfManaged,
440+
},
441+
InjectorInfo: state.InjectorInfo{Port: 5000},
442+
},
443+
opts: InitStateOptions{
444+
RegistryInfo: state.RegistryInfo{RegistryMode: state.RegistryModeProxy},
445+
},
446+
expected: state.State{
447+
RegistryInfo: state.RegistryInfo{
448+
RegistryMode: state.RegistryModeProxy,
449+
MTLSStrategy: state.MTLSStrategyZarfManaged,
450+
},
451+
InjectorInfo: state.InjectorInfo{Port: 5000},
452+
},
453+
},
454+
}
455+
for _, tt := range tests {
456+
t.Run(tt.name, func(t *testing.T) {
457+
ctx := context.Background()
458+
cs := fake.NewClientset()
459+
c := &Cluster{
460+
Clientset: cs,
461+
Watcher: healthchecks.NewImmediateWatcher(status.CurrentStatus),
462+
}
463+
464+
// Seed the fake cluster with the minimum objects InitState expects:
465+
// a node, the zarf namespace, the state secret, and the IP family service.
466+
tt.current.Distro = DistroIsK3d
467+
tt.current.RegistryInfo.PushUsername = "push-user"
468+
tt.current.RegistryInfo.PullUsername = "pull-user"
469+
tt.current.RegistryInfo.Secret = "secret"
470+
if tt.current.RegistryInfo.NodePort == 0 {
471+
tt.current.RegistryInfo.NodePort = state.ZarfInClusterContainerRegistryNodePort
472+
}
473+
tt.current.RegistryInfo.Address = fmt.Sprintf("127.0.0.1:%d", tt.current.RegistryInfo.NodePort)
474+
currentData, err := json.Marshal(tt.current)
475+
require.NoError(t, err)
476+
477+
_, err = cs.CoreV1().Nodes().Create(ctx, &corev1.Node{
478+
ObjectMeta: metav1.ObjectMeta{Name: "node"},
479+
}, metav1.CreateOptions{})
480+
require.NoError(t, err)
481+
_, err = cs.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{
482+
ObjectMeta: metav1.ObjectMeta{Name: state.ZarfNamespaceName},
483+
}, metav1.CreateOptions{})
484+
require.NoError(t, err)
485+
_, err = cs.CoreV1().Secrets(state.ZarfNamespaceName).Create(ctx, &corev1.Secret{
486+
ObjectMeta: metav1.ObjectMeta{Namespace: state.ZarfNamespaceName, Name: state.ZarfStateSecretName},
487+
Data: map[string][]byte{state.ZarfStateDataKey: currentData},
488+
}, metav1.CreateOptions{})
489+
require.NoError(t, err)
490+
_, err = cs.CoreV1().Services(state.ZarfNamespaceName).Create(ctx, &corev1.Service{
491+
ObjectMeta: metav1.ObjectMeta{Name: "zarf-ip-family-test", Namespace: state.ZarfNamespaceName},
492+
Spec: corev1.ServiceSpec{IPFamilies: []corev1.IPFamily{corev1.IPv4Protocol}},
493+
}, metav1.CreateOptions{})
494+
require.NoError(t, err)
495+
496+
result, err := c.InitState(ctx, tt.opts)
497+
require.NoError(t, err)
498+
499+
require.Equal(t, tt.expected.RegistryInfo.RegistryMode, result.RegistryInfo.RegistryMode)
500+
require.Equal(t, tt.expected.InjectorInfo.Port, result.InjectorInfo.Port)
501+
require.Equal(t, tt.expected.RegistryInfo.MTLSStrategy, result.RegistryInfo.MTLSStrategy)
502+
if tt.expected.RegistryInfo.NodePort != 0 {
503+
require.Equal(t, tt.expected.RegistryInfo.NodePort, result.RegistryInfo.NodePort)
504+
}
505+
})
506+
}
507+
}

0 commit comments

Comments
 (0)