Repository navigation
Expand file tree
/
Copy pathpackage_reconciler.go
More file actions
1646 lines (1497 loc) · 65.9 KB
/
Copy pathpackage_reconciler.go
File metadata and controls
1646 lines (1497 loc) · 65.9 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/*
Copyright 2025 The Cozystack Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package operator
import (
"context"
"errors"
"fmt"
"slices"
"strings"
"time"
cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1"
"github.com/cozystack/cozystack/internal/marketplace/collision"
"github.com/cozystack/cozystack/pkg/config"
helmv2 "github.com/fluxcd/helm-controller/api/v2"
appsv1 "k8s.io/api/apps/v1"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
apiequality "k8s.io/apimachinery/pkg/api/equality"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
corev1ac "k8s.io/client-go/applyconfigurations/core/v1"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
"sigs.k8s.io/controller-runtime/pkg/handler"
"sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
)
const (
// AnnotationSkipCozystackValues disables injection of cozystack-values secret into HelmRelease
// This annotation should be placed on PackageSource
AnnotationSkipCozystackValues = "operator.cozystack.io/skip-cozystack-values"
// SecretCozystackValues is the name of the secret containing cluster and namespace configuration
SecretCozystackValues = "cozystack-values"
// SystemDefaultsLimitRangeName is the LimitRange the reconciler maintains in every
// system namespace to default container memory requests and limits.
SystemDefaultsLimitRangeName = "cozystack-system-defaults"
// packageControllerFieldOwner is the server-side-apply field manager used for every
// cluster object the Package reconciler owns outright.
packageControllerFieldOwner = "cozystack-package-controller"
// managedByLabel marks the objects this reconciler owns outright, so a same-named
// object it did not create can be told apart and left alone.
managedByLabel = "app.kubernetes.io/managed-by"
)
// parseCRDPolicy maps ComponentInstall.UpgradeCRDs to a helmv2.CRDsPolicy.
// Empty / nil preserves the helm-controller default (Skip on upgrade);
// the CRD enum marker restricts the string to Skip/Create/CreateReplace.
func parseCRDPolicy(install *cozyv1alpha1.ComponentInstall) helmv2.CRDsPolicy {
if install == nil || install.UpgradeCRDs == "" {
return ""
}
return helmv2.CRDsPolicy(install.UpgradeCRDs)
}
// PackageReconciler reconciles Package resources
type PackageReconciler struct {
client.Client
// APIReader reads straight from the API server, bypassing the manager's cache.
// Used for memory-policy ownership and state reads, and the workload scan behind the system defaults
// LimitRange: routing that through the cached client would start cluster-wide informers
// and cost the operator far more memory than the LimitRange saves. Optional —
// when nil the scan falls back to Client, which is what the unit tests use.
APIReader client.Reader
Scheme *runtime.Scheme
HelmReleaseInterval time.Duration
HelmReleaseRetryInterval time.Duration
HelmReleaseInstallTimeout time.Duration
HelmReleaseUpgradeTimeout time.Duration
HelmReleaseMaxHistory int
// SystemNamespaceMemoryLimit is the default container memory limit applied through a
// LimitRange in every system namespace. A zero quantity disables the LimitRange and
// removes any the reconciler previously created.
SystemNamespaceMemoryLimit resource.Quantity
// SystemNamespaceMemoryRequest is the matching default container memory request. It
// must be set whenever the limit is, because Kubernetes otherwise defaults each
// request to the limit and reserves that much at schedule time.
SystemNamespaceMemoryRequest resource.Quantity
}
// buildHelmReleaseSpec assembles the Spec applied to every generated
// HelmRelease. RetryInterval drives recovery from failed install/upgrade
// attempts; Interval polls healthy releases.
func (r *PackageReconciler) buildHelmReleaseSpec(componentInstall *cozyv1alpha1.ComponentInstall, artifactName string) helmv2.HelmReleaseSpec {
maxHistory := r.HelmReleaseMaxHistory
spec := helmv2.HelmReleaseSpec{
Interval: metav1.Duration{Duration: r.HelmReleaseInterval},
MaxHistory: &maxHistory,
ChartRef: &helmv2.CrossNamespaceSourceReference{
Kind: "ExternalArtifact",
Name: artifactName,
Namespace: "cozy-system",
},
Install: &helmv2.Install{
Timeout: &metav1.Duration{Duration: r.HelmReleaseInstallTimeout},
Strategy: &helmv2.InstallStrategy{
Name: string(helmv2.ActionStrategyRetryOnFailure),
RetryInterval: &metav1.Duration{Duration: r.HelmReleaseRetryInterval},
},
},
Upgrade: &helmv2.Upgrade{
Timeout: &metav1.Duration{Duration: r.HelmReleaseUpgradeTimeout},
Strategy: &helmv2.UpgradeStrategy{
Name: string(helmv2.ActionStrategyRetryOnFailure),
RetryInterval: &metav1.Duration{Duration: r.HelmReleaseRetryInterval},
},
CRDs: parseCRDPolicy(componentInstall),
},
}
// kstatus readiness (issue #2642): gate the release on the CR(s) it renders.
// ResolveWaitStrategy couples the default — expressions imply poller when no
// strategy is set, since healthCheckExprs are only evaluated under poller.
if componentInstall != nil {
spec.HealthCheckExprs = componentInstall.HealthCheckExprs
spec.WaitStrategy = config.ResolveWaitStrategy(componentInstall.WaitStrategy, len(componentInstall.HealthCheckExprs) > 0)
}
return spec
}
// +kubebuilder:rbac:groups=cozystack.io,resources=packages,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=cozystack.io,resources=packages/status,verbs=get;update;patch
// +kubebuilder:rbac:groups=cozystack.io,resources=packagesources,verbs=get;list;watch
// +kubebuilder:rbac:groups=helm.toolkit.fluxcd.io,resources=helmreleases,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=core,resources=namespaces,verbs=get;list;watch;create;update;patch
// +kubebuilder:rbac:groups=core,resources=limitranges,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch
// +kubebuilder:rbac:groups=apps,resources=deployments;statefulsets;daemonsets,verbs=get;list;watch
// +kubebuilder:rbac:groups=batch,resources=cronjobs;jobs,verbs=get;list;watch
// Reconcile is part of the main kubernetes reconciliation loop
func (r *PackageReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, reconcileErr error) {
logger := log.FromContext(ctx)
pkg := &cozyv1alpha1.Package{}
if err := r.Get(ctx, req.NamespacedName, pkg); err != nil {
if apierrors.IsNotFound(err) {
// The final Package deletion must also retire its namespace defaults.
return ctrl.Result{}, r.cleanupUnusedMemoryDefaults(ctx)
}
return ctrl.Result{}, err
}
// Safety scans and administrator acknowledgements must progress on a quiet cluster
// too. Keep this bounded instead of adding cluster-wide workload informers.
defer func() {
if reconcileErr == nil && (result.RequeueAfter == 0 || result.RequeueAfter > 5*time.Minute) {
result.RequeueAfter = 5 * time.Minute
}
}()
if err := r.cleanupUnusedMemoryDefaults(ctx); err != nil {
logger.Error(err, "failed to clean up unused system memory defaults")
}
// Get PackageSource with the same name
packageSource := &cozyv1alpha1.PackageSource{}
if err := r.Get(ctx, types.NamespacedName{Name: pkg.Name}, packageSource); err != nil {
if apierrors.IsNotFound(err) {
meta.SetStatusCondition(&pkg.Status.Conditions, metav1.Condition{
Type: "Ready",
Status: metav1.ConditionFalse,
Reason: "PackageSourceNotFound",
Message: fmt.Sprintf("PackageSource %s not found", pkg.Name),
})
if err := r.Status().Update(ctx, pkg); err != nil {
return ctrl.Result{}, err
}
return ctrl.Result{}, nil
}
return ctrl.Result{}, err
}
// Determine variant (default to "default" if not specified)
variantName := pkg.Spec.Variant
if variantName == "" {
variantName = "default"
}
// Find the variant in PackageSource
var variant *cozyv1alpha1.Variant
for i := range packageSource.Spec.Variants {
if packageSource.Spec.Variants[i].Name == variantName {
variant = &packageSource.Spec.Variants[i]
break
}
}
if variant == nil {
meta.SetStatusCondition(&pkg.Status.Conditions, metav1.Condition{
Type: "Ready",
Status: metav1.ConditionFalse,
Reason: "VariantNotFound",
Message: fmt.Sprintf("Variant %s not found in PackageSource %s", variantName, pkg.Name),
})
if err := r.Status().Update(ctx, pkg); err != nil {
return ctrl.Result{}, err
}
return ctrl.Result{}, nil
}
// Reconcile namespaces from components
if err := r.reconcileNamespaces(ctx, pkg, variant); err != nil {
logger.Error(err, "failed to reconcile namespaces")
return ctrl.Result{}, err
}
// Update dependencies status
if err := r.updateDependenciesStatus(ctx, pkg, variant); err != nil {
logger.Error(err, "failed to update dependencies status")
// Don't return error, continue with reconciliation
}
// Validate variant dependencies before creating HelmReleases
// Check if all dependencies are ready based on status
if !r.areDependenciesReady(pkg, variant) {
logger.Info("variant dependencies not ready, skipping HelmRelease creation", "package", pkg.Name)
meta.SetStatusCondition(&pkg.Status.Conditions, metav1.Condition{
Type: "Ready",
Status: metav1.ConditionFalse,
Reason: "DependenciesNotReady",
Message: "One or more dependencies are not ready",
})
if err := r.Status().Update(ctx, pkg); err != nil {
return ctrl.Result{}, err
}
// Return success to avoid requeue, but don't create HelmReleases
return ctrl.Result{}, nil
}
// Create HelmReleases for components with Install section
helmReleaseCount := 0
var skippedPrivileged []string
for _, component := range variant.Components {
// Skip components without Install section
if component.Install == nil {
continue
}
// Check if component is disabled via Package spec
if pkgComponent, ok := pkg.Spec.Components[component.Name]; ok {
if pkgComponent.Enabled != nil && !*pkgComponent.Enabled {
logger.V(1).Info("skipping disabled component", "package", pkg.Name, "component", component.Name)
continue
}
}
// Refuse an unconfirmed privileged install here, at the install site,
// rather than in the tap materializer where the guard races this
// reconciler. A tap auto-registration Package carries the marketplace-tap
// label and is created without confirmation; `cozypkg add --allow-privileged`
// sheds the label, which is the operator's confirmation. A platform Package
// never carries the label and is unaffected.
if component.Install.Privileged && !collision.PrivilegedConfirmed(pkg) {
logger.Info("skipping unconfirmed privileged component", "package", pkg.Name, "component", component.Name)
skippedPrivileged = append(skippedPrivileged, component.Name)
continue
}
// Build artifact name: <packagesource>-<variant>-<componentname> (with dots replaced by dashes)
artifactName := fmt.Sprintf("%s-%s-%s",
strings.ReplaceAll(packageSource.Name, ".", "-"),
strings.ReplaceAll(variantName, ".", "-"),
strings.ReplaceAll(component.Name, ".", "-"))
// Namespace must be set
namespace := component.Install.Namespace
if namespace == "" {
logger.Error(fmt.Errorf("component %s has empty namespace in Install section", component.Name), "namespace validation failed")
meta.SetStatusCondition(&pkg.Status.Conditions, metav1.Condition{
Type: "Ready",
Status: metav1.ConditionFalse,
Reason: "InvalidConfiguration",
Message: fmt.Sprintf("Component %s has empty namespace in Install section", component.Name),
})
if err := r.Status().Update(ctx, pkg); err != nil {
return ctrl.Result{}, err
}
return ctrl.Result{}, fmt.Errorf("component %s has empty namespace in Install section", component.Name)
}
// Determine release name (from Install or use component name)
releaseName := component.Install.ReleaseName
if releaseName == "" {
releaseName = component.Name
}
// Build labels
labels := make(map[string]string)
labels["cozystack.io/package"] = pkg.Name
if component.Install.Privileged {
labels["cozystack.io/privileged"] = "true"
}
// Create HelmRelease
hr := &helmv2.HelmRelease{
ObjectMeta: metav1.ObjectMeta{
Name: releaseName,
Namespace: namespace,
Labels: labels,
},
Spec: r.buildHelmReleaseSpec(component.Install, artifactName),
}
// Add valuesFrom for cozystack-values secret unless disabled by annotation on PackageSource
if packageSource.GetAnnotations()[AnnotationSkipCozystackValues] != "true" {
hr.Spec.ValuesFrom = []helmv2.ValuesReference{
{
Kind: "Secret",
Name: SecretCozystackValues,
},
}
}
// Set ownerReference
gvk, err := apiutil.GVKForObject(pkg, r.Scheme)
if err != nil {
logger.Error(err, "failed to get GVK for Package")
meta.SetStatusCondition(&pkg.Status.Conditions, metav1.Condition{
Type: "Ready",
Status: metav1.ConditionFalse,
Reason: "InternalError",
Message: fmt.Sprintf("Failed to get GVK for Package: %v", err),
})
if err := r.Status().Update(ctx, pkg); err != nil {
return ctrl.Result{}, err
}
return ctrl.Result{}, fmt.Errorf("failed to get GVK for Package: %w", err)
}
hr.OwnerReferences = []metav1.OwnerReference{
{
APIVersion: gvk.GroupVersion().String(),
Kind: gvk.Kind,
Name: pkg.Name,
UID: pkg.UID,
Controller: func() *bool { b := true; return &b }(),
},
}
// Merge values from Package spec if provided
if pkgComponent, ok := pkg.Spec.Components[component.Name]; ok && pkgComponent.Values != nil {
hr.Spec.Values = pkgComponent.Values
}
// Build DependsOn from component Install and variant DependsOn
dependsOn, err := r.buildDependsOn(ctx, pkg, packageSource, variant, &component)
if err != nil {
logger.Error(err, "failed to build DependsOn", "component", component.Name)
meta.SetStatusCondition(&pkg.Status.Conditions, metav1.Condition{
Type: "Ready",
Status: metav1.ConditionFalse,
Reason: "DependsOnFailed",
Message: fmt.Sprintf("Failed to build DependsOn for component %s: %v", component.Name, err),
})
if err := r.Status().Update(ctx, pkg); err != nil {
return ctrl.Result{}, err
}
// Return nil to stop reconciliation, error is recorded in status
return ctrl.Result{}, nil
}
if len(dependsOn) > 0 {
hr.Spec.DependsOn = dependsOn
}
// Set valuesFiles annotation
if len(component.ValuesFiles) > 0 {
if hr.Annotations == nil {
hr.Annotations = make(map[string]string)
}
hr.Annotations["cozyhr.cozystack.io/values-files"] = strings.Join(component.ValuesFiles, ",")
}
if err := r.createOrUpdateHelmRelease(ctx, hr); err != nil {
logger.Error(err, "failed to reconcile HelmRelease", "name", releaseName, "namespace", namespace)
meta.SetStatusCondition(&pkg.Status.Conditions, metav1.Condition{
Type: "Ready",
Status: metav1.ConditionFalse,
Reason: "HelmReleaseFailed",
Message: fmt.Sprintf("Failed to create HelmRelease %s: %v", releaseName, err),
})
if err := r.Status().Update(ctx, pkg); err != nil {
return ctrl.Result{}, err
}
return ctrl.Result{}, err
}
helmReleaseCount++
logger.Info("reconciled HelmRelease", "package", pkg.Name, "component", component.Name, "releaseName", releaseName, "namespace", namespace)
}
// Cleanup orphaned HelmReleases
if err := r.cleanupOrphanedHelmReleases(ctx, pkg, variant); err != nil {
logger.Error(err, "failed to cleanup orphaned HelmReleases")
// Don't return error, continue with status update
}
// Update status with success message
if len(skippedPrivileged) > 0 {
meta.SetStatusCondition(&pkg.Status.Conditions, metav1.Condition{
Type: "Ready",
Status: metav1.ConditionFalse,
Reason: "PrivilegedNotConfirmed",
Message: fmt.Sprintf("privileged install component(s) %v are not confirmed; register with 'cozypkg add --allow-privileged'", skippedPrivileged),
})
if err := r.Status().Update(ctx, pkg); err != nil {
return ctrl.Result{}, err
}
return ctrl.Result{}, nil
}
message := fmt.Sprintf("reconciliation succeeded, generated %d helmrelease(s)", helmReleaseCount)
meta.SetStatusCondition(&pkg.Status.Conditions, metav1.Condition{
Type: "Ready",
Status: metav1.ConditionTrue,
Reason: "ReconciliationSucceeded",
Message: message,
})
if err := r.Status().Update(ctx, pkg); err != nil {
return ctrl.Result{}, err
}
logger.Info("reconciled Package", "name", pkg.Name, "helmReleaseCount", helmReleaseCount)
// Update dependencies status for Packages that depend on this Package
// This ensures they get re-enqueued when their dependency becomes ready
if err := r.updateDependentPackagesDependencies(ctx, pkg.Name); err != nil {
logger.V(1).Error(err, "failed to update dependent packages dependencies", "package", pkg.Name)
// Don't return error, this is best-effort
}
// Dependent Packages will be automatically enqueued by the watch handler
// when this Package's status is updated (see SetupWithManager watch handler)
return ctrl.Result{}, nil
}
// createOrUpdateHelmRelease creates or updates a HelmRelease
func (r *PackageReconciler) createOrUpdateHelmRelease(ctx context.Context, hr *helmv2.HelmRelease) error {
existing := &helmv2.HelmRelease{}
key := types.NamespacedName{
Name: hr.Name,
Namespace: hr.Namespace,
}
err := r.Get(ctx, key, existing)
if apierrors.IsNotFound(err) {
return r.Create(ctx, hr)
} else if err != nil {
return err
}
// Preserve resource version
hr.SetResourceVersion(existing.GetResourceVersion())
// Merge labels
labels := hr.GetLabels()
if labels == nil {
labels = make(map[string]string)
}
for k, v := range existing.GetLabels() {
if _, ok := labels[k]; !ok {
labels[k] = v
}
}
hr.SetLabels(labels)
// Merge annotations
annotations := hr.GetAnnotations()
if annotations == nil {
annotations = make(map[string]string)
}
for k, v := range existing.GetAnnotations() {
if _, ok := annotations[k]; !ok {
annotations[k] = v
}
}
hr.SetAnnotations(annotations)
hr.Spec.Suspend = existing.Spec.Suspend
// Update Spec
existing.Spec = hr.Spec
existing.SetLabels(hr.GetLabels())
existing.SetAnnotations(hr.GetAnnotations())
existing.SetOwnerReferences(hr.GetOwnerReferences())
return r.Update(ctx, existing)
}
// getVariantForPackage retrieves the Variant for a given Package
// Returns the Variant and an error if not found
// If c is nil, uses the reconciler's client
func (r *PackageReconciler) getVariantForPackage(ctx context.Context, pkg *cozyv1alpha1.Package, c client.Client) (*cozyv1alpha1.Variant, error) {
// Use provided client or fall back to reconciler's client
cl := c
if cl == nil {
cl = r.Client
}
// Determine variant name (default to "default" if not specified)
variantName := pkg.Spec.Variant
if variantName == "" {
variantName = "default"
}
// Get the PackageSource
packageSource := &cozyv1alpha1.PackageSource{}
if err := cl.Get(ctx, types.NamespacedName{Name: pkg.Name}, packageSource); err != nil {
if apierrors.IsNotFound(err) {
return nil, fmt.Errorf("PackageSource %s not found", pkg.Name)
}
return nil, fmt.Errorf("failed to get PackageSource %s: %w", pkg.Name, err)
}
// Find the variant in PackageSource
var variant *cozyv1alpha1.Variant
for i := range packageSource.Spec.Variants {
if packageSource.Spec.Variants[i].Name == variantName {
variant = &packageSource.Spec.Variants[i]
break
}
}
if variant == nil {
return nil, fmt.Errorf("variant %s not found in PackageSource %s", variantName, pkg.Name)
}
return variant, nil
}
// buildDependsOn builds DependsOn list for a component
// Includes:
// 1. Dependencies from component.Install.DependsOn (with namespace from referenced component)
// 2. Dependencies from variant.DependsOn (all components with Install from referenced Package)
func (r *PackageReconciler) buildDependsOn(ctx context.Context, pkg *cozyv1alpha1.Package, packageSource *cozyv1alpha1.PackageSource, variant *cozyv1alpha1.Variant, component *cozyv1alpha1.Component) ([]helmv2.DependencyReference, error) {
logger := log.FromContext(ctx)
dependsOn := []helmv2.DependencyReference{}
// Build map of component names to their release names and namespaces in current variant
componentMap := make(map[string]struct {
releaseName string
namespace string
})
for _, comp := range variant.Components {
if comp.Install == nil {
continue
}
compNamespace := comp.Install.Namespace
if compNamespace == "" {
return nil, fmt.Errorf("component %s has empty namespace in Install section", comp.Name)
}
compReleaseName := comp.Install.ReleaseName
if compReleaseName == "" {
compReleaseName = comp.Name
}
componentMap[comp.Name] = struct {
releaseName string
namespace string
}{
releaseName: compReleaseName,
namespace: compNamespace,
}
}
// Add dependencies from component.Install.DependsOn
if len(component.Install.DependsOn) > 0 {
for _, depName := range component.Install.DependsOn {
depComp, ok := componentMap[depName]
if !ok {
return nil, fmt.Errorf("component %s not found in variant for dependency %s", depName, component.Name)
}
dependsOn = append(dependsOn, helmv2.DependencyReference{
Name: depComp.releaseName,
Namespace: depComp.namespace,
})
logger.V(1).Info("added component dependency", "component", component.Name, "dependsOn", depName, "releaseName", depComp.releaseName, "namespace", depComp.namespace)
}
}
// Add dependencies from variant.DependsOn
if len(variant.DependsOn) > 0 {
for _, depPackageName := range variant.DependsOn {
// Check if dependency is in IgnoreDependencies
ignore := slices.Contains(pkg.Spec.IgnoreDependencies, depPackageName)
if ignore {
logger.V(1).Info("ignoring dependency", "package", pkg.Name, "dependency", depPackageName)
continue
}
// Get the Package
depPackage := &cozyv1alpha1.Package{}
if err := r.Get(ctx, types.NamespacedName{Name: depPackageName}, depPackage); err != nil {
if apierrors.IsNotFound(err) {
return nil, fmt.Errorf("dependent Package %s not found", depPackageName)
}
return nil, fmt.Errorf("failed to get dependent Package %s: %w", depPackageName, err)
}
// Get the variant from dependent Package
depVariant, err := r.getVariantForPackage(ctx, depPackage, nil)
if err != nil {
return nil, fmt.Errorf("failed to get variant for dependent Package %s: %w", depPackageName, err)
}
// Add all components with Install from dependent variant
for _, depComp := range depVariant.Components {
if depComp.Install == nil {
continue
}
// Check if component is disabled in dependent Package
if depPkgComponent, ok := depPackage.Spec.Components[depComp.Name]; ok {
if depPkgComponent.Enabled != nil && !*depPkgComponent.Enabled {
continue
}
}
depCompNamespace := depComp.Install.Namespace
if depCompNamespace == "" {
return nil, fmt.Errorf("component %s in dependent Package %s has empty namespace in Install section", depComp.Name, depPackageName)
}
depCompReleaseName := depComp.Install.ReleaseName
if depCompReleaseName == "" {
depCompReleaseName = depComp.Name
}
dependsOn = append(dependsOn, helmv2.DependencyReference{
Name: depCompReleaseName,
Namespace: depCompNamespace,
})
logger.V(1).Info("added variant dependency", "package", pkg.Name, "dependency", depPackageName, "component", depComp.Name, "releaseName", depCompReleaseName, "namespace", depCompNamespace)
}
}
}
return dependsOn, nil
}
// updateDependenciesStatus updates the dependencies status in Package status
// It checks the readiness of each dependency and updates pkg.Status.Dependencies
// Old dependency keys that are no longer in the dependency list are removed
func (r *PackageReconciler) updateDependenciesStatus(ctx context.Context, pkg *cozyv1alpha1.Package, variant *cozyv1alpha1.Variant) error {
logger := log.FromContext(ctx)
// Initialize dependencies map if nil
if pkg.Status.Dependencies == nil {
pkg.Status.Dependencies = make(map[string]cozyv1alpha1.DependencyStatus)
}
// Build set of current dependencies (excluding ignored ones)
currentDeps := make(map[string]bool)
if len(variant.DependsOn) > 0 {
for _, depPackageName := range variant.DependsOn {
// Check if dependency is in IgnoreDependencies
ignore := slices.Contains(pkg.Spec.IgnoreDependencies, depPackageName)
if ignore {
logger.V(1).Info("ignoring dependency", "package", pkg.Name, "dependency", depPackageName)
continue
}
currentDeps[depPackageName] = true
}
}
// Remove old dependencies that are no longer in the list
for depName := range pkg.Status.Dependencies {
if !currentDeps[depName] {
delete(pkg.Status.Dependencies, depName)
logger.V(1).Info("removed old dependency from status", "package", pkg.Name, "dependency", depName)
}
}
// Update status for each current dependency
for depPackageName := range currentDeps {
// Get the Package
depPackage := &cozyv1alpha1.Package{}
if err := r.Get(ctx, types.NamespacedName{Name: depPackageName}, depPackage); err != nil {
if apierrors.IsNotFound(err) {
// Dependency not found, mark as not ready
pkg.Status.Dependencies[depPackageName] = cozyv1alpha1.DependencyStatus{
Ready: false,
}
logger.V(1).Info("dependency not found, marking as not ready", "package", pkg.Name, "dependency", depPackageName)
continue
}
// Error getting dependency, keep existing status or mark as not ready
if _, exists := pkg.Status.Dependencies[depPackageName]; !exists {
pkg.Status.Dependencies[depPackageName] = cozyv1alpha1.DependencyStatus{
Ready: false,
}
}
logger.V(1).Error(err, "failed to get dependency, keeping existing status", "package", pkg.Name, "dependency", depPackageName)
continue
}
// Check Ready condition
readyCondition := meta.FindStatusCondition(depPackage.Status.Conditions, "Ready")
isReady := readyCondition != nil && readyCondition.Status == metav1.ConditionTrue
// Update dependency status
pkg.Status.Dependencies[depPackageName] = cozyv1alpha1.DependencyStatus{
Ready: isReady,
}
logger.V(1).Info("updated dependency status", "package", pkg.Name, "dependency", depPackageName, "ready", isReady)
}
return nil
}
// areDependenciesReady checks if all dependencies are ready based on status
func (r *PackageReconciler) areDependenciesReady(pkg *cozyv1alpha1.Package, variant *cozyv1alpha1.Variant) bool {
if len(variant.DependsOn) == 0 {
return true
}
for _, depPackageName := range variant.DependsOn {
// Check if dependency is in IgnoreDependencies
ignore := slices.Contains(pkg.Spec.IgnoreDependencies, depPackageName)
if ignore {
continue
}
// Check dependency status
depStatus, exists := pkg.Status.Dependencies[depPackageName]
if !exists || !depStatus.Ready {
return false
}
}
return true
}
// updateDependentPackagesDependencies updates dependencies status for all Packages that depend on the given Package
// This ensures dependent packages get re-enqueued when their dependency status changes
func (r *PackageReconciler) updateDependentPackagesDependencies(ctx context.Context, packageName string) error {
logger := log.FromContext(ctx)
// Get all Packages
packageList := &cozyv1alpha1.PackageList{}
if err := r.List(ctx, packageList); err != nil {
return fmt.Errorf("failed to list Packages: %w", err)
}
// Get the updated Package to check its readiness
updatedPkg := &cozyv1alpha1.Package{}
if err := r.Get(ctx, types.NamespacedName{Name: packageName}, updatedPkg); err != nil {
if apierrors.IsNotFound(err) {
return nil // Package not found, nothing to update
}
return fmt.Errorf("failed to get Package %s: %w", packageName, err)
}
// Check Ready condition of the updated Package
readyCondition := meta.FindStatusCondition(updatedPkg.Status.Conditions, "Ready")
isReady := readyCondition != nil && readyCondition.Status == metav1.ConditionTrue
// For each Package, check if it depends on the given Package
for _, pkg := range packageList.Items {
// Skip the Package itself
if pkg.Name == packageName {
continue
}
// Get variant
variant, err := r.getVariantForPackage(ctx, &pkg, nil)
if err != nil {
// Continue if PackageSource or variant not found (best-effort operation)
logger.V(1).Info("skipping package, failed to get variant", "package", pkg.Name, "error", err)
continue
}
// Check if this Package depends on the given Package
dependsOn := false
for _, dep := range variant.DependsOn {
// Check if dependency is in IgnoreDependencies
ignore := slices.Contains(pkg.Spec.IgnoreDependencies, dep)
if ignore {
continue
}
if dep == packageName {
dependsOn = true
break
}
}
if dependsOn {
// Update the dependency status in this Package
if pkg.Status.Dependencies == nil {
pkg.Status.Dependencies = make(map[string]cozyv1alpha1.DependencyStatus)
}
pkg.Status.Dependencies[packageName] = cozyv1alpha1.DependencyStatus{
Ready: isReady,
}
if err := r.Status().Update(ctx, &pkg); err != nil {
logger.V(1).Error(err, "failed to update dependency status for dependent Package", "package", pkg.Name, "dependency", packageName)
continue
}
logger.V(1).Info("updated dependency status for dependent Package", "package", pkg.Name, "dependency", packageName, "ready", isReady)
}
}
return nil
}
// reconcileNamespaces creates or updates namespaces based on components in the variant.
// For each namespace, it checks ALL Packages sharing that namespace to determine whether
// the namespace should be privileged — it is privileged if ANY Package has a privileged
// component installed in it.
func (r *PackageReconciler) reconcileNamespaces(ctx context.Context, pkg *cozyv1alpha1.Package, variant *cozyv1alpha1.Variant) error {
logger := log.FromContext(ctx)
limitRangeDisabled := r.SystemNamespaceMemoryLimit.IsZero()
if limitRangeDisabled {
// Disabled is global, unlike targetNamespaces below, which belongs only to the
// Package currently reconciling. Sweep by the ownership label once here so a
// namespace whose component or Package disappeared is still reached, then keep
// the per-namespace path out of the way.
if err := r.deleteManagedSystemDefaultsLimitRanges(ctx); err != nil {
// The LimitRange is opportunistic hardening. Its cleanup must not stop the
// Package's namespaces and components from reconciling; retry on the next pass.
logger.Error(err, "failed to remove managed system defaults LimitRanges while the feature is disabled")
}
}
// Collect namespaces from this Package's components
targetNamespaces := make(map[string]struct{})
for _, component := range variant.Components {
if component.Install == nil {
continue
}
if pkgComponent, ok := pkg.Spec.Components[component.Name]; ok {
if pkgComponent.Enabled != nil && !*pkgComponent.Enabled {
continue
}
}
namespace := component.Install.Namespace
if namespace == "" {
return fmt.Errorf("component %s has empty namespace in Install section", component.Name)
}
targetNamespaces[namespace] = struct{}{}
}
// Determine which namespaces should be privileged by checking ALL Packages
privileged, err := r.resolvePrivilegedNamespaces(ctx, targetNamespaces)
if err != nil {
return fmt.Errorf("failed to resolve privileged namespaces: %w", err)
}
// Create or update all namespaces
for nsName := range targetNamespaces {
isSystem := !strings.HasPrefix(nsName, "tenant-")
namespace := &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: nsName,
Labels: make(map[string]string),
Annotations: map[string]string{
"helm.sh/resource-policy": "keep",
},
},
}
if isSystem {
namespace.Labels["cozystack.io/system"] = "true"
}
if privileged[nsName] {
namespace.Labels["pod-security.kubernetes.io/enforce"] = "privileged"
}
if err := r.createOrUpdateNamespace(ctx, namespace); err != nil {
logger.Error(err, "failed to reconcile namespace", "name", nsName, "privileged", privileged[nsName])
return fmt.Errorf("failed to reconcile namespace %s: %w", nsName, err)
}
logger.Info("reconciled namespace", "name", nsName, "privileged", privileged[nsName])
// Cleanup excludes deleting Packages; their remaining reconciliation must
// not reinstate a retired policy or consume an acknowledgement.
if isSystem && !limitRangeDisabled && pkg.DeletionTimestamp.IsZero() {
// Logged and stepped over rather than returned. The LimitRange is
// opportunistic hardening against the Talos OOM handler's victim
// selection; a namespace that does not get one is back to the
// behaviour of every release before it. Failing the Package reconcile
// instead would leave the namespace's components uninstalled, making
// the hardening more disruptive than the problem it mitigates.
if err := r.reconcileSystemDefaultsLimitRange(ctx, nsName); err != nil {
logger.Error(err, "failed to reconcile system defaults LimitRange", "namespace", nsName)
}
}
}
return nil
}
// deleteManagedSystemDefaultsLimitRanges removes every LimitRange owned by this feature,
// including objects in namespaces no active Package targets anymore. The selector keeps the
// cluster-wide List narrow; the label check inside the loop is a second guard against a client
// that does not honor selectors. Delete failures are aggregated so one namespace cannot keep
// stale defaults in every namespace that follows it in the List.
func (r *PackageReconciler) deleteManagedSystemDefaultsLimitRanges(ctx context.Context) error {
managed := &corev1.LimitRangeList{}
selector := client.MatchingLabels{managedByLabel: packageControllerFieldOwner}
if err := r.List(ctx, managed, selector); err != nil {
return fmt.Errorf("failed to list managed system defaults LimitRanges: %w", err)
}
var deleteErrs []error
for i := range managed.Items {
lr := &managed.Items[i]
if lr.Labels[managedByLabel] != packageControllerFieldOwner {
continue
}
if err := r.deleteSystemDefaultsLimitRange(ctx, lr.Namespace); err != nil {
deleteErrs = append(deleteErrs, fmt.Errorf("failed to delete managed LimitRange %s/%s: %w", lr.Namespace, lr.Name, err))
}
}
return errors.Join(deleteErrs...)
}
// reconcileSystemDefaultsLimitRange maintains the LimitRange that gives every container
// in a system namespace a default memory request and limit.
//
// The Talos userspace OOM handler (v1.12+) selects its victim by ranking cgroups with
// `memory_max.hasValue() ? 0.0 : {Besteffort: 1.0, Burstable: 0.5, ...}[class] *
// memory_current`, and discards every cgroup that scores zero. A pod whose containers all
// carry a memory limit has memory.max set on its cgroup, scores zero, and is never
// selected. A pod without one stays a candidate however little memory it is using and
// whatever actually caused the pressure — victim selection is decoupled from the trigger.
// System components are overwhelmingly the pods without limits, so they were the ones
// being killed on behalf of tenant workloads that were the real source of the pressure.
//
// Defaulting a limit across system namespaces takes those components out of the candidate
// set. Tenant namespaces are skipped because the tenant chart owns LimitRange policy
// there: packages/apps/tenant ships tenant-range-limits, which defaults container memory
// to 128Mi in every tenant namespace that declares resourceQuotas. Handing a second,
// system-owned LimitRange to those namespaces would layer a competing default on top of
// the chart's own, so the operator stays out.
//
// The limit is a ceiling rather than a reservation, so it is set well above real usage —
// the point is that memory.max exists, not that it binds. It must still stay above the
// largest memory request in any system namespace, because a defaulted limit below a
// container's own request is rejected at admission; findRequestAboveDefaultLimit is the
// guard for that and the long comment there explains why it is needed in this repo.
func (r *PackageReconciler) reconcileSystemDefaultsLimitRange(ctx context.Context, nsName string) error {
logger := log.FromContext(ctx)
// Disabled: drop a LimitRange left over from an earlier configuration so the knob
// stays reversible.
if r.SystemNamespaceMemoryLimit.IsZero() {
return r.deleteSystemDefaultsLimitRange(ctx, nsName)
}
// Ownership decisions use a fresh read; writes also carry an object precondition.
existing := &corev1.LimitRange{}
err := r.memoryPolicyReader().Get(ctx, types.NamespacedName{Name: SystemDefaultsLimitRangeName, Namespace: nsName}, existing)
if err != nil && !apierrors.IsNotFound(err) {
return fmt.Errorf("failed to read the system defaults LimitRange in namespace %s: %w", nsName, err)
}
existingOwned := err == nil && existing.Labels[managedByLabel] == packageControllerFieldOwner
if err == nil && !existingOwned {
// The name itself is occupied by an object we do not own. Its spec does not
// have to mention memory: even a cpu-only object cannot coexist with the
// desired LimitRange under the same Kubernetes object key, and replacing its
// atomic Limits list would destroy the administrator's policy.
logger.Info("leaving the system defaults LimitRange unmanaged: its name is already held by an object this reconciler does not own",
"namespace", nsName,
"limitRange", existing.Name)
return nil
}
// Another LimitRange already governing container memory here makes this namespace one
// to stay out of, for the same reason tenant namespaces are excluded: LimitRanger
// applies defaults from every LimitRange in the namespace and then validates the pod
// against every one, and nothing orders them. Adding a second default leaves which one
// wins to the order the plugin happens to iterate, so a container's ceiling moves
// between the administrator's value and this one for reasons no chart author can see —
// exactly the hazard cited above to keep this reconciler out of tenant namespaces, and
// it does not stop being one because the namespace is a system namespace.
//
// A max is the sharper half. This reconciler writes a default and no max, so an
// administrator's LimitRange carrying a container memory max below the configured
// default has every container that declares no limit of its own rejected: the default