Skip to content

Commit e410b02

Browse files
myasnikovdaniilmyasnikovdaniil
committed
feat(operator): expose Install/Upgrade timeouts and MaxHistory as flags
The HelmRelease generation in package_reconciler hardcoded three values that have caused recurring pain: - Install.Timeout = 10m (line 214) - Upgrade.Timeout = 10m (line 220) - MaxHistory = unset (Helm default 5) The 10m timeout in particular contradicts the per-Application timeout work done in `pkg/config/config.go` and `pkg/registry/apps/application/rest.go` (commit 7b146cb), which lets individual Applications override the install/upgrade timeout via annotation. The PackageSource side had no equivalent — etcd works around it by hand-rolling its HR YAML in `packages/apps/tenant/templates/etcd.yaml` to set a 30m timeout, harbor has had multiple commits chasing the same problem from the other side. This closes that gap. New operator flags (each maps to a chart value with empty default; operator uses its own default when value is empty): - `--helmrelease-install-timeout` (default 10m) → Spec.Install.Timeout - `--helmrelease-upgrade-timeout` (default 10m) → Spec.Upgrade.Timeout - `--helmrelease-max-history` (default 5) → Spec.MaxHistory Production behaviour unchanged. E2E and edge cases (cert rotation, slow-installing charts) can override per-cluster without modifying chart manifests. Per-component overrides on `ComponentInstall` (mirroring the existing `UpgradeCRDs` pattern) would be a strictly better fix for charts like etcd that need an outlier timeout — left as a deliberate followup since it touches the Package CR API. Signed-off-by: Myasnikov Daniil <[email protected]>
1 parent c4c70e0 commit e410b02

4 files changed

Lines changed: 67 additions & 10 deletions

File tree

‎cmd/cozystack-operator/main.go‎

Lines changed: 36 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,9 @@ func main() {
8585
var telemetryInterval string
8686
var helmReleaseInterval string
8787
var helmReleaseRetryInterval string
88+
var helmReleaseInstallTimeout string
89+
var helmReleaseUpgradeTimeout string
90+
var helmReleaseMaxHistory int
8891
var cozyValuesSecretName string
8992
var cozyValuesSecretNamespace string
9093
var cozyValuesNamespaceSelector string
@@ -119,6 +122,18 @@ func main() {
119122
"controller waits between failed install/upgrade attempts. Decoupled from --helmrelease-interval "+
120123
"(which is the healthy reconcile cadence) so failures recover fast without polling healthy "+
121124
"releases at the same fast cadence.")
125+
flag.StringVar(&helmReleaseInstallTimeout, "helmrelease-install-timeout", "10m",
126+
"Timeout for the Helm install action of HelmReleases created by the Package reconciler "+
127+
"(Spec.Install.Timeout). Bounds how long an individual Kubernetes operation (Job/hook/wait) "+
128+
"may take during install.")
129+
flag.StringVar(&helmReleaseUpgradeTimeout, "helmrelease-upgrade-timeout", "10m",
130+
"Timeout for the Helm upgrade action of HelmReleases created by the Package reconciler "+
131+
"(Spec.Upgrade.Timeout). Bounds how long an individual Kubernetes operation (Job/hook/wait) "+
132+
"may take during upgrade.")
133+
flag.IntVar(&helmReleaseMaxHistory, "helmrelease-max-history", 5,
134+
"Number of release revisions Helm keeps for HelmReleases created by the Package reconciler "+
135+
"(Spec.MaxHistory). 0 means unlimited; 5 matches Helm's default. Lower values reduce "+
136+
"per-release Secret accumulation in clusters that bounce HRs frequently (e.g. E2E sandboxes).")
122137
flag.StringVar(&platformSourceURL, "platform-source-url", "", "Platform source URL (oci:// or https://). If specified, generates OCIRepository or GitRepository resource.")
123138
flag.StringVar(&platformSourceName, "platform-source-name", "cozystack-platform", "Name for the generated platform source resource and PackageSource")
124139
flag.StringVar(&platformSourceRef, "platform-source-ref", "", "Reference specification as key=value pairs (e.g., 'branch=main' or 'digest=sha256:...,tag=v1.0'). For OCI: digest, semver, semverFilter, tag. For Git: branch, tag, semver, name, commit.")
@@ -144,6 +159,20 @@ func main() {
144159
setupLog.Error(err, "invalid --helmrelease-retry-interval value", "value", helmReleaseRetryInterval)
145160
os.Exit(1)
146161
}
162+
hrInstallTimeoutDuration, err := time.ParseDuration(helmReleaseInstallTimeout)
163+
if err != nil {
164+
setupLog.Error(err, "invalid --helmrelease-install-timeout value", "value", helmReleaseInstallTimeout)
165+
os.Exit(1)
166+
}
167+
hrUpgradeTimeoutDuration, err := time.ParseDuration(helmReleaseUpgradeTimeout)
168+
if err != nil {
169+
setupLog.Error(err, "invalid --helmrelease-upgrade-timeout value", "value", helmReleaseUpgradeTimeout)
170+
os.Exit(1)
171+
}
172+
if helmReleaseMaxHistory < 0 {
173+
setupLog.Error(fmt.Errorf("--helmrelease-max-history must be >= 0"), "invalid value", "value", helmReleaseMaxHistory)
174+
os.Exit(1)
175+
}
147176

148177
config := ctrl.GetConfigOrDie()
149178

@@ -281,10 +310,13 @@ func main() {
281310

282311
// Setup Package reconciler
283312
if err := (&operator.PackageReconciler{
284-
Client: mgr.GetClient(),
285-
Scheme: mgr.GetScheme(),
286-
HelmReleaseInterval: hrIntervalDuration,
287-
HelmReleaseRetryInterval: hrRetryIntervalDuration,
313+
Client: mgr.GetClient(),
314+
Scheme: mgr.GetScheme(),
315+
HelmReleaseInterval: hrIntervalDuration,
316+
HelmReleaseRetryInterval: hrRetryIntervalDuration,
317+
HelmReleaseInstallTimeout: hrInstallTimeoutDuration,
318+
HelmReleaseUpgradeTimeout: hrUpgradeTimeoutDuration,
319+
HelmReleaseMaxHistory: helmReleaseMaxHistory,
288320
}).SetupWithManager(mgr); err != nil {
289321
setupLog.Error(err, "unable to create controller", "controller", "Package")
290322
os.Exit(1)

‎internal/operator/package_reconciler.go‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -59,9 +59,12 @@ func parseCRDPolicy(install *cozyv1alpha1.ComponentInstall) helmv2.CRDsPolicy {
5959
// PackageReconciler reconciles Package resources
6060
type PackageReconciler struct {
6161
client.Client
62-
Scheme *runtime.Scheme
63-
HelmReleaseInterval time.Duration
64-
HelmReleaseRetryInterval time.Duration
62+
Scheme *runtime.Scheme
63+
HelmReleaseInterval time.Duration
64+
HelmReleaseRetryInterval time.Duration
65+
HelmReleaseInstallTimeout time.Duration
66+
HelmReleaseUpgradeTimeout time.Duration
67+
HelmReleaseMaxHistory int
6568
}
6669

6770
// +kubebuilder:rbac:groups=cozystack.io,resources=packages,verbs=get;list;watch;create;update;patch;delete
@@ -217,14 +220,15 @@ func (r *PackageReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
217220
Labels: labels,
218221
},
219222
Spec: helmv2.HelmReleaseSpec{
220-
Interval: metav1.Duration{Duration: r.HelmReleaseInterval},
223+
Interval: metav1.Duration{Duration: r.HelmReleaseInterval},
224+
MaxHistory: &r.HelmReleaseMaxHistory,
221225
ChartRef: &helmv2.CrossNamespaceSourceReference{
222226
Kind: "ExternalArtifact",
223227
Name: artifactName,
224228
Namespace: "cozy-system",
225229
},
226230
Install: &helmv2.Install{
227-
Timeout: &metav1.Duration{Duration: 10 * time.Minute},
231+
Timeout: &metav1.Duration{Duration: r.HelmReleaseInstallTimeout},
228232
// Strategy=RetryOnFailure (with RetryInterval) replaces the previous
229233
// Remediation{Retries:-1} setup. Functionally equivalent ("retry forever
230234
// on failure"), but decouples retry timing from spec.Interval so failed
@@ -236,7 +240,7 @@ func (r *PackageReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
236240
},
237241
},
238242
Upgrade: &helmv2.Upgrade{
239-
Timeout: &metav1.Duration{Duration: 10 * time.Minute},
243+
Timeout: &metav1.Duration{Duration: r.HelmReleaseUpgradeTimeout},
240244
Strategy: &helmv2.UpgradeStrategy{
241245
Name: string(helmv2.ActionStrategyRetryOnFailure),
242246
RetryInterval: &metav1.Duration{Duration: r.HelmReleaseRetryInterval},

‎packages/core/installer/templates/cozystack-operator.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,15 @@ spec:
7474
{{- if .Values.cozystackOperator.helmReleaseRetryInterval }}
7575
- --helmrelease-retry-interval={{ .Values.cozystackOperator.helmReleaseRetryInterval }}
7676
{{- end }}
77+
{{- if .Values.cozystackOperator.helmReleaseInstallTimeout }}
78+
- --helmrelease-install-timeout={{ .Values.cozystackOperator.helmReleaseInstallTimeout }}
79+
{{- end }}
80+
{{- if .Values.cozystackOperator.helmReleaseUpgradeTimeout }}
81+
- --helmrelease-upgrade-timeout={{ .Values.cozystackOperator.helmReleaseUpgradeTimeout }}
82+
{{- end }}
83+
{{- if .Values.cozystackOperator.helmReleaseMaxHistory }}
84+
- --helmrelease-max-history={{ .Values.cozystackOperator.helmReleaseMaxHistory }}
85+
{{- end }}
7786
- --platform-source-name=cozystack-platform
7887
- --platform-source-url={{ .Values.cozystackOperator.platformSourceUrl }}
7988
{{- if .Values.cozystackOperator.platformSourceRef }}

‎packages/core/installer/values.yaml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,18 @@ cozystackOperator:
1111
# (operator default: 30s). Controls how fast failed install/upgrade attempts
1212
# are retried. Decoupled from helmReleaseInterval (healthy reconcile cadence).
1313
helmReleaseRetryInterval: ""
14+
# When non-empty, overrides the operator's --helmrelease-install-timeout flag
15+
# (operator default: 10m). Bounds individual k8s operations (job/hook/wait)
16+
# during Helm install. Charts with long-running install hooks may need longer.
17+
helmReleaseInstallTimeout: ""
18+
# When non-empty, overrides the operator's --helmrelease-upgrade-timeout flag
19+
# (operator default: 10m). Same semantics as helmReleaseInstallTimeout, for
20+
# the Helm upgrade action.
21+
helmReleaseUpgradeTimeout: ""
22+
# When non-empty, overrides the operator's --helmrelease-max-history flag
23+
# (operator default: 5). Number of release revisions Helm keeps per HR.
24+
# Use 0 for unlimited; lower values reduce Secret accumulation in test clusters.
25+
helmReleaseMaxHistory: ""
1426
# Generic variant configuration (only used when cozystackOperator.variant=generic)
1527
cozystack:
1628
# Kubernetes API server host (IP only, no protocol/port)

0 commit comments

Comments
 (0)