Skip to content

Commit ae42240

Browse files
litaocdlYanniHu1996armrugbartolini
committed
feat: change default value of startDelay to 3600 (#2847)
IMPORTANT: The default value of `startDelay` has now been changed to 3600 seconds, equivalent of 1 hour. Previously, it was set to 30 seconds. This patch also replaces the livenessProbe's initial delay with a more proper Kubernetes startup probe to deal with the start of a Postgres server. Both the startup probe and the higher default time should improve the self-healing capabilities of larger Postgres deployments where the default value of 30 seconds was not sufficient for example to complete a crash recovery operation, causing infinite restarts. Closes #2843 Signed-off-by: Tao Li <[email protected]> Signed-off-by: YanniHu1996 <[email protected]> Signed-off-by: Armando Ruocco <[email protected]> Signed-off-by: Gabriele Bartolini <[email protected]> Co-authored-by: YanniHu1996 <[email protected]> Co-authored-by: Armando Ruocco <[email protected]> Co-authored-by: Gabriele Bartolini <[email protected]>
1 parent 4cc1c78 commit ae42240

5 files changed

Lines changed: 57 additions & 14 deletions

File tree

‎api/v1/cluster_types.go‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -229,8 +229,10 @@ type ClusterSpec struct {
229229
WalStorage *StorageConfiguration `json:"walStorage,omitempty"`
230230

231231
// The time in seconds that is allowed for a PostgreSQL instance to
232-
// successfully start up (default 30)
233-
// +kubebuilder:default:=30
232+
// successfully start up (default 3600).
233+
// The startup probe failure threshold is derived from this value using the formula:
234+
// ceiling(startDelay / 10).
235+
// +kubebuilder:default:=3600
234236
// +optional
235237
MaxStartDelay int32 `json:"startDelay,omitempty"`
236238

@@ -913,6 +915,11 @@ const (
913915
// is gracefully shutdown during a switchover.
914916
// It is greater than one year in seconds, big enough to simulate an infinite timeout
915917
DefaultMaxSwitchoverDelay = 3600
918+
919+
// DefaultStartupDelay is the default value for startupDelay, startupDelay will be used to calculate the
920+
// FailureThreshold of startupProbe, the formula is `FailureThreshold = ceiling(startDelay / periodSeconds)`,
921+
// the minimum value is 1
922+
DefaultStartupDelay = 3600
916923
)
917924

918925
// PostgresConfiguration defines the PostgreSQL configuration
@@ -2308,7 +2315,7 @@ func (cluster *Cluster) GetMaxStartDelay() int32 {
23082315
if cluster.Spec.MaxStartDelay > 0 {
23092316
return cluster.Spec.MaxStartDelay
23102317
}
2311-
return 30
2318+
return DefaultStartupDelay
23122319
}
23132320

23142321
// GetMaxStopDelay get the amount of time PostgreSQL has to stop

‎config/crd/bases/postgresql.cnpg.io_clusters.yaml‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2974,9 +2974,11 @@ spec:
29742974
- metadata
29752975
type: object
29762976
startDelay:
2977-
default: 30
2978-
description: The time in seconds that is allowed for a PostgreSQL
2979-
instance to successfully start up (default 30)
2977+
default: 3600
2978+
description: 'The time in seconds that is allowed for a PostgreSQL
2979+
instance to successfully start up (default 3600). The startup probe
2980+
failure threshold is derived from this value using the formula:
2981+
ceiling(startDelay / 10).'
29802982
format: int32
29812983
type: integer
29822984
stopDelay:

‎docs/src/cloudnative-pg.v1.md‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1422,7 +1422,9 @@ user by setting it to <code>NULL</code>. Enabled by default.</p>
14221422
</td>
14231423
<td>
14241424
<p>The time in seconds that is allowed for a PostgreSQL instance to
1425-
successfully start up (default 30)</p>
1425+
successfully start up (default 3600).
1426+
The startup probe failure threshold is derived from this value using the formula:
1427+
ceiling(startDelay / 10).</p>
14261428
</td>
14271429
</tr>
14281430
<tr><td><code>stopDelay</code><br/>

‎pkg/specs/pods.go‎

Lines changed: 29 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ package specs
2020

2121
import (
2222
"fmt"
23+
"math"
2324
"reflect"
2425
"strconv"
2526

@@ -80,6 +81,12 @@ const (
8081

8182
// ReadinessProbePeriod is the period set for the postgres instance readiness probe
8283
ReadinessProbePeriod = 10
84+
85+
// StartupProbePeriod is the period set for the postgres instance startup probe
86+
StartupProbePeriod = 10
87+
88+
// LivenessProbePeriod is the period set for the postgres instance liveness probe
89+
LivenessProbePeriod = 10
8390
)
8491

8592
// EnvConfig carries the environment configuration of a container
@@ -183,6 +190,17 @@ func createPostgresContainers(cluster apiv1.Cluster, envConfig EnvConfig) []core
183190
Env: envConfig.EnvVars,
184191
EnvFrom: envConfig.EnvFrom,
185192
VolumeMounts: createPostgresVolumeMounts(cluster),
193+
StartupProbe: &corev1.Probe{
194+
FailureThreshold: getStartupProbeFailureThreshold(cluster.GetMaxStartDelay()),
195+
PeriodSeconds: StartupProbePeriod,
196+
TimeoutSeconds: 5,
197+
ProbeHandler: corev1.ProbeHandler{
198+
HTTPGet: &corev1.HTTPGetAction{
199+
Path: url.PathHealth,
200+
Port: intstr.FromInt32(int32(url.StatusPort)),
201+
},
202+
},
203+
},
186204
ReadinessProbe: &corev1.Probe{
187205
TimeoutSeconds: 5,
188206
PeriodSeconds: ReadinessProbePeriod,
@@ -193,14 +211,9 @@ func createPostgresContainers(cluster apiv1.Cluster, envConfig EnvConfig) []core
193211
},
194212
},
195213
},
196-
// From K8s 1.17 and newer, startup probes will be available for
197-
// all users and not just protected from feature gates. For now
198-
// let's use the LivenessProbe. When we will drop support for K8s
199-
// 1.16, we'll configure a StartupProbe and this will lead to a
200-
// better LivenessProbe (without InitialDelaySeconds).
201214
LivenessProbe: &corev1.Probe{
202-
InitialDelaySeconds: cluster.GetMaxStartDelay(),
203-
TimeoutSeconds: 5,
215+
PeriodSeconds: LivenessProbePeriod,
216+
TimeoutSeconds: 5,
204217
ProbeHandler: corev1.ProbeHandler{
205218
HTTPGet: &corev1.HTTPGetAction{
206219
Path: url.PathHealth,
@@ -240,6 +253,15 @@ func createPostgresContainers(cluster apiv1.Cluster, envConfig EnvConfig) []core
240253
return containers
241254
}
242255

256+
// getStartupProbeFailureThreshold get the startup probe failure threshold
257+
// FAILURE_THRESHOLD = ceil(startDelay / periodSeconds) and minimum value is 1
258+
func getStartupProbeFailureThreshold(startupDelay int32) int32 {
259+
if startupDelay <= StartupProbePeriod {
260+
return 1
261+
}
262+
return int32(math.Ceil(float64(startupDelay) / float64(StartupProbePeriod)))
263+
}
264+
243265
// CreateAffinitySection creates the affinity sections for Pods, given the configuration
244266
// from the user
245267
func CreateAffinitySection(clusterName string, config apiv1.AffinityConfiguration) *corev1.Affinity {

‎pkg/specs/pods_test.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -829,3 +829,13 @@ var _ = Describe("PodSpec drift detection", func() {
829829
Expect(specsMatch).To(BeFalse())
830830
})
831831
})
832+
833+
var _ = Describe("Compute startup probe failure threshold", func() {
834+
It("should take the minimum value 1", func() {
835+
Expect(getStartupProbeFailureThreshold(5)).To(BeNumerically("==", 1))
836+
})
837+
838+
It("should take the value from 'startDelay / periodSeconds'", func() {
839+
Expect(getStartupProbeFailureThreshold(109)).To(BeNumerically("==", 11))
840+
})
841+
})

0 commit comments

Comments
 (0)