Skip to content

Commit 197263b

Browse files
leonardocemnenciaarmrugbartolini
authored
feat: move shutdown logic to instance management (#3039)
This patch moves the shutdown logic from the lifecycle manager to the instance manager, simplifying its structure. The `.spec.smartStopDelay` field is used now as the time given to the PostgreSQL smart shutdown sequence. After that, we'll request PostgreSQL to shut down using the `fast` strategy within the overall `.spec.stopDelay` time, which is the `terminationGracePeriodSeconds` of the Pods. Closes #3032 Signed-off-by: Marco Nenciarini <[email protected]> Signed-off-by: Armando Ruocco <[email protected]> Signed-off-by: Gabriele Bartolini <[email protected]> Signed-off-by: Leonardo Cecchi <[email protected]> Co-authored-by: Marco Nenciarini <[email protected]> Co-authored-by: Armando Ruocco <[email protected]> Co-authored-by: Gabriele Bartolini <[email protected]>
1 parent 3a56af6 commit 197263b

8 files changed

Lines changed: 121 additions & 152 deletions

File tree

‎api/v1/cluster_types.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -280,7 +280,8 @@ type ClusterSpec struct {
280280
MaxStopDelay int32 `json:"stopDelay,omitempty"`
281281

282282
// The time in seconds that controls the window of time reserved for the smart shutdown of Postgres to complete.
283-
// this formula to compute the timeout of smart shutdown is `max(stopDelay - smartStopDelay, 30)`
283+
// Make sure you reserve enough time for the operator to request a fast shutdown of Postgres
284+
// (that is: `stopDelay` - `smartStopDelay`).
284285
// +kubebuilder:default:=180
285286
// +optional
286287
SmartStopDelay int32 `json:"smartStopDelay,omitempty"`

‎config/crd/bases/postgresql.cnpg.io_clusters.yaml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3036,9 +3036,9 @@ spec:
30363036
smartStopDelay:
30373037
default: 180
30383038
description: The time in seconds that controls the window of time
3039-
reserved for the smart shutdown of Postgres to complete. this formula
3040-
to compute the timeout of smart shutdown is `max(stopDelay - smartStopDelay,
3041-
30)`
3039+
reserved for the smart shutdown of Postgres to complete. Please
3040+
ensure that at least 15 seconds are left for the operator to request
3041+
a fast shutdown of Postgres.
30423042
format: int32
30433043
type: integer
30443044
startDelay:

‎docs/src/cloudnative-pg.v1.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1500,7 +1500,7 @@ gracefully shutdown (default 1800)</p>
15001500
</td>
15011501
<td>
15021502
<p>The time in seconds that controls the window of time reserved for the smart shutdown of Postgres to complete.
1503-
this formula to compute the timeout of smart shutdown is <code>max(stopDelay - smartStopDelay, 30)</code></p>
1503+
Make sure you reserve enough time for the operator to request a fast shutdown of Postgres.</p>
15041504
</td>
15051505
</tr>
15061506
<tr><td><code>switchoverDelay</code><br/>

‎docs/src/instance_manager.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,8 +62,8 @@ new connection to PostgreSQL. This step will last for up to
6262
shut down, terminating any existing connection and exiting promptly.
6363
If the instance is archiving and/or streaming WAL files, the process
6464
will wait for up to the remaining time set in `.spec.stopDelay` to complete the
65-
operation and then forcibly shut down. Such timeout is calculated using the
66-
following formula: `max(stopDelay - smartStopDelay, 30)`.
65+
operation and then forcibly shut down. Such a timeout needs to be at least 15
66+
seconds.
6767

6868
!!! Important
6969
In order to avoid any data loss in the Postgres cluster, which impacts

‎internal/cmd/manager/instance/run/lifecycle/lifecycle.go‎

Lines changed: 9 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ package lifecycle
1919
import (
2020
"context"
2121
"errors"
22-
"fmt"
2322
"os"
2423
"os/exec"
2524
"os/signal"
@@ -104,17 +103,15 @@ func (i *PostgresLifecycle) Start(ctx context.Context) error {
104103
}
105104
case <-ctx.Done():
106105
// The controller manager asked us to terminate our operations.
107-
// We shut down PostgreSQL and terminate using the maximum available
108-
// stop delay. We are doing that because we are not going to receive
109-
// a SIGKILL by the Kubelet, which is not informed about what's
110-
// happening.
106+
// We shut down PostgreSQL and terminate using the smart
107+
// stop delay.
111108
if i.instance.InstanceManagerIsUpgrading.Load() {
112109
log.Info("Context has been cancelled, but an instance manager online upgrade is in progress, " +
113110
"will just exit")
114111
return nil
115112
}
116113
log.Info("Context has been cancelled, shutting down and exiting")
117-
if err := tryShuttingDownSmartFast(i.instance.MaxStopDelay, i.instance); err != nil {
114+
if err := i.instance.TryShuttingDownSmartFast(); err != nil {
118115
log.Error(err, "error shutting down instance, proceeding")
119116
}
120117
return nil
@@ -124,9 +121,11 @@ func (i *PostgresLifecycle) Start(ctx context.Context) error {
124121
// to our process. In this case we terminate as fast as we can,
125122
// otherwise we'll receive a SIGKILL by the Kubelet, possibly
126123
// resulting in a data corruption.
127-
smartShutdownTimeout := i.instance.GetSmartShutdownTimeout()
128-
log.Info("Received termination signal", "signal", sig, "smartShutdownTimeout", smartShutdownTimeout)
129-
if err := tryShuttingDownSmartFast(smartShutdownTimeout, i.instance); err != nil {
124+
log.Info("Received termination signal",
125+
"signal", sig,
126+
"smartShutdownTimeout", i.instance.SmartStopDelay,
127+
)
128+
if err := i.instance.TryShuttingDownSmartFast(); err != nil {
130129
log.Error(err, "error while shutting down instance, proceeding")
131130
}
132131
return nil
@@ -137,7 +136,7 @@ func (i *PostgresLifecycle) Start(ctx context.Context) error {
137136
log.Info("Received request for postgres", "req", req)
138137

139138
// We execute the requested operation
140-
restartNeeded, err := i.handleInstanceCommandRequests(req)
139+
restartNeeded, err := i.instance.HandleInstanceCommandRequests(req)
141140
if err != nil {
142141
log.Error(err, "while handling instance command request")
143142
}
@@ -153,40 +152,3 @@ func (i *PostgresLifecycle) Start(ctx context.Context) error {
153152
// process
154153
}
155154
}
156-
157-
// handleInstanceCommandRequests execute a command requested by the reconciliation
158-
// loop.
159-
func (i *PostgresLifecycle) handleInstanceCommandRequests(
160-
req postgres.InstanceCommand,
161-
) (restartNeeded bool, err error) {
162-
if i.instance.IsFenced() {
163-
switch req {
164-
case postgres.FenceOff:
165-
log.Info("Fence lifting request received, will proceed with restarting the instance if needed")
166-
i.instance.SetFencing(false)
167-
return true, nil
168-
default:
169-
log.Warning("Received request while fencing, ignored", "req", req)
170-
return false, nil
171-
}
172-
}
173-
switch req {
174-
case postgres.FenceOn:
175-
log.Info("Fencing request received, will proceed shutting down the instance")
176-
i.instance.SetFencing(true)
177-
err := tryShuttingDownSmartFast(i.instance.MaxStopDelay, i.instance)
178-
if err != nil {
179-
err = fmt.Errorf("while shutting down the instance to fence it: %w", err)
180-
}
181-
return false, err
182-
case postgres.RestartSmartFast:
183-
return true, tryShuttingDownSmartFast(i.instance.MaxStopDelay, i.instance)
184-
case postgres.ShutDownFastImmediate:
185-
if err := tryShuttingDownFastImmediate(i.instance.MaxSwitchoverDelay, i.instance); err != nil {
186-
log.Error(err, "error shutting down instance, proceeding")
187-
}
188-
return false, nil
189-
default:
190-
return false, fmt.Errorf("unrecognized request: %s", req)
191-
}
192-
}

‎internal/cmd/manager/instance/run/lifecycle/shutdown.go‎

Lines changed: 0 additions & 74 deletions
This file was deleted.

‎pkg/management/postgres/instance.go‎

Lines changed: 104 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,7 @@ type Instance struct {
167167
// MaxStopDelay is the current MaxStopDelay of the cluster
168168
MaxStopDelay int32
169169

170-
// SmartStopDelay is used to compute the timeout of smart shutdown by the formula `stopDelay - smartStopDelay`
170+
// SmartStopDelay is used to control PostgreSQL smart shutdown timeout
171171
SmartStopDelay int32
172172

173173
// canCheckReadiness specifies whether the instance can start being checked for readiness
@@ -260,17 +260,6 @@ func (instance *Instance) VerifyPgDataCoherence(ctx context.Context) error {
260260
return WritePostgresUserMaps(instance.PgData)
261261
}
262262

263-
// GetSmartShutdownTimeout gets the duration in seconds as the timeout of smart shutdown
264-
// we calculate smart shutdown with following formula
265-
// max(stopDelay - smartStopDelay, 30)
266-
func (instance *Instance) GetSmartShutdownTimeout() int32 {
267-
timeout := instance.MaxStopDelay - instance.SmartStopDelay
268-
if timeout < 30 {
269-
timeout = 30
270-
}
271-
return timeout
272-
}
273-
274263
// InstanceCommand are commands for the goroutine managing postgres
275264
type InstanceCommand string
276265

@@ -435,6 +424,72 @@ func (instance *Instance) Shutdown(options ShutdownOptions) error {
435424
return nil
436425
}
437426

427+
// TryShuttingDownSmartFast first tries to shut down the instance with mode smart,
428+
// then in case of failure or the given timeout expiration,
429+
// it will issue a fast shutdown request and wait for it to complete.
430+
func (instance *Instance) TryShuttingDownSmartFast() error {
431+
var err error
432+
433+
smartTimeout := instance.SmartStopDelay
434+
if instance.MaxStopDelay <= instance.SmartStopDelay {
435+
log.Warning("Ignoring smartStopDelay",
436+
"smartStopDelay", instance.SmartStopDelay,
437+
"maxStopDelay", instance.MaxStopDelay,
438+
)
439+
smartTimeout = 0
440+
}
441+
442+
if smartTimeout > 0 {
443+
log.Info("Requesting smart shutdown of the PostgreSQL instance")
444+
err = instance.Shutdown(ShutdownOptions{
445+
Mode: ShutdownModeSmart,
446+
Wait: true,
447+
Timeout: &smartTimeout,
448+
})
449+
if err != nil {
450+
log.Warning("Error while handling the smart shutdown request", "err", err)
451+
}
452+
}
453+
454+
if err != nil || smartTimeout == 0 {
455+
log.Info("Requesting fast shutdown of the PostgreSQL instance")
456+
err = instance.Shutdown(ShutdownOptions{
457+
Mode: ShutdownModeFast,
458+
Wait: true,
459+
})
460+
}
461+
if err != nil {
462+
log.Error(err, "Error while shutting down the PostgreSQL instance")
463+
return err
464+
}
465+
466+
log.Info("PostgreSQL instance shut down")
467+
return nil
468+
}
469+
470+
// TryShuttingDownFastImmediate first tries to shut down the instance with mode fast,
471+
// then in case of failure or the given timeout expiration,
472+
// it will issue an immediate shutdown request and wait for it to complete.
473+
// N.B. immediate shutdown can cause data loss.
474+
func (instance *Instance) TryShuttingDownFastImmediate() error {
475+
log.Info("Requesting fast shutdown of the PostgreSQL instance")
476+
err := instance.Shutdown(ShutdownOptions{
477+
Mode: ShutdownModeFast,
478+
Wait: true,
479+
Timeout: &instance.MaxSwitchoverDelay,
480+
})
481+
var exitError *exec.ExitError
482+
if errors.As(err, &exitError) {
483+
log.Info("Graceful shutdown failed. Issuing immediate shutdown",
484+
"exitCode", exitError.ExitCode())
485+
err = instance.Shutdown(ShutdownOptions{
486+
Mode: ShutdownModeImmediate,
487+
Wait: true,
488+
})
489+
}
490+
return err
491+
}
492+
438493
// isStatusRunning checks the status of a running server using pg_ctl status
439494
func (instance *Instance) isStatusRunning() bool {
440495
options := []string{
@@ -1061,3 +1116,40 @@ func (instance *Instance) DropConnections() error {
10611116
func (instance *Instance) GetPrimaryConnInfo() string {
10621117
return buildPrimaryConnInfo(instance.ClusterName+"-rw", instance.PodName)
10631118
}
1119+
1120+
// HandleInstanceCommandRequests execute a command requested by the reconciliation
1121+
// loop.
1122+
func (instance *Instance) HandleInstanceCommandRequests(
1123+
req InstanceCommand,
1124+
) (restartNeeded bool, err error) {
1125+
if instance.IsFenced() {
1126+
switch req {
1127+
case FenceOff:
1128+
log.Info("Fence lifting request received, will proceed with restarting the instance if needed")
1129+
instance.SetFencing(false)
1130+
return true, nil
1131+
default:
1132+
log.Warning("Received request while fencing, ignored", "req", req)
1133+
return false, nil
1134+
}
1135+
}
1136+
switch req {
1137+
case FenceOn:
1138+
log.Info("Fencing request received, will proceed shutting down the instance")
1139+
instance.SetFencing(true)
1140+
err := instance.TryShuttingDownSmartFast()
1141+
if err != nil {
1142+
err = fmt.Errorf("while shutting down the instance to fence it: %w", err)
1143+
}
1144+
return false, err
1145+
case RestartSmartFast:
1146+
return true, instance.TryShuttingDownSmartFast()
1147+
case ShutDownFastImmediate:
1148+
if err := instance.TryShuttingDownFastImmediate(); err != nil {
1149+
log.Error(err, "error shutting down instance, proceeding")
1150+
}
1151+
return false, nil
1152+
default:
1153+
return false, fmt.Errorf("unrecognized request: %s", req)
1154+
}
1155+
}

‎pkg/management/postgres/instance_test.go‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -224,15 +224,3 @@ var _ = Describe("check atomic bool", func() {
224224
Expect(unAvailable).To(BeTrue())
225225
})
226226
})
227-
228-
var _ = Describe("testing the compute of smart shutdown time", func() {
229-
It("should be 30 as the result of 'MaxStopDelay - SmartStopDelay' is less than it", func() {
230-
instance := Instance{MaxStopDelay: 100, SmartStopDelay: 100}
231-
Expect(instance.GetSmartShutdownTimeout()).To(BeNumerically("==", 30))
232-
})
233-
234-
It("should be the result of 'MaxStopDelay - SmartStopDelay'", func() {
235-
instance := Instance{MaxStopDelay: 200, SmartStopDelay: 100}
236-
Expect(instance.GetSmartShutdownTimeout()).To(BeNumerically("==", 100))
237-
})
238-
})

0 commit comments

Comments
 (0)