Skip to content

Commit 3e4dfad

Browse files
leonardocemnenciaarmrugbartolini
committed
feat: move shutdown logic to instance management (#3039)
This patch moves the shutdown logic from the lifecycle manager to the instance manager, simplifying its structure. The `.spec.smartStopDelay` field is used now as the time given to the PostgreSQL smart shutdown sequence. After that, we'll request PostgreSQL to shut down using the `fast` strategy within the overall `.spec.stopDelay` time, which is the `terminationGracePeriodSeconds` of the Pods. Closes #3032 Signed-off-by: Marco Nenciarini <[email protected]> Signed-off-by: Armando Ruocco <[email protected]> Signed-off-by: Gabriele Bartolini <[email protected]> Signed-off-by: Leonardo Cecchi <[email protected]> Co-authored-by: Marco Nenciarini <[email protected]> Co-authored-by: Armando Ruocco <[email protected]> Co-authored-by: Gabriele Bartolini <[email protected]>
1 parent e18d485 commit 3e4dfad

8 files changed

Lines changed: 123 additions & 142 deletions

File tree

‎api/v1/cluster_types.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -243,7 +243,8 @@ type ClusterSpec struct {
243243
MaxStopDelay int32 `json:"stopDelay,omitempty"`
244244

245245
// The time in seconds that controls the window of time reserved for the smart shutdown of Postgres to complete.
246-
// this formula to compute the timeout of smart shutdown is `max(stopDelay - smartStopDelay, 30)`,
246+
// Make sure you reserve enough time for the operator to request a fast shutdown of Postgres
247+
// (that is: `stopDelay` - `smartStopDelay`).
247248
// +kubebuilder:default:=180
248249
// +optional
249250
SmartStopDelay int32 `json:"smartStopDelay,omitempty"`

‎config/crd/bases/postgresql.cnpg.io_clusters.yaml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2884,10 +2884,10 @@ spec:
28842884
type: object
28852885
smartStopDelay:
28862886
default: 180
2887-
description: The time in seconds that controls the window of time
2888-
reserved for the smart shutdown of Postgres to complete. this formula
2889-
to compute the timeout of smart shutdown is `max(stopDelay - smartStopDelay,
2890-
30)`,
2887+
description: 'The time in seconds that controls the window of time
2888+
reserved for the smart shutdown of Postgres to complete. Make sure
2889+
you reserve enough time for the operator to request a fast shutdown
2890+
of Postgres (that is: `stopDelay` - `smartStopDelay`).'
28912891
format: int32
28922892
type: integer
28932893
startDelay:

‎docs/src/cloudnative-pg.v1.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1440,7 +1440,8 @@ gracefully shutdown (default 1800)</p>
14401440
</td>
14411441
<td>
14421442
<p>The time in seconds that controls the window of time reserved for the smart shutdown of Postgres to complete.
1443-
this formula to compute the timeout of smart shutdown is <code>max(stopDelay - smartStopDelay, 30)</code>,</p>
1443+
Make sure you reserve enough time for the operator to request a fast shutdown of Postgres
1444+
(that is: <code>stopDelay</code> - <code>smartStopDelay</code>).</p>
14441445
</td>
14451446
</tr>
14461447
<tr><td><code>switchoverDelay</code><br/>

‎docs/src/instance_manager.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,8 +62,8 @@ new connection to PostgreSQL. This step will last for up to
6262
shut down, terminating any existing connection and exiting promptly.
6363
If the instance is archiving and/or streaming WAL files, the process
6464
will wait for up to the remaining time set in `.spec.stopDelay` to complete the
65-
operation and then forcibly shut down. Such timeout is calculated using the
66-
following formula: `max(stopDelay - smartStopDelay, 30)`.
65+
operation and then forcibly shut down. Such a timeout needs to be at least 15
66+
seconds.
6767

6868
!!! Important
6969
In order to avoid any data loss in the Postgres cluster, which impacts

‎internal/cmd/manager/instance/run/lifecycle/lifecycle.go‎

Lines changed: 9 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ package lifecycle
1919
import (
2020
"context"
2121
"errors"
22-
"fmt"
2322
"os"
2423
"os/exec"
2524
"os/signal"
@@ -104,17 +103,15 @@ func (i *PostgresLifecycle) Start(ctx context.Context) error {
104103
}
105104
case <-ctx.Done():
106105
// The controller manager asked us to terminate our operations.
107-
// We shut down PostgreSQL and terminate using the maximum available
108-
// stop delay. We are doing that because we are not going to receive
109-
// a SIGKILL by the Kubelet, which is not informed about what's
110-
// happening.
106+
// We shut down PostgreSQL and terminate using the smart
107+
// stop delay.
111108
if i.instance.InstanceManagerIsUpgrading.Load() {
112109
log.Info("Context has been cancelled, but an instance manager online upgrade is in progress, " +
113110
"will just exit")
114111
return nil
115112
}
116113
log.Info("Context has been cancelled, shutting down and exiting")
117-
if err := tryShuttingDownSmartFast(i.instance.MaxStopDelay, i.instance); err != nil {
114+
if err := i.instance.TryShuttingDownSmartFast(); err != nil {
118115
log.Error(err, "error shutting down instance, proceeding")
119116
}
120117
return nil
@@ -124,9 +121,11 @@ func (i *PostgresLifecycle) Start(ctx context.Context) error {
124121
// to our process. In this case we terminate as fast as we can,
125122
// otherwise we'll receive a SIGKILL by the Kubelet, possibly
126123
// resulting in a data corruption.
127-
smartShutdownTimeout := i.instance.GetSmartShutdownTimeout()
128-
log.Info("Received termination signal", "signal", sig, "smartShutdownTimeout", smartShutdownTimeout)
129-
if err := tryShuttingDownSmartFast(smartShutdownTimeout, i.instance); err != nil {
124+
log.Info("Received termination signal",
125+
"signal", sig,
126+
"smartShutdownTimeout", i.instance.SmartStopDelay,
127+
)
128+
if err := i.instance.TryShuttingDownSmartFast(); err != nil {
130129
log.Error(err, "error while shutting down instance, proceeding")
131130
}
132131
return nil
@@ -137,7 +136,7 @@ func (i *PostgresLifecycle) Start(ctx context.Context) error {
137136
log.Info("Received request for postgres", "req", req)
138137

139138
// We execute the requested operation
140-
restartNeeded, err := i.handleInstanceCommandRequests(req)
139+
restartNeeded, err := i.instance.HandleInstanceCommandRequests(req)
141140
if err != nil {
142141
log.Error(err, "while handling instance command request")
143142
}
@@ -153,40 +152,3 @@ func (i *PostgresLifecycle) Start(ctx context.Context) error {
153152
// process
154153
}
155154
}
156-
157-
// handleInstanceCommandRequests execute a command requested by the reconciliation
158-
// loop.
159-
func (i *PostgresLifecycle) handleInstanceCommandRequests(
160-
req postgres.InstanceCommand,
161-
) (restartNeeded bool, err error) {
162-
if i.instance.IsFenced() {
163-
switch req {
164-
case postgres.FenceOff:
165-
log.Info("Fence lifting request received, will proceed with restarting the instance if needed")
166-
i.instance.SetFencing(false)
167-
return true, nil
168-
default:
169-
log.Warning("Received request while fencing, ignored", "req", req)
170-
return false, nil
171-
}
172-
}
173-
switch req {
174-
case postgres.FenceOn:
175-
log.Info("Fencing request received, will proceed shutting down the instance")
176-
i.instance.SetFencing(true)
177-
err := tryShuttingDownSmartFast(i.instance.MaxStopDelay, i.instance)
178-
if err != nil {
179-
err = fmt.Errorf("while shutting down the instance to fence it: %w", err)
180-
}
181-
return false, err
182-
case postgres.RestartSmartFast:
183-
return true, tryShuttingDownSmartFast(i.instance.MaxStopDelay, i.instance)
184-
case postgres.ShutDownFastImmediate:
185-
if err := tryShuttingDownFastImmediate(i.instance.MaxSwitchoverDelay, i.instance); err != nil {
186-
log.Error(err, "error shutting down instance, proceeding")
187-
}
188-
return false, nil
189-
default:
190-
return false, fmt.Errorf("unrecognized request: %s", req)
191-
}
192-
}

‎internal/cmd/manager/instance/run/lifecycle/shutdown.go‎

Lines changed: 0 additions & 74 deletions
This file was deleted.

‎pkg/management/postgres/instance.go‎

Lines changed: 104 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,7 @@ type Instance struct {
167167
// MaxStopDelay is the current MaxStopDelay of the cluster
168168
MaxStopDelay int32
169169

170-
// SmartStopDelay is used to compute the timeout of smart shutdown by the formula `stopDelay - smartStopDelay`
170+
// SmartStopDelay is used to control PostgreSQL smart shutdown timeout
171171
SmartStopDelay int32
172172

173173
// canCheckReadiness specifies whether the instance can start being checked for readiness
@@ -405,6 +405,72 @@ func (instance *Instance) Shutdown(options ShutdownOptions) error {
405405
return nil
406406
}
407407

408+
// TryShuttingDownSmartFast first tries to shut down the instance with mode smart,
409+
// then in case of failure or the given timeout expiration,
410+
// it will issue a fast shutdown request and wait for it to complete.
411+
func (instance *Instance) TryShuttingDownSmartFast() error {
412+
var err error
413+
414+
smartTimeout := instance.SmartStopDelay
415+
if instance.MaxStopDelay <= instance.SmartStopDelay {
416+
log.Warning("Ignoring smartStopDelay",
417+
"smartStopDelay", instance.SmartStopDelay,
418+
"maxStopDelay", instance.MaxStopDelay,
419+
)
420+
smartTimeout = 0
421+
}
422+
423+
if smartTimeout > 0 {
424+
log.Info("Requesting smart shutdown of the PostgreSQL instance")
425+
err = instance.Shutdown(ShutdownOptions{
426+
Mode: ShutdownModeSmart,
427+
Wait: true,
428+
Timeout: &smartTimeout,
429+
})
430+
if err != nil {
431+
log.Warning("Error while handling the smart shutdown request", "err", err)
432+
}
433+
}
434+
435+
if err != nil || smartTimeout == 0 {
436+
log.Info("Requesting fast shutdown of the PostgreSQL instance")
437+
err = instance.Shutdown(ShutdownOptions{
438+
Mode: ShutdownModeFast,
439+
Wait: true,
440+
})
441+
}
442+
if err != nil {
443+
log.Error(err, "Error while shutting down the PostgreSQL instance")
444+
return err
445+
}
446+
447+
log.Info("PostgreSQL instance shut down")
448+
return nil
449+
}
450+
451+
// TryShuttingDownFastImmediate first tries to shut down the instance with mode fast,
452+
// then in case of failure or the given timeout expiration,
453+
// it will issue an immediate shutdown request and wait for it to complete.
454+
// N.B. immediate shutdown can cause data loss.
455+
func (instance *Instance) TryShuttingDownFastImmediate() error {
456+
log.Info("Requesting fast shutdown of the PostgreSQL instance")
457+
err := instance.Shutdown(ShutdownOptions{
458+
Mode: ShutdownModeFast,
459+
Wait: true,
460+
Timeout: &instance.MaxSwitchoverDelay,
461+
})
462+
var exitError *exec.ExitError
463+
if errors.As(err, &exitError) {
464+
log.Info("Graceful shutdown failed. Issuing immediate shutdown",
465+
"exitCode", exitError.ExitCode())
466+
err = instance.Shutdown(ShutdownOptions{
467+
Mode: ShutdownModeImmediate,
468+
Wait: true,
469+
})
470+
}
471+
return err
472+
}
473+
408474
// isStatusRunning checks the status of a running server using pg_ctl status
409475
func (instance *Instance) isStatusRunning() bool {
410476
options := []string{
@@ -1014,3 +1080,40 @@ func (instance *Instance) DropConnections() error {
10141080
func (instance *Instance) GetPrimaryConnInfo() string {
10151081
return buildPrimaryConnInfo(instance.ClusterName+"-rw", instance.PodName)
10161082
}
1083+
1084+
// HandleInstanceCommandRequests execute a command requested by the reconciliation
1085+
// loop.
1086+
func (instance *Instance) HandleInstanceCommandRequests(
1087+
req InstanceCommand,
1088+
) (restartNeeded bool, err error) {
1089+
if instance.IsFenced() {
1090+
switch req {
1091+
case FenceOff:
1092+
log.Info("Fence lifting request received, will proceed with restarting the instance if needed")
1093+
instance.SetFencing(false)
1094+
return true, nil
1095+
default:
1096+
log.Warning("Received request while fencing, ignored", "req", req)
1097+
return false, nil
1098+
}
1099+
}
1100+
switch req {
1101+
case FenceOn:
1102+
log.Info("Fencing request received, will proceed shutting down the instance")
1103+
instance.SetFencing(true)
1104+
err := instance.TryShuttingDownSmartFast()
1105+
if err != nil {
1106+
err = fmt.Errorf("while shutting down the instance to fence it: %w", err)
1107+
}
1108+
return false, err
1109+
case RestartSmartFast:
1110+
return true, instance.TryShuttingDownSmartFast()
1111+
case ShutDownFastImmediate:
1112+
if err := instance.TryShuttingDownFastImmediate(); err != nil {
1113+
log.Error(err, "error shutting down instance, proceeding")
1114+
}
1115+
return false, nil
1116+
default:
1117+
return false, fmt.Errorf("unrecognized request: %s", req)
1118+
}
1119+
}

‎pkg/management/postgres/instance_test.go‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -224,15 +224,3 @@ var _ = Describe("check atomic bool", func() {
224224
Expect(unAvailable).To(BeTrue())
225225
})
226226
})
227-
228-
var _ = Describe("testing the compute of smart shutdown time", func() {
229-
It("should be 30 as the result of 'MaxStopDelay - SmartStopDelay' is less than it", func() {
230-
instance := Instance{MaxStopDelay: 100, SmartStopDelay: 100}
231-
Expect(instance.GetSmartShutdownTimeout()).To(BeNumerically("==", 30))
232-
})
233-
234-
It("should be the result of 'MaxStopDelay - SmartStopDelay'", func() {
235-
instance := Instance{MaxStopDelay: 200, SmartStopDelay: 100}
236-
Expect(instance.GetSmartShutdownTimeout()).To(BeNumerically("==", 100))
237-
})
238-
})

0 commit comments

Comments
 (0)