@@ -167,7 +167,7 @@ type Instance struct {
167167 // MaxStopDelay is the current MaxStopDelay of the cluster
168168 MaxStopDelay int32
169169
170- // SmartStopDelay is used to compute the timeout of smart shutdown by the formula `stopDelay - smartStopDelay`
170+ // SmartStopDelay is used to control PostgreSQL smart shutdown timeout
171171 SmartStopDelay int32
172172
173173 // canCheckReadiness specifies whether the instance can start being checked for readiness
@@ -260,17 +260,6 @@ func (instance *Instance) VerifyPgDataCoherence(ctx context.Context) error {
260260 return WritePostgresUserMaps (instance .PgData )
261261}
262262
263- // GetSmartShutdownTimeout gets the duration in seconds as the timeout of smart shutdown
264- // we calculate smart shutdown with following formula
265- // max(stopDelay - smartStopDelay, 30)
266- func (instance * Instance ) GetSmartShutdownTimeout () int32 {
267- timeout := instance .MaxStopDelay - instance .SmartStopDelay
268- if timeout < 30 {
269- timeout = 30
270- }
271- return timeout
272- }
273-
274263// InstanceCommand are commands for the goroutine managing postgres
275264type InstanceCommand string
276265
@@ -435,6 +424,72 @@ func (instance *Instance) Shutdown(options ShutdownOptions) error {
435424 return nil
436425}
437426
427+ // TryShuttingDownSmartFast first tries to shut down the instance with mode smart,
428+ // then in case of failure or the given timeout expiration,
429+ // it will issue a fast shutdown request and wait for it to complete.
430+ func (instance * Instance ) TryShuttingDownSmartFast () error {
431+ var err error
432+
433+ smartTimeout := instance .SmartStopDelay
434+ if instance .MaxStopDelay <= instance .SmartStopDelay {
435+ log .Warning ("Ignoring smartStopDelay" ,
436+ "smartStopDelay" , instance .SmartStopDelay ,
437+ "maxStopDelay" , instance .MaxStopDelay ,
438+ )
439+ smartTimeout = 0
440+ }
441+
442+ if smartTimeout > 0 {
443+ log .Info ("Requesting smart shutdown of the PostgreSQL instance" )
444+ err = instance .Shutdown (ShutdownOptions {
445+ Mode : ShutdownModeSmart ,
446+ Wait : true ,
447+ Timeout : & smartTimeout ,
448+ })
449+ if err != nil {
450+ log .Warning ("Error while handling the smart shutdown request" , "err" , err )
451+ }
452+ }
453+
454+ if err != nil || smartTimeout == 0 {
455+ log .Info ("Requesting fast shutdown of the PostgreSQL instance" )
456+ err = instance .Shutdown (ShutdownOptions {
457+ Mode : ShutdownModeFast ,
458+ Wait : true ,
459+ })
460+ }
461+ if err != nil {
462+ log .Error (err , "Error while shutting down the PostgreSQL instance" )
463+ return err
464+ }
465+
466+ log .Info ("PostgreSQL instance shut down" )
467+ return nil
468+ }
469+
470+ // TryShuttingDownFastImmediate first tries to shut down the instance with mode fast,
471+ // then in case of failure or the given timeout expiration,
472+ // it will issue an immediate shutdown request and wait for it to complete.
473+ // N.B. immediate shutdown can cause data loss.
474+ func (instance * Instance ) TryShuttingDownFastImmediate () error {
475+ log .Info ("Requesting fast shutdown of the PostgreSQL instance" )
476+ err := instance .Shutdown (ShutdownOptions {
477+ Mode : ShutdownModeFast ,
478+ Wait : true ,
479+ Timeout : & instance .MaxSwitchoverDelay ,
480+ })
481+ var exitError * exec.ExitError
482+ if errors .As (err , & exitError ) {
483+ log .Info ("Graceful shutdown failed. Issuing immediate shutdown" ,
484+ "exitCode" , exitError .ExitCode ())
485+ err = instance .Shutdown (ShutdownOptions {
486+ Mode : ShutdownModeImmediate ,
487+ Wait : true ,
488+ })
489+ }
490+ return err
491+ }
492+
438493// isStatusRunning checks the status of a running server using pg_ctl status
439494func (instance * Instance ) isStatusRunning () bool {
440495 options := []string {
@@ -1044,3 +1099,40 @@ func (instance *Instance) DropConnections() error {
10441099func (instance * Instance ) GetPrimaryConnInfo () string {
10451100 return buildPrimaryConnInfo (instance .ClusterName + "-rw" , instance .PodName )
10461101}
1102+
1103+ // HandleInstanceCommandRequests execute a command requested by the reconciliation
1104+ // loop.
1105+ func (instance * Instance ) HandleInstanceCommandRequests (
1106+ req InstanceCommand ,
1107+ ) (restartNeeded bool , err error ) {
1108+ if instance .IsFenced () {
1109+ switch req {
1110+ case FenceOff :
1111+ log .Info ("Fence lifting request received, will proceed with restarting the instance if needed" )
1112+ instance .SetFencing (false )
1113+ return true , nil
1114+ default :
1115+ log .Warning ("Received request while fencing, ignored" , "req" , req )
1116+ return false , nil
1117+ }
1118+ }
1119+ switch req {
1120+ case FenceOn :
1121+ log .Info ("Fencing request received, will proceed shutting down the instance" )
1122+ instance .SetFencing (true )
1123+ err := instance .TryShuttingDownSmartFast ()
1124+ if err != nil {
1125+ err = fmt .Errorf ("while shutting down the instance to fence it: %w" , err )
1126+ }
1127+ return false , err
1128+ case RestartSmartFast :
1129+ return true , instance .TryShuttingDownSmartFast ()
1130+ case ShutDownFastImmediate :
1131+ if err := instance .TryShuttingDownFastImmediate (); err != nil {
1132+ log .Error (err , "error shutting down instance, proceeding" )
1133+ }
1134+ return false , nil
1135+ default :
1136+ return false , fmt .Errorf ("unrecognized request: %s" , req )
1137+ }
1138+ }
0 commit comments