Skip to content

Commit f6d03dd

Browse files
mnenciaarmru
mnencia
and
armru
authored
fix(fencing): allow fencing when WAL disk is full (#10302)
Move the WAL disk space check from the instance manager startup path into the PostgreSQL lifecycle loop, after the fencing check. This ensures the controller-runtime manager always starts, allowing fencing annotations to be processed even when WAL disk is full. Signed-off-by: Marco Nenciarini <[email protected]> Signed-off-by: Armando Ruocco <[email protected]> Co-authored-by: Armando Ruocco <[email protected]>
1 parent e20d4d5 commit f6d03dd

3 files changed

Lines changed: 23 additions & 17 deletions

File tree

‎internal/cmd/manager/instance/run/cmd.go‎

Lines changed: 12 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -68,10 +68,6 @@ import (
6868
var (
6969
scheme = runtime.NewScheme()
7070

71-
// errNoFreeWALSpace is returned when there isn't enough disk space
72-
// available to store at least two WAL files.
73-
errNoFreeWALSpace = fmt.Errorf("no free disk space for WALs")
74-
7571
// errWALArchivePluginNotAvailable is returned when the configured
7672
// WAL archiving plugin is not available or cannot be found.
7773
errWALArchivePluginNotAvailable = fmt.Errorf("WAL archive plugin not available")
@@ -127,7 +123,7 @@ func NewCmd() *cobra.Command {
127123
return runSubCommand(ctx, instance, pprofHTTPServer, skipNameValidation)
128124
})
129125

130-
if errors.Is(err, errNoFreeWALSpace) {
126+
if errors.Is(err, postgres.ErrNoFreeWALSpace) {
131127
os.Exit(apiv1.MissingWALDiskSpaceExitCode)
132128
}
133129
if errors.Is(err, errWALArchivePluginNotAvailable) {
@@ -165,7 +161,7 @@ func NewCmd() *cobra.Command {
165161
return cmd
166162
}
167163

168-
func runSubCommand(
164+
func runSubCommand( //nolint: gocyclo,gocognit
169165
ctx context.Context,
170166
instance *postgres.Instance,
171167
pprofServer bool,
@@ -179,15 +175,6 @@ func runSubCommand(
179175
"build", versions.Info,
180176
"skipNameValidation", skipNameValidation)
181177

182-
contextLogger.Info("Checking for free disk space for WALs before starting PostgreSQL")
183-
hasDiskSpaceForWals, err := instance.CheckHasDiskSpaceForWAL(ctx)
184-
if err != nil {
185-
contextLogger.Error(err, "Error while checking if there is enough disk space for WALs, skipping")
186-
} else if !hasDiskSpaceForWals {
187-
contextLogger.Info("Detected low-disk space condition, avoid starting the instance")
188-
return errNoFreeWALSpace
189-
}
190-
191178
mgr, err := ctrl.NewManager(config.GetConfigOrDie(), ctrl.Options{
192179
Scheme: scheme,
193180
Cache: cache.Options{
@@ -408,16 +395,24 @@ func runSubCommand(
408395
contextLogger.Info("starting controller-runtime manager")
409396
if err := mgr.Start(onlineUpgradeCtx); err != nil {
410397
contextLogger.Error(err, "unable to run controller-runtime manager")
398+
if errors.Is(err, postgres.ErrNoFreeWALSpace) {
399+
return makeUnretryableError(postgres.ErrNoFreeWALSpace)
400+
}
401+
if hasSpace, checkErr := instance.CheckHasDiskSpaceForWAL(ctx); checkErr == nil && !hasSpace {
402+
contextLogger.Warning("Detected low WAL disk space, but the manager error is not WAL-space related",
403+
"originalError", err)
404+
return makeUnretryableError(fmt.Errorf("%w: %w", postgres.ErrNoFreeWALSpace, err))
405+
}
411406
return makeUnretryableError(err)
412407
}
413408

414409
contextLogger.Info("Checking for free disk space for WALs after PostgreSQL finished")
415-
hasDiskSpaceForWals, err = instance.CheckHasDiskSpaceForWAL(ctx)
410+
hasDiskSpaceForWals, err := instance.CheckHasDiskSpaceForWAL(ctx)
416411
if err != nil {
417412
contextLogger.Error(err, "Error while checking if there is enough disk space for WALs, skipping")
418413
} else if !hasDiskSpaceForWals {
419414
contextLogger.Info("Detected low-disk space condition")
420-
return makeUnretryableError(errNoFreeWALSpace)
415+
return makeUnretryableError(postgres.ErrNoFreeWALSpace)
421416
}
422417

423418
enabledArchiverPluginName := instance.GetClusterOrDefault().GetEnabledWALArchivePluginName()

‎internal/cmd/manager/instance/run/lifecycle/run.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,13 @@ func (i *PostgresLifecycle) runPostgresAndWait(ctx context.Context) <-chan error
9090
return nil
9191
}
9292

93+
if hasDiskSpace, err := i.instance.CheckHasDiskSpaceForWAL(postgresContext); err != nil {
94+
contextLogger.Error(err, "Error checking WAL disk space, skipping")
95+
} else if !hasDiskSpace {
96+
contextLogger.Info("Not enough WAL disk space, avoid starting PostgreSQL")
97+
return postgres.ErrNoFreeWALSpace
98+
}
99+
93100
i.instance.LogPgControldata(postgresContext, "postmaster start up")
94101
defer i.instance.LogPgControldata(postgresContext, "postmaster has exited")
95102

‎pkg/management/postgres/instance.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,10 @@ var (
134134

135135
// ErrNoConnectionEstablished postgres is alive, but rejecting connections
136136
ErrNoConnectionEstablished = fmt.Errorf("could not establish connection")
137+
138+
// ErrNoFreeWALSpace is returned when there isn't enough disk space
139+
// available to store at least two WAL files.
140+
ErrNoFreeWALSpace = errors.New("no free disk space for WALs")
137141
)
138142

139143
// Instance represent a PostgreSQL instance to be executed

0 commit comments

Comments
 (0)