Skip to content

Commit 8bcb709

Browse files
committed
fix(fluxcd): refuse a flux-plunger drain deadline that leaves no time
A drain deadline of zero or less is already spent when the drain starts, so flux-plunger would exit holding every uninstall fence without listing one. The chart could produce one: its 40-second floor was a literal, so raising the manager's stop bound or the margin before SIGKILL, which the drain subtraction is already tied to, would leave the floor rendering a non-positive deadline. The binary now refuses such a flag at startup, before it reaches for a cluster, and the chart's floor and the timings the README and the values comment state are held to the same constants. A whole number of a million seconds or more printed in exponent form and was refused as a fraction; the range is now checked first, so it gets the range message. Assisted-by: LLM Signed-off-by: Alexey Artamonov <[email protected]>
1 parent 3a46360 commit 8bcb709

6 files changed

Lines changed: 117 additions & 3 deletions

File tree

‎cmd/flux-plunger/main.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,11 @@ func main() {
8282

8383
ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
8484

85+
if err := fluxplunger.ValidateShutdownDrainTimeout(shutdownDrainTimeout); err != nil {
86+
setupLog.Error(err, "invalid flag")
87+
os.Exit(1)
88+
}
89+
8590
// if the enable-http2 flag is false (the default), http/2 should be disabled
8691
// due to its vulnerabilities. More specifically, disabling http/2 will
8792
// prevent from being vulnerable to the HTTP/2 Stream Cancellation and

‎internal/controller/fluxplunger/drain.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,16 @@ const (
2727
ShutdownDrainTimeoutFlag = "shutdown-drain-timeout"
2828
)
2929

30+
// ValidateShutdownDrainTimeout refuses a drain deadline that is already spent
31+
// when the drain starts: the drain would leave every fence behind without
32+
// listing one.
33+
func ValidateShutdownDrainTimeout(d time.Duration) error {
34+
if d <= 0 {
35+
return fmt.Errorf("--%s must be positive, got %v", ShutdownDrainTimeoutFlag, d)
36+
}
37+
return nil
38+
}
39+
3040
// DrainFences gives up every fence flux-plunger holds, for a flux-plunger that is
3141
// stopping. A live release has its recovery aborted, the suspension and the fence
3242
// lifted together, which leaves it as recovery found it for the next flux-plunger

‎internal/controller/fluxplunger/hook_contract_test.go‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package fluxplunger
22

33
import (
44
"os"
5+
"os/exec"
56
"regexp"
67
"strconv"
78
"strings"
@@ -68,6 +69,80 @@ func TestDeploymentTemplate_SubtractsTheStopAndTheMarginTheBinaryUses(t *testing
6869
}
6970
}
7071

72+
// The binary must refuse a drain deadline it could not drain under, whatever
73+
// the chart or a hand-written Deployment passes, and before it reaches for a
74+
// cluster: the run has no kubeconfig, so a binary that skipped the check fails
75+
// on that instead, with another message.
76+
func TestMain_RefusesADrainTimeoutThatLeavesNoTime(t *testing.T) {
77+
bin := t.TempDir() + "/flux-plunger"
78+
if out, err := exec.Command("go", "build", "-o", bin, "../../../cmd/flux-plunger").CombinedOutput(); err != nil {
79+
t.Fatalf("build flux-plunger: %v\n%s", err, out)
80+
}
81+
cmd := exec.Command(bin, "--"+ShutdownDrainTimeoutFlag+"=0s")
82+
cmd.Env = append(os.Environ(), "KUBECONFIG=/dev/null", "KUBERNETES_SERVICE_HOST=")
83+
out, err := cmd.CombinedOutput()
84+
if err == nil {
85+
t.Fatalf("flux-plunger started with a drain deadline of 0s:\n%s", out)
86+
}
87+
if want := "--" + ShutdownDrainTimeoutFlag + " must be positive"; !strings.Contains(string(out), want) {
88+
t.Fatalf("flux-plunger did not refuse the deadline itself, want %q in:\n%s", want, out)
89+
}
90+
}
91+
92+
// The chart's lowest accepted grace period has to leave the drain time after the
93+
// manager's stop and the margin before SIGKILL, or raising either constant makes
94+
// the chart render a drain deadline of zero or less at its own minimum.
95+
func TestDeploymentTemplate_TheLowestGracePeriodLeavesTheDrainTime(t *testing.T) {
96+
raw, err := os.ReadFile(deploymentTemplate)
97+
if err != nil {
98+
t.Fatalf("read %s: %v", deploymentTemplate, err)
99+
}
100+
m := regexp.MustCompile(`\(lt \$grace (\d+)\)`).FindStringSubmatch(string(raw))
101+
if m == nil {
102+
t.Fatal("deployment has no lower bound on shutdownGracePeriodSeconds")
103+
}
104+
floor, err := strconv.Atoi(m[1])
105+
if err != nil {
106+
t.Fatalf("lower bound %q: %v", m[1], err)
107+
}
108+
if drain := time.Duration(floor)*time.Second - managerStopTimeout - killMargin; drain <= 0 {
109+
t.Fatalf("at the lowest grace period %ds the drain gets %v (manager stop %v, margin %v)", floor, drain, managerStopTimeout, killMargin)
110+
}
111+
if want := strconv.Itoa(int(managerStopTimeout/time.Second)) + "s stop"; !strings.Contains(string(raw), want) {
112+
t.Errorf("the range message does not name the manager's %q", want)
113+
}
114+
}
115+
116+
// The README and the values comment restate the drain's timings for operators;
117+
// they must name the numbers the binary and the chart use, or a change to either
118+
// leaves the prose stating the old ones.
119+
func TestDocs_StateTheTimingsTheBinaryAndTheChartUse(t *testing.T) {
120+
raw, err := os.ReadFile(deploymentTemplate)
121+
if err != nil {
122+
t.Fatalf("read %s: %v", deploymentTemplate, err)
123+
}
124+
m := regexp.MustCompile(`\(lt \$grace (\d+)\) \(gt \$grace (\d+)\)`).FindStringSubmatch(string(raw))
125+
if m == nil {
126+
t.Fatal("deployment has no range on shutdownGracePeriodSeconds")
127+
}
128+
want := []string{
129+
"between " + m[1] + " and " + m[2],
130+
strconv.Itoa(int(managerStopTimeout/time.Second)) + " seconds",
131+
strconv.Itoa(int((managerStopTimeout+killMargin)/time.Second)) + " seconds",
132+
}
133+
for _, doc := range []string{"../../../packages/system/flux-plunger/README.md", "../../../packages/system/flux-plunger/values.yaml"} {
134+
text, err := os.ReadFile(doc)
135+
if err != nil {
136+
t.Fatalf("read %s: %v", doc, err)
137+
}
138+
for _, w := range want {
139+
if !strings.Contains(string(text), w) {
140+
t.Errorf("%s does not state %q", doc, w)
141+
}
142+
}
143+
}
144+
}
145+
71146
// The chart passes the drain deadline as a flag; a name the binary does not
72147
// define makes flux-plunger exit on start.
73148
func TestDeploymentTemplate_PassesTheDrainFlagTheBinaryDefines(t *testing.T) {

‎internal/controller/fluxplunger/shutdown_test.go‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,21 @@ func TestApplyShutdownTimings_TheLeaseOutlastsTheStopAndTheDrain(t *testing.T) {
177177
}
178178
}
179179

180+
// A drain deadline that is not positive is already spent when the drain starts,
181+
// so every fence would be left behind without one listing.
182+
func TestValidateShutdownDrainTimeout_RefusesADeadlineThatLeavesNoTime(t *testing.T) {
183+
for _, d := range []time.Duration{0, -5 * time.Second} {
184+
if err := ValidateShutdownDrainTimeout(d); err == nil {
185+
t.Errorf("drain timeout %v accepted", d)
186+
}
187+
}
188+
for _, d := range []time.Duration{time.Second, 65 * time.Second} {
189+
if err := ValidateShutdownDrainTimeout(d); err != nil {
190+
t.Errorf("drain timeout %v refused: %v", d, err)
191+
}
192+
}
193+
}
194+
180195
// controller-runtime reports a stop that overran GracefulShutdownTimeout as an
181196
// error from Start. A reconcile slow to notice its cancelled context must not
182197
// cost the drain the fence exists for; the exit stays non-zero.

‎packages/system/flux-plunger/templates/deployment.yaml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,12 @@ spec:
2222
rejected on upgrade.
2323
*/}}
2424
{{- $grace := int .Values.shutdownGracePeriodSeconds }}
25-
{{- if ne (toString .Values.shutdownGracePeriodSeconds) (toString $grace) }}
26-
{{- fail "shutdownGracePeriodSeconds must be a whole number of seconds: terminationGracePeriodSeconds takes an integer" }}
27-
{{- end }}
2825
{{- if or (lt $grace 40) (gt $grace 140) }}
2926
{{- fail "shutdownGracePeriodSeconds must be between 40 and 140: below, the drain gets no time after the manager's 10s stop; above, the pre-delete hook cannot wait the pod out inside its deadline" }}
3027
{{- end }}
28+
{{- if ne (toString .Values.shutdownGracePeriodSeconds) (toString $grace) }}
29+
{{- fail "shutdownGracePeriodSeconds must be a whole number of seconds: terminationGracePeriodSeconds takes an integer" }}
30+
{{- end }}
3131
replicas: 1
3232
strategy:
3333
type: RollingUpdate

‎packages/system/flux-plunger/tests/deployment_test.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,3 +56,12 @@ tests:
5656
asserts:
5757
- failedTemplate:
5858
errorPattern: shutdownGracePeriodSeconds must be between 40 and 140
59+
60+
# A whole number of a million or more prints in exponent form, which the
61+
# whole-number check would misread as a fraction.
62+
- it: names the range, not the fraction, for a whole number far above it
63+
set:
64+
shutdownGracePeriodSeconds: 1000000
65+
asserts:
66+
- failedTemplate:
67+
errorPattern: shutdownGracePeriodSeconds must be between 40 and 140

0 commit comments

Comments
 (0)