Skip to content

Commit 527570d

Browse files
committed
fix(linstor): retain shutdown command failure diagnostics
Signed-off-by: Yan Bondarenko <[email protected]>
1 parent b8c86f2 commit 527570d

2 files changed

Lines changed: 70 additions & 7 deletions

File tree

‎packages/system/linstor/hack/satellite-pre-stop.py‎

Lines changed: 24 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,13 @@
66
import time
77

88

9+
def stderr_tail(value):
10+
"""Keep the last 2048 characters, including diagnostics from timeouts."""
11+
if isinstance(value, bytes):
12+
value = value.decode("utf-8", errors="replace")
13+
return (value or "")[-2048:]
14+
15+
916
def connect_container_stdout():
1017
# Piraeus shares the pod PID namespace, so /proc/1 belongs to the sandbox.
1118
with open("/proc/self/cgroup") as stream:
@@ -75,9 +82,9 @@ def read_status(name=None, timeout=4):
7582
if name is not None:
7683
args.append(name)
7784
args.append("--json")
78-
result = subprocess.run(args, capture_output=True, text=True, timeout=timeout)
85+
result = subprocess.run(args, capture_output=True, text=True, errors="replace", timeout=timeout)
7986
if result.returncode != 0:
80-
raise RuntimeError("status-command-failed")
87+
raise RuntimeError("status-command-failed (exit=%s): %s" % (result.returncode, stderr_tail(result.stderr)))
8188
return parse_status(result.stdout)
8289

8390

@@ -110,19 +117,29 @@ def main():
110117
result = subprocess.run(
111118
["/usr/sbin/drbdsetup", "down", name],
112119
stdout=subprocess.DEVNULL,
113-
stderr=subprocess.DEVNULL,
120+
stderr=subprocess.PIPE,
121+
text=True,
122+
errors="replace",
114123
timeout=min(10, remaining),
115124
)
116125
if result.returncode != 0:
117-
raise RuntimeError("down-command-failed")
126+
raise RuntimeError("down-command-failed (exit=%s): %s" % (result.returncode, stderr_tail(result.stderr)))
118127
log("down-finished", resource=name)
119128
log("finished", execute=execute, resources=len(initial))
120129

121130

122-
if __name__ == "__main__":
131+
def run_hook():
123132
try:
124133
connect_container_stdout()
125134
main()
126135
except Exception as error:
127-
log("abort-no-further-actions", error_type=type(error).__name__)
128-
sys.exit(1)
136+
details = {"error_type": type(error).__name__, "error": str(error)}
137+
if isinstance(error, subprocess.TimeoutExpired):
138+
details["stderr"] = stderr_tail(error.stderr)
139+
log("abort-no-further-actions", **details)
140+
return 1
141+
return 0
142+
143+
144+
if __name__ == "__main__":
145+
sys.exit(run_hook())

‎packages/system/linstor/tests/test_satellite_pre_stop.py‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,52 @@ def test_down_failure_or_timeout_stops_before_next_resource(self):
122122
hook.main()
123123
self.assertEqual(self.run.call_count, 3)
124124

125+
def test_abort_logs_bounded_status_diagnostics_without_down(self):
126+
stderr = "discarded-prefix" + "x" * 2048 + "permission denied"
127+
self.run.return_value = subprocess.CompletedProcess([], 12, "", stderr)
128+
with patch.object(hook, "connect_container_stdout"):
129+
self.assertEqual(hook.run_hook(), 1)
130+
self.assertEqual(self.run.call_count, 1)
131+
self.assertEqual(self.run.call_args.kwargs["errors"], "replace")
132+
self.events.assert_called_once_with(
133+
"abort-no-further-actions", error_type="RuntimeError",
134+
error="status-command-failed (exit=12): " + stderr[-2048:],
135+
)
136+
137+
def test_abort_logs_down_diagnostics_and_stops_before_next_resource(self):
138+
stderr = "discarded-prefix" + "x" * 2048 + "device busy"
139+
self.run.side_effect = [self.status(resource(), resource("r2")), self.status(resource()),
140+
subprocess.CompletedProcess([], 17, stderr=stderr)]
141+
with patch.object(hook, "connect_container_stdout"):
142+
self.assertEqual(hook.run_hook(), 1)
143+
self.assertEqual(self.run.call_count, 3)
144+
self.assertEqual(self.run.call_args.kwargs["stderr"], subprocess.PIPE)
145+
self.assertTrue(self.run.call_args.kwargs["text"])
146+
self.assertEqual(self.run.call_args.kwargs["errors"], "replace")
147+
self.events.assert_called_with(
148+
"abort-no-further-actions", error_type="RuntimeError",
149+
error="down-command-failed (exit=17): " + stderr[-2048:],
150+
)
151+
152+
def test_abort_logs_partial_timeout_stderr_for_status_and_down(self):
153+
stderr = b"discarded-prefix" + b"x" * 2048 + b"device busy\xff"
154+
for command in ["status", "down"]:
155+
with self.subTest(command=command):
156+
self.run.reset_mock()
157+
self.events.reset_mock()
158+
failure = subprocess.TimeoutExpired(command, 4, stderr=stderr)
159+
responses = [failure] if command == "status" else [
160+
self.status(resource(), resource("r2")), self.status(resource()), failure,
161+
]
162+
self.run.side_effect = responses
163+
with patch.object(hook, "connect_container_stdout"):
164+
self.assertEqual(hook.run_hook(), 1)
165+
self.assertEqual(self.run.call_count, len(responses))
166+
self.events.assert_called_with(
167+
"abort-no-further-actions", error_type="TimeoutExpired",
168+
error=str(failure), stderr=stderr.decode("utf-8", errors="replace")[-2048:],
169+
)
170+
125171
def test_deadline_prevents_further_down_and_bounds_command_timeout(self):
126172
self.clock.side_effect = [0, 60]
127173
self.run.side_effect = [self.status(resource()), self.status(resource())]

0 commit comments

Comments
 (0)