Skip to content

Commit 90fbb16

Browse files
committed
fix(linstor): retry drbdadm secondary after mkfs on transient EBUSY
LINSTOR satellite aborts resource initialization with "Device is held open by someone" (errno -12) when an external probe (Talos block-controller, udev, multipathd) opens the freshly-promoted DRBD device just before the post-mkfs demotion. The controller view then stays in SyncTarget / Inconsistent / DELETING indefinitely, blocking cleanup of orphan PVs in Released phase. Add a retry wrapper secondaryAfterMkfs() with exponential backoff (7 attempts, 250ms -> 2s) that triggers only on this specific error message. DrbdPrimary.close() now uses the wrapper so the retry only applies to the post-mkfs demotion path; the existing public secondary() contract is untouched. Related: LINBIT/linstor-server#268, LINBIT/drbd#74 Signed-off-by: IvanHunters <[email protected]>
1 parent f45facc commit 90fbb16

2 files changed

Lines changed: 107 additions & 0 deletions

File tree

‎packages/system/linstor/images/piraeus-server/patches/README.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,3 +15,6 @@ Custom patches for piraeus-server (linstor-server) v1.33.2.
1515
- **retry-adjust-after-stale-bitmap.diff** — Retry `drbdadm adjust` after detaching a stale local bitmap state
1616
- Source PR: [#491](https://github.com/LINBIT/linstor-server/pull/491)
1717
- Backported from commit: [`51ae50a84`](https://github.com/kvaps/linstor-server/commit/51ae50a84dcb98093f543b819652c750a94d96c9)
18+
- **retry-secondary-after-mkfs.diff** — Retry `drbdadm secondary` after mkfs when DRBD reports the device as held open by an external probe (Talos block-controller, udev, multipathd, etc.). Without this retry, a transient `Device is held open by someone` aborts resource initialization, leaves the satellite in an intermediate state, and prevents the controller from receiving the final `UpToDate` event — orphan PVs in `Released` then cannot be cleaned up.
19+
- Related upstream issue: [#268](https://github.com/LINBIT/linstor-server/issues/268)
20+
- Related upstream issue: [drbd #74](https://github.com/LINBIT/drbd/issues/74) (same EBUSY pattern with multipathd)
Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
diff --git a/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/DrbdAdm.java b/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/DrbdAdm.java
2+
index 71c1df6..f91903b 100644
3+
--- a/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/DrbdAdm.java
4+
+++ b/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/DrbdAdm.java
5+
@@ -62,6 +62,10 @@ public class DrbdAdm
6+
private static final long FORCE_DETACH_RETRY_WAIT_MS = 250;
7+
private static final String BITMAP_LEAK_ERR_MSG = "already has a bitmap, this should not happen";
8+
private static final Pattern BITMAP_LEAK_MINOR_PATTERN = Pattern.compile("\\bminor\\s+(\\d+)\\b");
9+
+ private static final int SECONDARY_RETRY_MAX_ATTEMPTS = 7;
10+
+ private static final long SECONDARY_RETRY_INITIAL_BACKOFF_MS = 250L;
11+
+ private static final long SECONDARY_RETRY_MAX_BACKOFF_MS = 2000L;
12+
+ private static final String DEVICE_HELD_OPEN_ERR_MSG = "Device is held open by someone";
13+
14+
private final ExtCmdFactory extCmdFactory;
15+
private final AccessContext sysCtx;
16+
@@ -945,6 +949,79 @@ public class DrbdAdm
17+
return true;
18+
}
19+
20+
+ /**
21+
+ * Switches a resource to secondary mode after mkfs, retrying when DRBD reports the device as held
22+
+ * open by another process. On Talos and other systems with active block-device probing
23+
+ * (machined/udev), the kernel uevent for a freshly-promoted device may cause an external probe
24+
+ * to open the device just before LINSTOR demotes it back to Secondary, producing
25+
+ * "Device is held open by someone" (errno -12) and aborting resource initialization.
26+
+ * Probes typically release the fd within milliseconds, so a short retry with exponential backoff
27+
+ * is sufficient to win the race without changing the contract of {@link #secondary}.
28+
+ */
29+
+ public void secondaryAfterMkfs(DrbdRscData<Resource> drbdRscData) throws ExtCmdFailedException
30+
+ {
31+
+ String[] commandArr = new String[] {
32+
+ DRBDADM_UTIL, "-vvv", "secondary", drbdRscData.getSuffixedResourceName()
33+
+ };
34+
+
35+
+ File nullDevice = new File(Platform.nullDevice());
36+
+
37+
+ OutputData lastOutput = null;
38+
+ for (int attempt = 1; attempt <= SECONDARY_RETRY_MAX_ATTEMPTS; attempt++)
39+
+ {
40+
+ try
41+
+ {
42+
+ ExtCmd extCmd = extCmdFactory.create();
43+
+ if (Platform.isWindows())
44+
+ {
45+
+ extCmd.setTimeout(TimeoutType.WAIT, 5 * 60 * 1000);
46+
+ }
47+
+
48+
+ lastOutput = extCmd.pipeExec(ProcessBuilder.Redirect.from(nullDevice), commandArr);
49+
+ if (lastOutput.exitCode == 0)
50+
+ {
51+
+ return;
52+
+ }
53+
+ if (!isDeviceHeldOpenError(lastOutput) || attempt == SECONDARY_RETRY_MAX_ATTEMPTS)
54+
+ {
55+
+ throw new ExtCmdFailedException(commandArr, lastOutput);
56+
+ }
57+
+ }
58+
+ catch (ChildProcessTimeoutException timeoutExc)
59+
+ {
60+
+ throw new ExtCmdFailedException(commandArr, timeoutExc);
61+
+ }
62+
+ catch (IOException ioExc)
63+
+ {
64+
+ throw new ExtCmdFailedException(commandArr, ioExc);
65+
+ }
66+
+
67+
+ long sleepMs = Math.min(
68+
+ SECONDARY_RETRY_INITIAL_BACKOFF_MS * (1L << (attempt - 1)),
69+
+ SECONDARY_RETRY_MAX_BACKOFF_MS
70+
+ );
71+
+ try
72+
+ {
73+
+ Thread.sleep(sleepMs);
74+
+ }
75+
+ catch (InterruptedException ie)
76+
+ {
77+
+ Thread.currentThread().interrupt();
78+
+ throw new ExtCmdFailedException(commandArr, lastOutput);
79+
+ }
80+
+ }
81+
+ }
82+
+
83+
+ static boolean isDeviceHeldOpenError(OutputData outputData)
84+
+ {
85+
+ if (outputData == null || outputData.stderrData == null)
86+
+ {
87+
+ return false;
88+
+ }
89+
+ String stderr = new String(outputData.stderrData, StandardCharsets.UTF_8);
90+
+ return stderr.contains(DEVICE_HELD_OPEN_ERR_MSG);
91+
+ }
92+
+
93+
public static class DrbdPrimary implements AutoCloseable
94+
{
95+
private final DrbdAdm drbdAdm;
96+
@@ -963,7 +1040,7 @@ public class DrbdAdm
97+
@Override
98+
public void close() throws ExtCmdFailedException
99+
{
100+
- drbdAdm.secondary(drbdRscData);
101+
+ drbdAdm.secondaryAfterMkfs(drbdRscData);
102+
}
103+
}
104+
}

0 commit comments

Comments
 (0)