Skip to content

Commit 66aec06

Browse files
kvapskvaps
committed
[linstor] Build linstor-server with custom patches (#1726)
## What this PR does Build piraeus-server (linstor-server) from source with custom patches: - **adjust-on-resfile-change.diff** — Use actual device path in res file during toggle-disk; fix LUKS data offset - Upstream: [#473](LINBIT/linstor-server#473), [#472](LINBIT/linstor-server#472) - **allow-toggle-disk-retry.diff** — Allow retry and cancellation of failed toggle-disk operations - Upstream: [#475](LINBIT/linstor-server#475) - **force-metadata-check-on-disk-add.diff** — Create metadata during toggle-disk from diskless to diskful - Upstream: [#474](LINBIT/linstor-server#474) - **skip-adjust-when-device-inaccessible.diff** — Skip DRBD adjust/res file regeneration when child layer device is inaccessible - Upstream: [#471](LINBIT/linstor-server#471) Also updates plunger-satellite script and values.yaml for the new build. ### Release note ```release-note [linstor] Build linstor-server with custom patches for improved disk handling ``` <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **New Features** * Added automatic DRBD stall detection and recovery, improving storage resync resilience without manual intervention. * Introduced configurable container image references via Helm values for streamlined deployment. * **Bug Fixes** * Enhanced disk toggle operations with retry and cancellation support for better error handling. * Improved metadata creation during disk state transitions. * Added device accessibility checks to prevent errors when underlying storage devices are unavailable. * Fixed LUKS encryption header sizing for consistent deployment across nodes. <sub>✏️ Tip: You can customize this high-level summary in your review settings.</sub> <!-- end of auto-generated comment: release notes by coderabbit.ai -->
1 parent 28b60e4 commit 66aec06

14 files changed

Lines changed: 773 additions & 30 deletions

‎Makefile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ build: build-deps
1717
make -C packages/system/cozystack-controller image
1818
make -C packages/system/lineage-controller-webhook image
1919
make -C packages/system/cilium image
20+
make -C packages/system/linstor image
2021
make -C packages/system/kubeovn-webhook image
2122
make -C packages/system/kubeovn-plunger image
2223
make -C packages/system/dashboard image

‎packages/system/linstor/Makefile‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,23 @@
11
export NAME=linstor
22
export NAMESPACE=cozy-$(NAME)
33

4+
include ../../../scripts/common-envs.mk
45
include ../../../scripts/package.mk
6+
7+
LINSTOR_VERSION ?= 1.32.3
8+
9+
image:
10+
docker buildx build images/piraeus-server \
11+
--build-arg LINSTOR_VERSION=$(LINSTOR_VERSION) \
12+
--build-arg K8S_AWAIT_ELECTION_VERSION=v0.4.2 \
13+
--tag $(REGISTRY)/piraeus-server:$(call settag,$(LINSTOR_VERSION)) \
14+
--tag $(REGISTRY)/piraeus-server:$(call settag,$(LINSTOR_VERSION)-$(TAG)) \
15+
--cache-from type=registry,ref=$(REGISTRY)/piraeus-server:latest \
16+
--cache-to type=inline \
17+
--metadata-file images/piraeus-server.json \
18+
$(BUILDX_ARGS)
19+
REPOSITORY="$(REGISTRY)/piraeus-server" \
20+
yq -i '.piraeusServer.image.repository = strenv(REPOSITORY)' values.yaml
21+
TAG="$(call settag,$(LINSTOR_VERSION))@$$(yq e '."containerimage.digest"' images/piraeus-server.json -o json -r)" \
22+
yq -i '.piraeusServer.image.tag = strenv(TAG)' values.yaml
23+
rm -f images/piraeus-server.json

‎packages/system/linstor/hack/plunger/plunger-satellite.sh‎

Lines changed: 120 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,18 +10,133 @@ trap terminate SIGINT SIGQUIT SIGTERM
1010

1111
echo "Starting Linstor per-satellite plunger"
1212

13+
INTERVAL_SEC="${INTERVAL_SEC:-30}"
14+
STALL_ITERS="${STALL_ITERS:-4}"
15+
STATE_FILE="${STATE_FILE:-/run/drbd-sync-watch.state}"
16+
17+
log() { printf '%s %s\n' "$(date -Is)" "$*" >&2; }
18+
19+
drbd_status_json() {
20+
drbdsetup status --json 2>/dev/null || true
21+
}
22+
23+
# Detect DRBD resources where resync is stuck:
24+
# - at least one local device is Inconsistent
25+
# - there is an active SyncTarget peer
26+
# - there are other peers suspended with resync-suspended:dependency
27+
# Output format: "<resource> <sync-peer> <percent-in-sync>"
28+
drbd_stall_candidates() {
29+
jq -r '
30+
.[]?
31+
| . as $r
32+
| select(any($r.devices[]?; ."disk-state" == "Inconsistent"))
33+
| (
34+
[ $r.connections[]?
35+
| . as $c
36+
| $c.peer_devices[]?
37+
| select(."replication-state" == "SyncTarget")
38+
| { peer: $c.name, pct: (."percent-in-sync" // empty) }
39+
] | .[0]?
40+
) as $sync
41+
| select($sync != null and ($sync.pct|tostring) != "")
42+
| select(any($r.connections[]?.peer_devices[]?; ."resync-suspended" == "dependency"))
43+
| "\($r.name) \($sync.peer) \($sync.pct)"
44+
'
45+
}
46+
47+
drbd_stall_load_state() {
48+
[ -f "$STATE_FILE" ] && cat "$STATE_FILE" || true
49+
}
50+
51+
drbd_stall_save_state() {
52+
local tmp="${STATE_FILE}.tmp"
53+
cat >"$tmp"
54+
mv "$tmp" "$STATE_FILE"
55+
}
56+
57+
# Break stalled resync by disconnecting the current SyncTarget peer.
58+
# After reconnect, DRBD will typically pick another eligible peer and continue syncing.
59+
drbd_stall_act() {
60+
local res="$1"
61+
local peer="$2"
62+
local pct="$3"
63+
log "STALL detected: res=$res sync_peer=$peer percent_in_sync=$pct -> disconnect/connect"
64+
drbdadm disconnect "${res}:${peer}" && drbdadm connect "$res" || log "WARN: action failed for ${res}:${peer}"
65+
}
66+
67+
# Track percent-in-sync progress across iterations.
68+
# If progress does not change for STALL_ITERS loops, trigger reconnect.
69+
drbd_fix_stalled_sync() {
70+
local now prev json out
71+
now="$(date +%s)"
72+
prev="$(drbd_stall_load_state)"
73+
74+
json="$(drbd_status_json)"
75+
[ -n "$json" ] || return 0
76+
77+
out="$(printf '%s' "$json" | drbd_stall_candidates)"
78+
79+
local new_state=""
80+
local acts=""
81+
82+
while IFS= read -r line; do
83+
[ -n "$line" ] || continue
84+
set -- $line
85+
local res="$1" peer="$2" pct="$3"
86+
local key="${res} ${peer}"
87+
88+
local prev_line
89+
prev_line="$(printf '%s\n' "$prev" | awk -v k="$key" '$1" "$2==k {print; exit}')"
90+
91+
local cnt last_act prev_pct prev_cnt prev_act
92+
if [ -n "$prev_line" ]; then
93+
set -- $prev_line
94+
prev_pct="$3"
95+
prev_cnt="$4"
96+
prev_act="$5"
97+
if [ "$pct" = "$prev_pct" ]; then
98+
cnt=$((prev_cnt + 1))
99+
else
100+
cnt=1
101+
fi
102+
last_act="$prev_act"
103+
else
104+
cnt=1
105+
last_act=0
106+
fi
107+
108+
if [ "$cnt" -ge "$STALL_ITERS" ]; then
109+
acts="${acts}${res} ${peer} ${pct}"$'\n'
110+
cnt=0
111+
last_act="$now"
112+
fi
113+
114+
new_state="${new_state}${res} ${peer} ${pct} ${cnt} ${last_act}"$'\n'
115+
done <<< "$out"
116+
117+
if [ -n "$acts" ]; then
118+
while IFS= read -r a; do
119+
[ -n "$a" ] || continue
120+
set -- $a
121+
drbd_stall_act "$1" "$2" "$3"
122+
done <<< "$acts"
123+
fi
124+
125+
printf '%s' "$new_state" | drbd_stall_save_state
126+
}
127+
13128
while true; do
14129

15130
# timeout at the start of the loop to give a chance for the fresh linstor-satellite instance to cleanup itself
16-
sleep 30 &
131+
sleep "$INTERVAL_SEC" &
17132
pid=$!
18133
wait $pid
19134

20135
# Detect orphaned loop devices and detach them
21136
# the `/` path could not be a backing file for a loop device, so it's a good indicator of a stuck loop device
22137
# TODO describe the issue in more detail
23138
# Using the direct /usr/sbin/losetup as the linstor-satellite image has own wrapper in /usr/local
24-
stale_loopbacks=$(/usr/sbin/losetup --json | jq -r '.[][] | select(."back-file" == "/" or ."back-file" == "/ (deleted)").name' )
139+
stale_loopbacks=$(/usr/sbin/losetup --json | jq -r '.[][] | select(."back-file" == "/" or ."back-file" == "/ (deleted)").name')
25140
for stale_device in $stale_loopbacks; do (
26141
echo "Detaching stuck loop device ${stale_device}"
27142
set -x
@@ -39,4 +154,7 @@ while true; do
39154
drbdadm up "${secondary}" || echo "Command failed"
40155
); done
41156

157+
# Detect and fix stalled DRBD resync by switching SyncTarget peer
158+
drbd_fix_stalled_sync || true
159+
42160
done

‎packages/system/linstor/images/piraeus-server/Dockerfile‎

Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
ARG DISTRO=bookworm
2+
ARG LINSTOR_VERSION
3+
4+
# ------------------------------------------------------------------------------
5+
# Build linstor-server from source
6+
FROM debian:bookworm AS builder
7+
8+
ARG LINSTOR_VERSION
9+
ARG VERSION=${LINSTOR_VERSION}
10+
ARG DISTRO
11+
12+
ENV DEBIAN_FRONTEND=noninteractive
13+
RUN apt-get update \
14+
&& apt-get -y upgrade \
15+
&& apt-get -y install build-essential git default-jdk-headless python3-all debhelper wget unzip && \
16+
wget https://services.gradle.org/distributions/gradle-8.9-bin.zip -O /tmp/gradle.zip && \
17+
unzip -d /opt /tmp/gradle.zip && \
18+
rm /tmp/gradle.zip && \
19+
ln -s /opt/gradle-8.9/bin/gradle /usr/local/bin/gradle
20+
21+
RUN git clone https://github.com/LINBIT/linstor-server.git /linstor-server
22+
WORKDIR /linstor-server
23+
RUN git checkout v${VERSION}
24+
25+
# Apply patches
26+
COPY patches /patches
27+
RUN git apply /patches/*.diff && \
28+
git config user.email "[email protected]" && \
29+
git config user.name "Cozystack Builder" && \
30+
git add -A && \
31+
git commit -m "Apply patches"
32+
33+
# Initialize git submodules
34+
RUN git submodule update --init --recursive || make check-submods
35+
36+
# Pre-download ALL dependencies before make tarball
37+
# This ensures all transitive dependencies are cached, including optional ones like AWS SDK
38+
RUN ./gradlew getProtoc
39+
RUN ./gradlew generateJava
40+
RUN ./gradlew --no-daemon --gradle-user-home .gradlehome downloadDependencies
41+
42+
# Manually create tarball without removing caches
43+
# make tarball removes .gradlehome/caches/[0-9]* which deletes dependencies
44+
# So we'll do the steps manually but keep the caches
45+
RUN make check-submods versioninfo gen-java FORCE=1 VERSION=${VERSION}
46+
RUN make server/jar.deps controller/jar.deps satellite/jar.deps jclcrypto/jar.deps FORCE=1 VERSION=${VERSION}
47+
RUN make .filelist FORCE=1 VERSION=${VERSION} PRESERVE_DEBIAN=1
48+
# Don't remove caches - we need them for offline build
49+
RUN rm -Rf .gradlehome/wrapper .gradlehome/native .gradlehome/.tmp || true
50+
RUN mkdir -p ./libs
51+
RUN make tgz VERSION=${VERSION}
52+
53+
# Extract tarball and build DEB packages from it
54+
RUN mv linstor-server-${VERSION}.tar.gz /linstor-server_${VERSION}.orig.tar.gz \
55+
&& tar -C / -xvf /linstor-server_${VERSION}.orig.tar.gz
56+
57+
WORKDIR /linstor-server-${VERSION}
58+
# Verify .gradlehome is present in extracted tarball
59+
RUN test -d .gradlehome && echo ".gradlehome found in tarball" || (echo ".gradlehome not found in tarball!" && exit 1)
60+
61+
# Build DEB packages from tarball
62+
# Override GRADLE_FLAGS to remove --offline flag, allowing Gradle to download missing dependencies
63+
RUN sed -i 's/GRADLE_FLAGS = --offline/GRADLE_FLAGS =/' debian/rules || true
64+
RUN LD_LIBRARY_PATH='' dpkg-buildpackage -rfakeroot -b -uc
65+
66+
# Copy built .deb packages to a location accessible from final image
67+
# dpkg-buildpackage creates packages in parent directory
68+
RUN mkdir -p /packages-output && \
69+
find .. -maxdepth 1 -name "linstor-*.deb" -exec cp {} /packages-output/ \; && \
70+
test -n "$(ls -A /packages-output)" || (echo "ERROR: No linstor .deb packages found after build." && exit 1)
71+
72+
# ------------------------------------------------------------------------------
73+
# Final image
74+
FROM debian:${DISTRO}
75+
76+
LABEL maintainer="Roland Kammerer <[email protected]>"
77+
78+
ARG LINSTOR_VERSION
79+
ARG DISTRO
80+
81+
# Copy built .deb packages from builder stage
82+
# dpkg-buildpackage creates packages in parent directory, we copied them to /packages-output
83+
COPY --from=builder /packages-output/ /packages/
84+
85+
RUN { echo 'APT::Install-Recommends "false";' ; echo 'APT::Install-Suggests "false";' ; } > /etc/apt/apt.conf.d/99_piraeus
86+
87+
RUN --mount=type=cache,target=/var/cache,sharing=private \
88+
--mount=type=cache,target=/var/lib/apt/lists,sharing=private \
89+
--mount=type=tmpfs,target=/var/log \
90+
# Install wget first for downloading keyring
91+
apt-get update && apt-get install -y wget ca-certificates && \
92+
# Enable contrib repos for zfsutils \
93+
. /etc/os-release && \
94+
sed -i -r 's/^Components: (.*)$/Components: \1 contrib/' /etc/apt/sources.list.d/debian.sources && \
95+
echo "deb http://deb.debian.org/debian $VERSION_CODENAME-backports contrib" > /etc/apt/sources.list.d/backports.list && \
96+
wget https://packages.linbit.com/public/linbit-keyring.deb -O /var/cache/linbit-keyring.deb && \
97+
dpkg -i /var/cache/linbit-keyring.deb && \
98+
echo "deb http://packages.linbit.com/public $VERSION_CODENAME misc" > /etc/apt/sources.list.d/linbit.list && \
99+
apt-get update && \
100+
# Install useful utilities and general dependencies
101+
apt-get install -y udev drbd-utils jq net-tools iputils-ping iproute2 dnsutils netcat-traditional sysstat curl util-linux && \
102+
# Install dependencies for optional features \
103+
apt-get install -y \
104+
# cryptsetup: luks layer
105+
cryptsetup \
106+
# e2fsprogs: LINSTOR can create file systems \
107+
e2fsprogs \
108+
# lsscsi: exos layer \
109+
lsscsi \
110+
# lvm2: manage lvm storage pools \
111+
lvm2 \
112+
# multipath-tools: exos layer \
113+
multipath-tools \
114+
# nvme-cli: nvme layer
115+
nvme-cli \
116+
# procps: used by LINSTOR to find orphaned send/receive processes \
117+
procps \
118+
# socat: used with thin-send-recv to send snapshots to another LINSTOR cluster
119+
socat \
120+
# thin-send-recv: used to send/receive snapshots of LVM thin volumes \
121+
thin-send-recv \
122+
# xfsprogs: LINSTOR can create file systems; xfs deps \
123+
xfsprogs \
124+
# zstd: used with thin-send-recv to send snapshots to another LINSTOR cluster \
125+
zstd \
126+
# zfsutils-linux: for zfs storage pools \
127+
zfsutils-linux/$VERSION_CODENAME-backports \
128+
&& \
129+
# remove udev, no need for it in the container \
130+
apt-get remove -y udev && \
131+
# Install linstor packages from built .deb files and linstor-client from repository
132+
apt-get install -y default-jre-headless python3-all python3-natsort linstor-client \
133+
&& ls packages/*.deb >/dev/null && (dpkg -i packages/*.deb || apt-get install -f -y) \
134+
&& rm -rf /packages \
135+
&& sed -i 's/"-Djdk.tls.acknowledgeCloseNotify=true"//g' /usr/share/linstor-server/bin/Controller \
136+
&& apt-get clean
137+
138+
# Log directory need to be group writable. OpenShift assigns random UID and GID, without extra RBAC changes we can only influence the GID.
139+
RUN mkdir /var/log/linstor-controller && \
140+
chown 0:1000 /var/log/linstor-controller && \
141+
chmod -R 0775 /var/log/linstor-controller && \
142+
# Ensure we log to files in containers, otherwise SOS reports won't show any logs at all
143+
sed -i 's#<!-- <appender-ref ref="FILE" /> -->#<appender-ref ref="FILE" />#' /usr/share/linstor-server/lib/conf/logback.xml
144+
145+
146+
RUN lvmconfig --type current --mergedconfig --config 'activation { udev_sync = 0 udev_rules = 0 monitoring = 0 } devices { global_filter = [ "r|^/dev/drbd|" ] obtain_device_list_from_udev = 0}' > /etc/lvm/lvm.conf.new && mv /etc/lvm/lvm.conf.new /etc/lvm/lvm.conf
147+
RUN echo 'global { usage-count no; }' > /etc/drbd.d/global_common.conf
148+
149+
# controller
150+
EXPOSE 3376/tcp 3377/tcp 3370/tcp 3371/tcp
151+
152+
# satellite
153+
EXPOSE 3366/tcp 3367/tcp
154+
155+
RUN wget https://raw.githubusercontent.com/piraeusdatastore/piraeus/refs/heads/master/dockerfiles/piraeus-server/entry.sh -O /usr/bin/piraeus-entry.sh \
156+
&& chmod +x /usr/bin/piraeus-entry.sh
157+
158+
ARG K8S_AWAIT_ELECTION_VERSION=v0.4.2
159+
# TARGETARCH is a docker special variable: https://docs.docker.com/engine/reference/builder/#automatic-platform-args-in-the-global-scope
160+
ARG TARGETARCH
161+
162+
RUN wget https://github.com/LINBIT/k8s-await-election/releases/download/${K8S_AWAIT_ELECTION_VERSION}/k8s-await-election-${K8S_AWAIT_ELECTION_VERSION}-linux-${TARGETARCH}.tar.gz -O - | tar -xvz -C /usr/bin/
163+
164+
ARG LOSETUP_CONTAINER_VERSION=v1.0.1
165+
RUN wget "https://github.com/LINBIT/losetup-container/releases/download/${LOSETUP_CONTAINER_VERSION}/losetup-container-$(uname -m)-unknown-linux-gnu.tar.gz" -O - | tar -xvz -C /usr/local/sbin && \
166+
printf '#!/bin/sh\nLOSETUP_CONTAINER_ORIGINAL_LOSETUP=%s exec /usr/local/sbin/losetup-container "$@"\n' $(command -v losetup) > /usr/local/sbin/losetup && \
167+
chmod +x /usr/local/sbin/losetup
168+
169+
RUN wget "https://dl.k8s.io/$(wget -O - https://dl.k8s.io/release/stable.txt)/bin/linux/${TARGETARCH}/kubectl" -O /usr/local/bin/kubectl && chmod +x /usr/local/bin/kubectl
170+
171+
CMD ["startSatellite"]
172+
ENTRYPOINT ["/usr/bin/k8s-await-election", "/usr/bin/piraeus-entry.sh"]

‎packages/system/linstor/images/piraeus-server/patches/README.md‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
# LINSTOR Server Patches
2+
3+
Custom patches for piraeus-server (linstor-server) v1.32.3.
4+
5+
- **adjust-on-resfile-change.diff** — Use actual device path in res file during toggle-disk; fix LUKS data offset
6+
- Upstream: [#473](https://github.com/LINBIT/linstor-server/pull/473), [#472](https://github.com/LINBIT/linstor-server/pull/472)
7+
- **allow-toggle-disk-retry.diff** — Allow retry and cancellation of failed toggle-disk operations
8+
- Upstream: [#475](https://github.com/LINBIT/linstor-server/pull/475)
9+
- **force-metadata-check-on-disk-add.diff** — Create metadata during toggle-disk from diskless to diskful
10+
- Upstream: [#474](https://github.com/LINBIT/linstor-server/pull/474)
11+
- **skip-adjust-when-device-inaccessible.diff** — Skip DRBD adjust/res file regeneration when child layer device is inaccessible
12+
- Upstream: [#471](https://github.com/LINBIT/linstor-server/pull/471)

0 commit comments

Comments
 (0)