Skip to content

Commit 4b67305

Browse files
authored
Merge pull request NVIDIA#24 from LobsterTrap/fix/rpm-modules-and-podman-socket
fix(bootstrap): load kernel modules on install and fix Podman socket detection
2 parents 9a4c43a + 5620171 commit 4b67305

7 files changed

Lines changed: 134 additions & 58 deletions

File tree

‎.github/workflows/midstream-container-build.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,7 @@ jobs:
135135
docker buildx imagetools create \
136136
-t "${{ env.IMAGE_REGISTRY }}/gateway:${{ github.sha }}" \
137137
-t "${{ env.IMAGE_REGISTRY }}/gateway:midstream" \
138+
-t "${{ env.IMAGE_REGISTRY }}/gateway:dev" \
138139
"${{ env.IMAGE_REGISTRY }}/gateway:${{ github.sha }}-amd64" \
139140
"${{ env.IMAGE_REGISTRY }}/gateway:${{ github.sha }}-arm64"
140141
@@ -159,5 +160,6 @@ jobs:
159160
docker buildx imagetools create \
160161
-t "${{ env.IMAGE_REGISTRY }}/cluster:${{ github.sha }}" \
161162
-t "${{ env.IMAGE_REGISTRY }}/cluster:midstream" \
163+
-t "${{ env.IMAGE_REGISTRY }}/cluster:dev" \
162164
"${{ env.IMAGE_REGISTRY }}/cluster:${{ github.sha }}-amd64" \
163165
"${{ env.IMAGE_REGISTRY }}/cluster:${{ github.sha }}-arm64"

‎crates/openshell-bootstrap/src/build.rs‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@
1010
use std::collections::HashMap;
1111
use std::path::Path;
1212

13-
use bollard::Docker;
1413
use bollard::query_parameters::BuildImageOptionsBuilder;
1514
use futures::StreamExt;
1615
use miette::{IntoDiagnostic, Result, WrapErr};
@@ -46,9 +45,8 @@ pub async fn build_and_push_image(
4645
on_log(format!(
4746
"Pushing image {tag} into gateway \"{gateway_name}\""
4847
));
49-
let local_docker = Docker::connect_with_local_defaults()
50-
.into_diagnostic()
51-
.wrap_err("failed to connect to local Docker daemon")?;
48+
let local_docker = crate::docker::connect_local_auto()
49+
.wrap_err("failed to connect to local container runtime")?;
5250
let container = container_name(gateway_name);
5351
let images: Vec<&str> = vec![tag];
5452
push_local_images(&local_docker, &local_docker, &container, &images, on_log).await?;
@@ -68,9 +66,8 @@ async fn build_image(
6866
build_args: &HashMap<String, String>,
6967
on_log: &mut impl FnMut(String),
7068
) -> Result<()> {
71-
let docker = Docker::connect_with_local_defaults()
72-
.into_diagnostic()
73-
.wrap_err("failed to connect to local Docker daemon")?;
69+
let docker = crate::docker::connect_local_auto()
70+
.wrap_err("failed to connect to local container runtime")?;
7471

7572
// Compute the relative path of the Dockerfile within the context.
7673
let dockerfile_relative = dockerfile_path

‎crates/openshell-bootstrap/src/docker.rs‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -263,6 +263,33 @@ pub(crate) fn connect_local(runtime: ContainerRuntime) -> Result<Docker> {
263263
}
264264
}
265265

266+
/// Connect to the local container runtime with auto-detection.
267+
///
268+
/// This is a convenience wrapper for code paths that need a Docker client
269+
/// but don't have a `ContainerRuntime` value available. It auto-detects
270+
/// the runtime (Podman preferred) and connects via `connect_local`.
271+
pub(crate) fn connect_local_auto() -> Result<Docker> {
272+
let runtime = crate::container_runtime::detect_runtime(None)?;
273+
connect_local(runtime)
274+
}
275+
276+
/// Connect to the local container runtime for an existing gateway.
277+
///
278+
/// Resolution order:
279+
/// 1. Stored runtime from gateway metadata (if metadata exists)
280+
/// 2. Auto-detect runtime via `detect_runtime` (propagates error on failure)
281+
///
282+
/// This is used by code paths that have a gateway `name` but no `runtime`
283+
/// in scope. Unlike `connect_local_auto()`, this checks metadata first so
284+
/// that gateways deployed with a specific runtime reconnect to the same one.
285+
pub(crate) fn connect_for_gateway(name: &str) -> Result<Docker> {
286+
let runtime = match crate::metadata::get_gateway_metadata(name) {
287+
Some(m) => m.container_runtime,
288+
None => crate::container_runtime::detect_runtime(None)?,
289+
};
290+
connect_local(runtime)
291+
}
292+
266293
/// Build a rich, user-friendly error when a container runtime is not reachable.
267294
fn runtime_not_reachable_error(
268295
runtime: ContainerRuntime,
@@ -851,6 +878,11 @@ pub async fn ensure_container(
851878
env_vars.push("GPU_ENABLED=true".to_string());
852879
}
853880

881+
// Pass the container runtime to the entrypoint so it can select the
882+
// appropriate networking stack (nftables kube-proxy for Podman, iptables
883+
// DNS proxy for Docker, etc.).
884+
env_vars.push(format!("CONTAINER_RUNTIME={}", runtime.binary_name()));
885+
854886
let env = Some(env_vars);
855887

856888
// Set the health check explicitly on the container config so it works

‎crates/openshell-bootstrap/src/lib.rs‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -544,7 +544,7 @@ where
544544
.collect();
545545
if !images.is_empty() {
546546
log("[status] Deploying components".to_string());
547-
let local_docker = Docker::connect_with_local_defaults().into_diagnostic()?;
547+
let local_docker = docker::connect_local(runtime)?;
548548
let container = container_name(&name);
549549
let on_log_ref = Arc::clone(&on_log);
550550
let mut push_log = move |msg: String| {
@@ -669,7 +669,7 @@ pub async fn extract_and_store_pki(
669669
) -> Result<()> {
670670
let docker = match remote {
671671
Some(r) => create_ssh_docker_client(r).await?,
672-
None => Docker::connect_with_local_defaults().into_diagnostic()?,
672+
None => docker::connect_for_gateway(name)?,
673673
};
674674
let cname = docker::find_gateway_container(&docker, port).await?;
675675
let bundle = load_existing_pki_bundle(&docker, &cname, constants::KUBECONFIG_PATH)
@@ -684,7 +684,7 @@ pub async fn ensure_gateway_image(
684684
registry_username: Option<&str>,
685685
registry_token: Option<&str>,
686686
) -> Result<String> {
687-
let docker = Docker::connect_with_local_defaults().into_diagnostic()?;
687+
let docker = docker::connect_local_auto()?;
688688
let image_ref = format!("{}:{version}", image::DEFAULT_GATEWAY_IMAGE);
689689
ensure_image(&docker, &image_ref, registry_username, registry_token).await?;
690690
Ok(image_ref)
@@ -712,7 +712,7 @@ pub async fn gateway_container_logs<W: std::io::Write>(
712712

713713
let docker = match remote {
714714
Some(remote_opts) => create_ssh_docker_client(remote_opts).await?,
715-
None => Docker::connect_with_local_defaults().into_diagnostic()?,
715+
None => docker::connect_for_gateway(name)?,
716716
};
717717

718718
let container = container_name(name);
@@ -765,7 +765,7 @@ pub async fn gateway_container_logs<W: std::io::Write>(
765765
/// Returns an empty string on any Docker/connection error so callers don't
766766
/// need to worry about error handling.
767767
pub async fn fetch_gateway_logs(name: &str, n: usize) -> String {
768-
let docker = match Docker::connect_with_local_defaults() {
768+
let docker = match docker::connect_local_auto() {
769769
Ok(d) => d,
770770
Err(_) => return String::new(),
771771
};

‎deploy/docker/Dockerfile.images‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -233,6 +233,7 @@ RUN dnf install -y fedora-repos && \
233233
dnf install -y \
234234
ca-certificates \
235235
iptables \
236+
nftables \
236237
util-linux \
237238
bind-utils \
238239
&& dnf clean all

‎deploy/docker/cluster-entrypoint.sh‎

Lines changed: 62 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -34,22 +34,13 @@ yaml_quote() {
3434
printf "'%s'" "$(printf '%s' "$1" | sed "s/'/''/g")"
3535
}
3636

37-
# ---------------------------------------------------------------------------
38-
# Select iptables backend
39-
# ---------------------------------------------------------------------------
40-
# Some kernels (e.g. Jetson Linux 5.15-tegra) have the nf_tables subsystem
41-
# but lack the nft_compat bridge that allows flannel and kube-proxy to use
42-
# xt extension modules (xt_comment, xt_conntrack). Detect this by probing
43-
# whether xt_comment is usable via the current iptables backend. If the
44-
# probe fails, switch to iptables-legacy. Set USE_IPTABLES_LEGACY=1
45-
# externally to skip the probe and force the legacy backend.
4637
# ---------------------------------------------------------------------------
4738
# Check br_netfilter kernel module
4839
# ---------------------------------------------------------------------------
4940
# br_netfilter makes the kernel pass bridge (pod-to-pod) traffic through
50-
# iptables. Without it, kube-proxy's DNAT rules for ClusterIP services are
51-
# never applied to pod traffic, so pods cannot reach services such as
52-
# kube-dns (10.43.0.10), breaking all in-cluster DNS resolution.
41+
# netfilter (iptables or nftables). Without it, kube-proxy's DNAT rules for
42+
# ClusterIP services are never applied to pod traffic, so pods cannot reach
43+
# services such as kube-dns (10.43.0.10), breaking all in-cluster DNS.
5344
#
5445
# The module must be loaded on the HOST before the container starts —
5546
# containers cannot load kernel modules themselves. If it is missing, log a
@@ -65,25 +56,37 @@ if [ ! -f /proc/sys/net/bridge/bridge-nf-call-iptables ]; then
6556
echo " echo br_netfilter | sudo tee /etc/modules-load.d/br_netfilter.conf" >&2
6657
fi
6758

68-
if [ -z "${USE_IPTABLES_LEGACY:-}" ]; then
69-
if iptables -t filter -N _xt_probe 2>/dev/null; then
70-
_probe_rc=0
71-
iptables -t filter -A _xt_probe -m comment --comment "probe" -j ACCEPT \
72-
2>/dev/null || _probe_rc=$?
73-
iptables -t filter -D _xt_probe -m comment --comment "probe" -j ACCEPT \
74-
2>/dev/null || true
75-
iptables -t filter -X _xt_probe 2>/dev/null || true
76-
[ "$_probe_rc" -ne 0 ] && USE_IPTABLES_LEGACY=1
59+
# ---------------------------------------------------------------------------
60+
# Select iptables backend (Docker only)
61+
# ---------------------------------------------------------------------------
62+
# Under Podman with nftables kube-proxy mode, the iptables backend probe is
63+
# unnecessary — kube-proxy uses nft directly. Flannel still uses the iptables
64+
# binary but through the nft compat shim which doesn't need the xt probe.
65+
#
66+
# Under Docker (or unset runtime), probe whether xt_comment is usable. Some
67+
# kernels (e.g. Jetson Linux 5.15-tegra) have nf_tables but lack the
68+
# nft_compat bridge. If the probe fails, switch to iptables-legacy.
69+
if [ "${CONTAINER_RUNTIME:-}" != "podman" ]; then
70+
if [ -z "${USE_IPTABLES_LEGACY:-}" ]; then
71+
if iptables -t filter -N _xt_probe 2>/dev/null; then
72+
_probe_rc=0
73+
iptables -t filter -A _xt_probe -m comment --comment "probe" -j ACCEPT \
74+
2>/dev/null || _probe_rc=$?
75+
iptables -t filter -D _xt_probe -m comment --comment "probe" -j ACCEPT \
76+
2>/dev/null || true
77+
iptables -t filter -X _xt_probe 2>/dev/null || true
78+
[ "$_probe_rc" -ne 0 ] && USE_IPTABLES_LEGACY=1
79+
fi
7780
fi
78-
fi
7981

80-
if [ "${USE_IPTABLES_LEGACY:-0}" = "1" ]; then
81-
echo "iptables nf_tables xt extension bridge unavailable — switching to iptables-legacy"
82-
if update-alternatives --set iptables /usr/sbin/iptables-legacy 2>/dev/null &&
83-
update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy 2>/dev/null; then
84-
echo "Now using iptables-legacy mode"
85-
else
86-
echo "Warning: could not switch to iptables-legacy — cluster networking may fail"
82+
if [ "${USE_IPTABLES_LEGACY:-0}" = "1" ]; then
83+
echo "iptables nf_tables xt extension bridge unavailable — switching to iptables-legacy"
84+
if update-alternatives --set iptables /usr/sbin/iptables-legacy 2>/dev/null &&
85+
update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy 2>/dev/null; then
86+
echo "Now using iptables-legacy mode"
87+
else
88+
echo "Warning: could not switch to iptables-legacy — cluster networking may fail"
89+
fi
8790
fi
8891
fi
8992

@@ -174,13 +177,20 @@ setup_dns_proxy() {
174177
echo "Configured k3s DNS to use ${CONTAINER_IP} (proxied to Docker DNS)"
175178
}
176179

177-
if ! setup_dns_proxy; then
178-
echo "DNS proxy setup failed, falling back to public DNS servers"
179-
echo "Note: this may not work on Docker Desktop (Mac/Windows)"
180-
cat >"$RESOLV_CONF" <<EOF
180+
if [ "${CONTAINER_RUNTIME:-}" = "podman" ]; then
181+
# Podman DNS is directly routable (aardvark-dns or host DNS) — no proxy
182+
# needed. Copy the container's resolv.conf so k3s has a stable path.
183+
cp /etc/resolv.conf "$RESOLV_CONF"
184+
echo "Podman detected — using host DNS resolution (no proxy needed)"
185+
else
186+
if ! setup_dns_proxy; then
187+
echo "DNS proxy setup failed, falling back to public DNS servers"
188+
echo "Note: this may not work on Docker Desktop (Mac/Windows)"
189+
cat >"$RESOLV_CONF" <<EOF
181190
nameserver 8.8.8.8
182191
nameserver 8.8.4.4
183192
EOF
193+
fi
184194
fi
185195

186196
# ---------------------------------------------------------------------------
@@ -632,7 +642,9 @@ fi
632642
# On kernels where xt_comment is unavailable, kube-router's network policy
633643
# controller panics at startup. Disable it when the iptables-legacy probe
634644
# triggered; sandbox isolation is enforced by the NSSH1 HMAC handshake instead.
635-
if [ "${USE_IPTABLES_LEGACY:-0}" = "1" ]; then
645+
# Under Podman with nftables kube-proxy, the xt probe is skipped entirely so
646+
# USE_IPTABLES_LEGACY is never set — network policy stays enabled.
647+
if [ "${USE_IPTABLES_LEGACY:-0}" = "1" ] && [ "${CONTAINER_RUNTIME:-}" != "podman" ]; then
636648
EXTRA_KUBELET_ARGS="$EXTRA_KUBELET_ARGS --disable-network-policy"
637649
fi
638650

@@ -659,8 +671,23 @@ if [ -n "${OPENSHELL_NODE_NAME:-}" ]; then
659671
echo "Using deterministic k3s node name: ${OPENSHELL_NODE_NAME}"
660672
fi
661673

674+
# ---------------------------------------------------------------------------
675+
# Select kube-proxy mode
676+
# ---------------------------------------------------------------------------
677+
# Under Podman, use native nftables kube-proxy mode so no legacy iptables
678+
# kernel modules (ip_tables, iptable_nat, etc.) are required on the host.
679+
# Docker retains the default iptables mode for maximum compatibility.
680+
EXTRA_KUBE_PROXY_ARGS=""
681+
if [ "${CONTAINER_RUNTIME:-}" = "podman" ]; then
682+
echo "Podman detected — using nftables kube-proxy mode"
683+
EXTRA_KUBE_PROXY_ARGS="--kube-proxy-arg=proxy-mode=nftables"
684+
fi
685+
662686
# Execute k3s with explicit resolv-conf passed as a kubelet arg.
663687
# k3s v1.35.2+ no longer accepts --resolv-conf as a top-level server flag;
664688
# it must be passed via --kubelet-arg instead.
665689
# shellcheck disable=SC2086
666-
exec /bin/k3s "$@" $NODE_NAME_ARG --kubelet-arg=resolv-conf="$RESOLV_CONF" $EXTRA_KUBELET_ARGS
690+
exec /bin/k3s "$@" $NODE_NAME_ARG \
691+
--kubelet-arg=resolv-conf="$RESOLV_CONF" \
692+
$EXTRA_KUBELET_ARGS \
693+
$EXTRA_KUBE_PROXY_ARGS

‎openshell.spec‎

Lines changed: 28 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -94,19 +94,29 @@ cargo build --release --bin openshell
9494
# Install CLI binary
9595
install -Dpm 0755 target/release/%{name} %{buildroot}%{_bindir}/%{name}
9696

97-
# Install modules-load.d config for legacy iptables kernel modules.
98-
# k3s (used by the gateway cluster) bundles its own legacy iptables binary
99-
# for flannel CNI. Modern distros (Fedora 41+, RHEL 10+) only load nf_tables
100-
# by default, so these legacy modules must be explicitly loaded.
97+
# Install modules-load.d config for br_netfilter.
98+
# br_netfilter makes the kernel pass bridged (pod-to-pod) traffic through
99+
# netfilter hooks so kube-proxy DNAT rules (iptables or nftables) apply to
100+
# ClusterIP service traffic. Legacy iptables modules are not required —
101+
# kube-proxy uses native nftables under Podman, and the iptables binary on
102+
# modern distros (Fedora 41+, RHEL 10+) is iptables-nft which uses the
103+
# nf_tables kernel path.
101104
install -d %{buildroot}%{_modulesloaddir}
102105
cat > %{buildroot}%{_modulesloaddir}/%{name}.conf << 'EOF'
103-
# Load legacy iptables kernel modules required by k3s flannel CNI.
104-
# Modern kernels use nf_tables by default; these modules provide the
105-
# legacy iptables interface that k3s's bundled iptables-legacy needs.
106-
ip_tables
107-
iptable_nat
108-
iptable_filter
109-
iptable_mangle
106+
# Load br_netfilter for K3s bridge networking.
107+
# Required so kube-proxy DNAT rules (iptables or nftables) apply to
108+
# bridged pod-to-pod traffic for ClusterIP service resolution.
109+
br_netfilter
110+
EOF
111+
112+
# Install sysctl.d config for bridge netfilter settings required by K3s.
113+
install -d %{buildroot}%{_sysctldir}
114+
cat > %{buildroot}%{_sysctldir}/99-%{name}.conf << 'EOF'
115+
# Enable bridge netfilter call chains for K3s pod-to-service networking.
116+
# Required after br_netfilter is loaded so kube-proxy DNAT rules apply
117+
# to bridged pod traffic.
118+
net.bridge.bridge-nf-call-iptables = 1
119+
net.bridge.bridge-nf-call-ip6tables = 1
110120
EOF
111121

112122
# Install Python SDK modules (test files are intentionally excluded)
@@ -138,6 +148,12 @@ echo "rpm" > %{buildroot}%{python3_sitelib}/%{name}-%{version}.dist-info/INSTALL
138148
# RECORD can be empty for RPM-managed installs
139149
touch %{buildroot}%{python3_sitelib}/%{name}-%{version}.dist-info/RECORD
140150

151+
%post
152+
# Load br_netfilter immediately so a reboot is not required after install.
153+
# The modules-load.d config handles subsequent boots.
154+
modprobe br_netfilter > /dev/null 2>&1 || :
155+
%sysctl_apply 99-%{name}.conf
156+
141157
%check
142158
# Smoke-test the CLI binary
143159
%{buildroot}%{_bindir}/%{name} --version
@@ -153,6 +169,7 @@ PYTHONPATH=%{buildroot}%{python3_sitelib} %{python3} -c "from importlib.metadata
153169
%doc README.md
154170
%{_bindir}/%{name}
155171
%{_modulesloaddir}/%{name}.conf
172+
%{_sysctldir}/99-%{name}.conf
156173

157174
%files -n python3-%{name}
158175
%license LICENSE

0 commit comments

Comments
 (0)