Skip to content

Commit c355de6

Browse files
IvanHuntersIvanHunters
committed
fix(backups): exec into a CLI Pod in the nats demo
nats_cli seeded and verified JetStream streams through a throwaway `kubectl run -i` Pod per call, with the same attach-race empty-read the redis demo fixed. NATS ships no long-lived Pod carrying the `nats` CLI, so run one nats-box Pod and `kubectl exec` into it; stream_message_count keeps the exec's stderr so a failed read says why. The password travels in NATS_PASSWORD on stdin rather than the --server URL the exec request records in the apiserver audit log. The Pod carries cozystack.io/backup-demo=nats: nats_cli_pod reuses only a Ready Pod the demo owns, replaces a leftover in a terminal phase, and refuses a same-named Pod it does not own; cleanup.sh reaps by that label with --grace-period=1. Unit tests under hack/ pin the exec path, the preserved stderr, and the password staying out of argv. Assisted-by: LLM Signed-off-by: IvanHunters <[email protected]>
1 parent 9cea7dc commit c355de6

3 files changed

Lines changed: 173 additions & 11 deletions

File tree

‎examples/backups/nats/00-helpers.sh‎

Lines changed: 54 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,10 @@ export RESTOREJOB_INPLACE_NAME="${RESTOREJOB_INPLACE_NAME:-nats-restore-inplace}
2828
export RESTOREJOB_TOCOPY_NAME="${RESTOREJOB_TOCOPY_NAME:-nats-restore-to-copy}"
2929
# natsio/nats-box ships the `nats` CLI plus curl + tar + sh - everything the
3030
# generic Job strategy needs, with no purpose-built backup image. The seed /
31-
# verify helpers below run it as a throwaway Pod via `kubectl run`.
31+
# verify helpers below run it as a long-lived Pod they `kubectl exec` into.
3232
export NATS_BOX_IMAGE="${NATS_BOX_IMAGE:-natsio/nats-box:0.14.5}"
33+
# Name of that long-lived CLI Pod; cleanup.sh removes it.
34+
export NATS_CLI_POD="${NATS_CLI_POD:-nats-cli}"
3335

3436
log_info() { echo -e "${BLUE}i${NC} $*" >&2; }
3537
log_success() { echo -e "${GREEN}OK${NC} $*" >&2; }
@@ -82,28 +84,69 @@ wait_for_field() {
8284
# names its client Service after the application (fullnameOverride=<name>) and
8385
# stores per-user passwords in the "<name>-credentials" Secret. The demo sets a
8486
# fixed password (NATS_PASSWORD) on both the source and restore-target apps, so
85-
# a single URL shape works everywhere.
87+
# a single URL shape works everywhere. The password is not in the URL: nats_cli
88+
# passes it to the CLI in NATS_PASSWORD on stdin, to keep it out of the exec
89+
# request URL the apiserver records in its audit log.
8690
nats_url() {
8791
local app="$1"
88-
echo "nats://${NATS_USER}:${NATS_PASSWORD}@${app}.${NAMESPACE}.svc:4222"
92+
echo "nats://${NATS_USER}@${app}.${NAMESPACE}.svc:4222"
8993
}
9094

91-
# Run a `nats` CLI invocation against an application instance from a throwaway
92-
# nats-box Pod. Args after the app name are passed verbatim to `nats`.
93-
# Example: nats_cli "$NATS_NAME" stream ls
95+
# Ensure the long-lived nats-box CLI Pod exists and is Ready, so nats_cli can
96+
# exec into it. Idempotent: a Ready Pod this demo owns is reused across calls and
97+
# across the numbered demo scripts; a leftover in a terminal phase (Succeeded /
98+
# Failed) is replaced rather than waited on; and a same-named Pod this demo does
99+
# not own is refused rather than hijacked or deleted. cleanup.sh removes it.
100+
nats_cli_pod() {
101+
local phase owner
102+
phase=$(kubectl -n "$NAMESPACE" get pod "$NATS_CLI_POD" -o jsonpath='{.status.phase}' 2>/dev/null || true)
103+
owner=$(kubectl -n "$NAMESPACE" get pod "$NATS_CLI_POD" -o jsonpath='{.metadata.labels.cozystack\.io/backup-demo}' 2>/dev/null || true)
104+
if [ -n "$phase" ] && [ "$owner" != "nats" ]; then
105+
log_error "Pod $NAMESPACE/$NATS_CLI_POD exists but this demo does not own it; refusing to use or delete it"
106+
return 1
107+
fi
108+
if [ "$phase" != "Running" ] && [ "$phase" != "Pending" ]; then
109+
kubectl -n "$NAMESPACE" delete pod "$NATS_CLI_POD" --grace-period=1 --ignore-not-found >/dev/null
110+
kubectl -n "$NAMESPACE" run "$NATS_CLI_POD" --image="$NATS_BOX_IMAGE" \
111+
--labels=cozystack.io/backup-demo=nats \
112+
--restart=Never --command -- sleep infinity >/dev/null
113+
fi
114+
kubectl -n "$NAMESPACE" wait --for=condition=Ready "pod/$NATS_CLI_POD" \
115+
--timeout=5m >/dev/null
116+
}
117+
118+
# Run a `nats` CLI invocation against an application instance. Args after the app
119+
# name are passed verbatim to `nats`. Example: nats_cli "$NATS_NAME" stream ls
120+
#
121+
# The command runs by `kubectl exec` in the long-lived nats-box Pod, not a
122+
# throwaway `kubectl run -i` Pod per call. A throwaway Pod's stdout comes back
123+
# over an attach that only carries what the container writes after the attach
124+
# registers, so a CLI that finishes in between returns empty with exit 0. An
125+
# exec'd process owns its pipes, so its output cannot be missed that way, and
126+
# kubectl's and the CLI's stderr stay attached so a failed read says why.
127+
#
128+
# The password travels in NATS_PASSWORD on stdin rather than in the --server URL,
129+
# which the exec request carries into the apiserver's audit log. The nats CLI
130+
# reads NATS_PASSWORD from its environment.
94131
nats_cli() {
95132
local app="$1"; shift
96-
kubectl -n "$NAMESPACE" run "nats-cli-$RANDOM" \
97-
--image="$NATS_BOX_IMAGE" --restart=Never --rm -i --quiet \
98-
--command -- nats --server "$(nats_url "$app")" "$@"
133+
nats_cli_pod
134+
printf '%s\n' "$NATS_PASSWORD" | kubectl -n "$NAMESPACE" exec -i "$NATS_CLI_POD" -- sh -c '
135+
IFS= read -r NATS_PASSWORD || true
136+
export NATS_PASSWORD
137+
server=$1; shift
138+
exec nats --server "$server" "$@"
139+
' sh "$(nats_url "$app")" "$@"
99140
}
100141

101142
# Number of messages currently stored in a JetStream stream, or "" if the
102-
# stream does not exist.
143+
# stream does not exist. Only jq's stderr is dropped (a missing stream prints a
144+
# harmless "stream not found" there); nats_cli's own stderr stays, so an exec
145+
# that fails says why instead of surfacing as an empty count.
103146
stream_message_count() {
104147
local app="$1"
105148
local stream="$2"
106-
nats_cli "$app" stream info "$stream" --json 2>/dev/null \
149+
nats_cli "$app" stream info "$stream" --json \
107150
| jq -r '.state.messages // empty' 2>/dev/null | tr -d '[:space:]'
108151
}
109152

‎examples/backups/nats/cleanup.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ kubectl -n "$NAMESPACE" delete secret "${NATS_RESTORE_NAME}-backup-s3" --ignore-
1616
kubectl -n "$NAMESPACE" delete secret "${NATS_NAME}-backup-s3" --ignore-not-found
1717
kubectl -n "$NAMESPACE" delete nats "$NATS_RESTORE_NAME" --ignore-not-found
1818
kubectl -n "$NAMESPACE" delete nats "$NATS_NAME" --ignore-not-found
19+
kubectl -n "$NAMESPACE" delete pod -l cozystack.io/backup-demo=nats --grace-period=1 --ignore-not-found
1920
kubectl -n "$NAMESPACE" delete bucket "$BUCKET_NAME" --ignore-not-found
2021
rm -f "$SCRIPT_DIR/.bucket-info.env"
2122
kubectl delete backupclass "$BACKUPCLASS_NAME" --ignore-not-found

‎hack/nats-backup-demo-helpers.bats‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
#!/usr/bin/env bats
2+
# Unit tests for nats_cli in examples/backups/nats/00-helpers.sh, the helper the
3+
# nats roundtrip suite seeds and verifies JetStream streams through.
4+
#
5+
# kubectl and the `nats` CLI are stub executables on PATH. The kubectl stub
6+
# answers the calls nats_cli makes -- get the CLI Pod's phase and owner label,
7+
# create it when absent, wait for it, and `exec` into it -- and runs the exec'd
8+
# `nats` invocation locally against the nats stub. STUB_PHASE / STUB_OWNER drive
9+
# the reuse guard, and STUB_EXEC_ERROR makes the exec fail so the "a failed read
10+
# says why" claim is pinned. The password is piped on stdin, so it must not
11+
# appear in the recorded argv.
12+
#
13+
# cozytest.sh's awk runner recognizes only @test blocks and a bare `}`; there is
14+
# no bats `run` or `$status`, and no setup/teardown, so each case calls
15+
# make_stubs itself and asserts with direct shell tests that exit non-zero. The
16+
# helper functions close on an indented brace because the runner rewrites a
17+
# column-0 `}` into `return 0`, which would mask run_stream_count's exit status.
18+
19+
make_stubs() {
20+
stub=$(mktemp -d)
21+
cat > "$stub/kubectl" <<'EOF'
22+
#!/bin/bash
23+
printf '%s\n' "$*" >> "$STUB_DIR/kubectl.argv"
24+
case "$1 $2 $3" in
25+
"-n tenant-root get")
26+
case "$*" in
27+
*"jsonpath={.status.phase}"*) printf '%s' "${STUB_PHASE-Running}" ;;
28+
*"jsonpath={.metadata.labels"*) printf '%s' "${STUB_OWNER-nats}" ;;
29+
esac
30+
exit 0 ;;
31+
"-n tenant-root delete")
32+
exit 0 ;;
33+
"-n tenant-root run")
34+
# Reachable only when the Pod is absent; a reuse that starts a Pod is a bug.
35+
[ -z "${STUB_PHASE-Running}" ] && exit 0
36+
echo "nats_cli started a Pod though a Ready one it owns exists" >&2; exit 95 ;;
37+
"-n tenant-root wait")
38+
exit 0 ;;
39+
"-n tenant-root exec")
40+
[ -n "${STUB_EXEC_ERROR:-}" ] && { echo "$STUB_EXEC_ERROR" >&2; exit 1; }
41+
shift 6 # drop: -n tenant-root exec -i <pod> --
42+
exec "$@" ;; # sh -c <script> sh <url> <args...>
43+
*)
44+
echo "unexpected kubectl call: $*" >&2; exit 97 ;;
45+
esac
46+
EOF
47+
cat > "$stub/nats" <<'EOF'
48+
#!/bin/sh
49+
# nats stub: only `stream info <name> --json` is exercised.
50+
case "$*" in
51+
*"stream info"*"--json"*)
52+
printf '{"state":{"messages":42}}\n'; exit 0 ;;
53+
esac
54+
echo "unexpected nats call: $*" >&2; exit 90
55+
EOF
56+
chmod +x "$stub/kubectl" "$stub/nats"
57+
: > "$stub/kubectl.argv"
58+
}
59+
60+
run_stream_count() {
61+
STUB_DIR=$stub STUB_PHASE="${STUB_PHASE-Running}" STUB_OWNER="${STUB_OWNER-nats}" \
62+
STUB_EXEC_ERROR="${STUB_EXEC_ERROR:-}" PATH="$stub:$PATH" NAMESPACE=tenant-root \
63+
NATS_USER=demo NATS_PASSWORD=s3cret bash -c '
64+
set -euo pipefail
65+
. examples/backups/nats/00-helpers.sh
66+
stream_message_count demo orders
67+
'
68+
}
69+
70+
@test "nats_cli reuses a Ready Pod it owns and reads the reply by exec" {
71+
make_stubs
72+
out=$(run_stream_count)
73+
[ "$out" = "42" ]
74+
grep -q 'exec -i nats-cli -- sh -c' "$stub/kubectl.argv"
75+
}
76+
77+
@test "nats_cli passes the password on stdin, keeping it out of the exec argv" {
78+
make_stubs
79+
out=$(run_stream_count)
80+
[ "$out" = "42" ]
81+
grep -q 'nats://[email protected]:4222' "$stub/kubectl.argv"
82+
if grep -q s3cret "$stub/kubectl.argv"; then
83+
echo "the password reached kubectl's argv" >&2
84+
exit 1
85+
fi
86+
}
87+
88+
@test "nats_cli creates the CLI Pod when it is absent, then execs into it" {
89+
make_stubs
90+
export STUB_PHASE=
91+
out=$(run_stream_count)
92+
[ "$out" = "42" ]
93+
grep -q 'run nats-cli --image=' "$stub/kubectl.argv"
94+
grep -q 'labels=cozystack.io/backup-demo=nats' "$stub/kubectl.argv"
95+
grep -q 'exec -i nats-cli -- sh -c' "$stub/kubectl.argv"
96+
}
97+
98+
@test "nats_cli refuses a same-named Pod the demo does not own" {
99+
make_stubs
100+
export STUB_OWNER=someone-else
101+
err=$(mktemp)
102+
if run_stream_count 2>"$err"; then
103+
echo "nats_cli used a Pod it does not own" >&2
104+
exit 1
105+
fi
106+
grep -q 'does not own it' "$err"
107+
}
108+
109+
@test "nats_cli fails with kubectl's own error when the exec fails" {
110+
make_stubs
111+
export STUB_EXEC_ERROR="error: unable to upgrade connection: container not found"
112+
err=$(mktemp)
113+
if run_stream_count 2>"$err"; then
114+
echo "a failed exec read as a successful empty count" >&2
115+
exit 1
116+
fi
117+
grep -q 'unable to upgrade connection: container not found' "$err"
118+
}

0 commit comments

Comments
 (0)