Skip to content

Commit 727343d

Browse files
committed
docs(backups): add NATS Job-strategy example
Add examples/backups/nats/, the first worked example of the generic Job backup strategy. It backs up a NATS JetStream stream that has no dedicated driver, using a stock natsio/nats-box image: 'nats stream backup' -> tar -> curl --aws-sigv4 PUT to the COSI/seaweedfs S3 endpoint, and the reverse for restore. One PodTemplateSpec serves both directions, branching at runtime on {{ .Mode }} (the engine templates each string leaf independently and cannot restructure the Pod per mode). Mirrors the clickhouse example's structure: numbered scripts (bucket -> NATS src + seeded stream -> strategy -> BackupClass -> BackupJob -> in-place and to-copy RestoreJob), shared helpers, run-all.sh, README, and scenario docs. Like every examples/backups/* dir it is a runnable demo, not CI-gated. Signed-off-by: Timofei Larkin <[email protected]>
1 parent 5029af6 commit 727343d

14 files changed

Lines changed: 868 additions & 0 deletions
Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
#!/bin/bash
2+
# Shared helpers for the NATS JetStream backup/restore demo.
3+
# Source this file in other scripts: source "$(dirname "$0")/00-helpers.sh"
4+
5+
export RED='\033[0;31m'
6+
export GREEN='\033[0;32m'
7+
export YELLOW='\033[1;33m'
8+
export BLUE='\033[0;34m'
9+
export MAGENTA='\033[0;35m'
10+
export CYAN='\033[0;36m'
11+
export WHITE='\033[1;37m'
12+
export NC='\033[0m'
13+
export BOLD='\033[1m'
14+
15+
# Default settings (override via environment).
16+
export NAMESPACE="${NAMESPACE:-tenant-test}"
17+
export NATS_NAME="${NATS_NAME:-nats-test}"
18+
export NATS_RESTORE_NAME="${NATS_RESTORE_NAME:-nats-restore}"
19+
export NATS_USER="${NATS_USER:-backup}"
20+
export NATS_PASSWORD="${NATS_PASSWORD:-jetstream-demo-pw}"
21+
export STREAM_NAME="${STREAM_NAME:-ORDERS}"
22+
export MESSAGE_COUNT="${MESSAGE_COUNT:-10}"
23+
export BUCKET_NAME="${BUCKET_NAME:-nats-backups}"
24+
export BACKUPCLASS_NAME="${BACKUPCLASS_NAME:-nats-backup}"
25+
export STRATEGY_NAME="${STRATEGY_NAME:-nats-job}"
26+
export BACKUPJOB_NAME="${BACKUPJOB_NAME:-nats-backup-job}"
27+
export RESTOREJOB_INPLACE_NAME="${RESTOREJOB_INPLACE_NAME:-nats-restore-inplace}"
28+
export RESTOREJOB_TOCOPY_NAME="${RESTOREJOB_TOCOPY_NAME:-nats-restore-to-copy}"
29+
# natsio/nats-box ships the `nats` CLI plus curl + tar + sh - everything the
30+
# generic Job strategy needs, with no purpose-built backup image. The seed /
31+
# verify helpers below run it as a throwaway Pod via `kubectl run`.
32+
export NATS_BOX_IMAGE="${NATS_BOX_IMAGE:-natsio/nats-box:0.14.5}"
33+
34+
log_info() { echo -e "${BLUE}i${NC} $*" >&2; }
35+
log_success() { echo -e "${GREEN}OK${NC} $*" >&2; }
36+
log_warning() { echo -e "${YELLOW}!${NC} $*" >&2; }
37+
log_error() { echo -e "${RED}x${NC} $*" >&2; }
38+
log_step() { echo -e "\n${MAGENTA}${BOLD}> $*${NC}" >&2; }
39+
log_substep() { echo -e "${CYAN} -> $*${NC}" >&2; }
40+
log_command() { echo -e "${WHITE} $ $*${NC}" >&2; }
41+
42+
separator() {
43+
echo -e "\n${CYAN}------------------------------------------------------------${NC}\n" >&2
44+
}
45+
46+
print_header() {
47+
local title="$1"
48+
echo -e "\n${MAGENTA}${BOLD}== $title ==${NC}\n" >&2
49+
}
50+
51+
# Wait until a JSONPath value on a resource matches the desired string.
52+
wait_for_field() {
53+
local resource_type="$1"
54+
local resource_name="$2"
55+
local jsonpath="$3"
56+
local desired="$4"
57+
local namespace="${5:-}"
58+
local timeout="${6:-300}"
59+
60+
log_substep "Waiting for $resource_type/$resource_name $jsonpath to become '$desired'..."
61+
local elapsed=0
62+
local ns_flag=()
63+
[[ -n "$namespace" ]] && ns_flag=(-n "$namespace")
64+
65+
while true; do
66+
local current
67+
current=$(kubectl get "$resource_type" "$resource_name" "${ns_flag[@]}" -o jsonpath="$jsonpath" 2>/dev/null || true)
68+
if [[ "$current" == "$desired" ]]; then
69+
log_success "$resource_type/$resource_name reached '$desired'"
70+
return 0
71+
fi
72+
if [[ $elapsed -ge $timeout ]]; then
73+
log_error "Timeout waiting for $resource_type/$resource_name (current: '$current', expected: '$desired')"
74+
return 1
75+
fi
76+
sleep 5
77+
elapsed=$((elapsed + 5))
78+
done
79+
}
80+
81+
# In-cluster NATS client URL for the given application instance. The NATS app
82+
# names its client Service after the application (fullnameOverride=<name>) and
83+
# stores per-user passwords in the "<name>-credentials" Secret. The demo sets a
84+
# fixed password (NATS_PASSWORD) on both the source and restore-target apps, so
85+
# a single URL shape works everywhere.
86+
nats_url() {
87+
local app="$1"
88+
echo "nats://${NATS_USER}:${NATS_PASSWORD}@${app}.${NAMESPACE}.svc:4222"
89+
}
90+
91+
# Run a `nats` CLI invocation against an application instance from a throwaway
92+
# nats-box Pod. Args after the app name are passed verbatim to `nats`.
93+
# Example: nats_cli "$NATS_NAME" stream ls
94+
nats_cli() {
95+
local app="$1"; shift
96+
kubectl -n "$NAMESPACE" run "nats-cli-$RANDOM" \
97+
--image="$NATS_BOX_IMAGE" --restart=Never --rm -i --quiet \
98+
--command -- nats --server "$(nats_url "$app")" "$@"
99+
}
100+
101+
# Number of messages currently stored in a JetStream stream, or "" if the
102+
# stream does not exist.
103+
stream_message_count() {
104+
local app="$1"
105+
local stream="$2"
106+
nats_cli "$app" stream info "$stream" --json 2>/dev/null \
107+
| jq -r '.state.messages // empty' 2>/dev/null | tr -d '[:space:]'
108+
}
109+
110+
# Create the "<app>-backup-s3" Secret the Job strategy Pod consumes, from the
111+
# bucket coordinates cached by 03-create-bucket.sh. The generic Job strategy -
112+
# unlike the app-specific drivers - has no chart support to emit this Secret,
113+
# so the tenant provides it. Called for the source app (step 04) and the
114+
# restore target (step 07).
115+
create_s3_secret() {
116+
local app="$1"
117+
local SCRIPT_DIR
118+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
119+
[[ -f "$SCRIPT_DIR/.bucket-info.env" ]] || { log_error "missing $SCRIPT_DIR/.bucket-info.env; run 03-create-bucket.sh first"; return 1; }
120+
# shellcheck disable=SC1091
121+
source "$SCRIPT_DIR/.bucket-info.env"
122+
for v in S3_ACCESS_KEY S3_SECRET_KEY S3_ENDPOINT S3_REGION S3_BUCKET; do
123+
[[ -n "${!v:-}" ]] || { log_error "required variable is missing or empty: ${v}"; return 1; }
124+
done
125+
kubectl -n "$NAMESPACE" create secret generic "${app}-backup-s3" \
126+
--from-literal=accessKey="$S3_ACCESS_KEY" \
127+
--from-literal=secretKey="$S3_SECRET_KEY" \
128+
--from-literal=endpoint="$S3_ENDPOINT" \
129+
--from-literal=region="$S3_REGION" \
130+
--from-literal=bucket="$S3_BUCKET" \
131+
--dry-run=client -o yaml | kubectl apply -f -
132+
}
Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
#!/bin/bash
2+
# Step 01: Create the cluster-scoped generic Job backup strategy for NATS
3+
# JetStream. Unlike the app-specific drivers (Altinity, CNPG, ...), the Job
4+
# strategy runs an arbitrary Pod the operator supplies. Here that Pod is a
5+
# stock natsio/nats-box image (which carries the `nats` CLI plus curl and tar)
6+
# running a single shell script that branches on `.Mode`:
7+
# backup : `nats stream backup` -> tar -> PUT the tarball to S3
8+
# restore: GET the tarball from S3 -> untar -> `nats stream restore`
9+
#
10+
# Everything the template injects is funnelled through container env vars so
11+
# the script body stays plain shell. Note how the same PodTemplateSpec serves
12+
# both directions: the engine templates each string leaf independently (it
13+
# cannot add/remove containers per mode), so the *one* image branches at
14+
# runtime on $MODE.
15+
set -euo pipefail
16+
17+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
18+
source "$SCRIPT_DIR/00-helpers.sh"
19+
20+
print_header "Step 01: Create Job backup strategy '${STRATEGY_NAME}'"
21+
log_command "kubectl apply -f - (Job strategy: $STRATEGY_NAME)"
22+
23+
kubectl apply -f - <<EOF
24+
apiVersion: strategy.backups.cozystack.io/v1alpha1
25+
kind: Job
26+
metadata:
27+
name: ${STRATEGY_NAME}
28+
spec:
29+
template:
30+
spec:
31+
restartPolicy: Never
32+
containers:
33+
- name: nats-backup
34+
image: ${NATS_BOX_IMAGE}
35+
imagePullPolicy: IfNotPresent
36+
env:
37+
# .Parameters carry the static knobs from the BackupClass: which
38+
# JetStream stream to back up and which NATS user to authenticate
39+
# as. On restore they are read back from the Backup's
40+
# driverMetadata, so the same values apply round-trip.
41+
- name: STREAM
42+
value: '{{ default "ORDERS" (index .Parameters "stream") }}'
43+
- name: NATS_USER
44+
value: '{{ default "backup" (index .Parameters "natsUser") }}'
45+
# .Release is the application being acted on: the source on backup,
46+
# the restore target on restore (in-place=source, to-copy=the new
47+
# app). The NATS app exposes its client Service as <name>.<ns>.svc
48+
# and stores per-user passwords in <name>-credentials.
49+
- name: NATS_HOST
50+
value: "{{ .Release.Name }}.{{ .Release.Namespace }}.svc"
51+
- name: NATS_PASSWORD
52+
valueFrom:
53+
secretKeyRef:
54+
name: "{{ .Release.Name }}-credentials"
55+
key: '{{ default "backup" (index .Parameters "natsUser") }}'
56+
# S3 object key is scoped by the SOURCE app name so a to-copy
57+
# restore reads what the source wrote. On backup the source is
58+
# .Release.Name; on restore it is .Backup.ApplicationRef.Name
59+
# (.Backup is only set on restore - the guard keeps backup mode
60+
# from rendering "<no value>" into an unused var).
61+
- name: SRC_BACKUP
62+
value: "{{ .Release.Name }}"
63+
- name: SRC_RESTORE
64+
value: "{{ if .Backup }}{{ .Backup.ApplicationRef.Name }}{{ end }}"
65+
- name: MODE
66+
value: "{{ .Mode }}"
67+
# S3 coordinates from the tenant-provided <release>-backup-s3 Secret
68+
# (created by create_s3_secret in 00-helpers.sh from the Bucket).
69+
- name: AWS_ACCESS_KEY_ID
70+
valueFrom:
71+
secretKeyRef:
72+
name: "{{ .Release.Name }}-backup-s3"
73+
key: accessKey
74+
- name: AWS_SECRET_ACCESS_KEY
75+
valueFrom:
76+
secretKeyRef:
77+
name: "{{ .Release.Name }}-backup-s3"
78+
key: secretKey
79+
- name: S3_ENDPOINT
80+
valueFrom:
81+
secretKeyRef:
82+
name: "{{ .Release.Name }}-backup-s3"
83+
key: endpoint
84+
- name: S3_REGION
85+
valueFrom:
86+
secretKeyRef:
87+
name: "{{ .Release.Name }}-backup-s3"
88+
key: region
89+
- name: S3_BUCKET
90+
valueFrom:
91+
secretKeyRef:
92+
name: "{{ .Release.Name }}-backup-s3"
93+
key: bucket
94+
command: ["/bin/sh", "-c"]
95+
args:
96+
- |
97+
set -eu
98+
NATS_URL="nats://\${NATS_USER}:\${NATS_PASSWORD}@\${NATS_HOST}:4222"
99+
100+
# BucketInfo endpoints are bare host:port; COSI/seaweedfs serves
101+
# S3 over HTTPS, so default to https:// when no scheme is given.
102+
case "\${S3_ENDPOINT}" in
103+
http://*|https://*) BASE="\${S3_ENDPOINT}" ;;
104+
*) BASE="https://\${S3_ENDPOINT}" ;;
105+
esac
106+
107+
if [ "\${MODE}" = backup ]; then SRC="\${SRC_BACKUP}"; else SRC="\${SRC_RESTORE}"; fi
108+
KEY="\${SRC}/\${STREAM}.tar"
109+
OBJ_URL="\${BASE}/\${S3_BUCKET}/\${KEY}"
110+
111+
# curl --aws-sigv4 signs the request (SigV4) so no separate S3
112+
# client image is needed. -k accepts seaweedfs's internal
113+
# self-signed cert; a production strategy would mount the tenant
114+
# CA and drop -k.
115+
s3() { curl -fsS -k --aws-sigv4 "aws:amz:\${S3_REGION}:s3" --user "\${AWS_ACCESS_KEY_ID}:\${AWS_SECRET_ACCESS_KEY}" "\$@"; }
116+
117+
rm -rf /tmp/bk && mkdir -p /tmp/bk
118+
if [ "\${MODE}" = backup ]; then
119+
echo "backing up JetStream stream \${STREAM} from \${NATS_HOST}"
120+
nats --server "\${NATS_URL}" stream backup "\${STREAM}" /tmp/bk
121+
tar -C /tmp/bk -cf /tmp/backup.tar .
122+
s3 -X PUT --upload-file /tmp/backup.tar "\${OBJ_URL}"
123+
echo "uploaded s3://\${S3_BUCKET}/\${KEY}"
124+
else
125+
echo "restoring JetStream stream \${STREAM} into \${NATS_HOST} from s3://\${S3_BUCKET}/\${KEY}"
126+
s3 -o /tmp/backup.tar "\${OBJ_URL}"
127+
tar -C /tmp/bk -xf /tmp/backup.tar
128+
nats --server "\${NATS_URL}" stream restore /tmp/bk
129+
echo "restore complete"
130+
fi
131+
resources:
132+
requests:
133+
cpu: 10m
134+
memory: 32Mi
135+
limits:
136+
cpu: 200m
137+
memory: 128Mi
138+
securityContext:
139+
# Minimal hardening, matching the ClickHouse example. nats-box runs
140+
# the script as its default user; tenants enforcing PSA "restricted"
141+
# should add runAsNonRoot/runAsUser for the image they pin.
142+
allowPrivilegeEscalation: false
143+
capabilities:
144+
drop: ["ALL"]
145+
seccompProfile:
146+
type: RuntimeDefault
147+
EOF
148+
149+
log_success "Job strategy '${STRATEGY_NAME}' created."
150+
echo -e "\n${GREEN}${BOLD}Next:${NC} ./02-create-backupclass.sh"
Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
#!/bin/bash
2+
# Step 02: Map the NATS application kind to the Job strategy from step 01. The
3+
# strategy parameters (which stream to back up, which NATS user to connect as)
4+
# travel through the BackupClass and are exposed to the strategy template as
5+
# `.Parameters`. They are also snapshotted onto the resulting Backup's
6+
# driverMetadata, so a later restore re-renders with the same values.
7+
set -euo pipefail
8+
9+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
10+
source "$SCRIPT_DIR/00-helpers.sh"
11+
12+
print_header "Step 02: Create BackupClass '${BACKUPCLASS_NAME}'"
13+
log_command "kubectl apply -f - (BackupClass: $BACKUPCLASS_NAME -> $STRATEGY_NAME)"
14+
15+
kubectl apply -f - <<EOF
16+
apiVersion: backups.cozystack.io/v1alpha1
17+
kind: BackupClass
18+
metadata:
19+
name: ${BACKUPCLASS_NAME}
20+
spec:
21+
strategies:
22+
- application:
23+
apiGroup: apps.cozystack.io
24+
kind: NATS
25+
strategyRef:
26+
apiGroup: strategy.backups.cozystack.io
27+
kind: Job
28+
name: ${STRATEGY_NAME}
29+
parameters:
30+
stream: "${STREAM_NAME}"
31+
natsUser: "${NATS_USER}"
32+
EOF
33+
34+
log_success "BackupClass '${BACKUPCLASS_NAME}' created."
35+
echo -e "\n${GREEN}${BOLD}Next:${NC} ./03-create-bucket.sh"
Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
#!/bin/bash
2+
# Step 03: Provision an in-cluster Bucket and cache its S3 coordinates. Unlike
3+
# the ClickHouse demo - where the chart emits the backup Secret from
4+
# spec.backup.* - the generic Job strategy has no chart support, so the tenant
5+
# materialises the "<app>-backup-s3" Secret itself. This step only caches the
6+
# coordinates; create_s3_secret (00-helpers.sh) turns them into per-app Secrets
7+
# in steps 04 and 07.
8+
set -euo pipefail
9+
10+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
11+
source "$SCRIPT_DIR/00-helpers.sh"
12+
13+
print_header "Step 03: Provision Bucket '${BUCKET_NAME}' in '${NAMESPACE}'"
14+
15+
log_command "kubectl apply -f - (Bucket: $BUCKET_NAME)"
16+
# spec.users.backup is required for the Bucket app to provision a
17+
# BucketAccess (and credentials Secret) the backup tooling can read.
18+
kubectl apply -f - <<EOF
19+
apiVersion: apps.cozystack.io/v1alpha1
20+
kind: Bucket
21+
metadata:
22+
name: ${BUCKET_NAME}
23+
namespace: ${NAMESPACE}
24+
spec:
25+
users:
26+
backup:
27+
readonly: false
28+
EOF
29+
30+
log_substep "Waiting for bucket HelmRelease to be Ready..."
31+
kubectl -n "$NAMESPACE" wait hr "bucket-${BUCKET_NAME}" --for=condition=ready --timeout=300s
32+
kubectl -n "$NAMESPACE" wait bucketclaims.objectstorage.k8s.io "bucket-${BUCKET_NAME}" --for=jsonpath='{.status.bucketReady}'=true --timeout=300s
33+
# Cozystack's bucket app provisions a BucketAccess named "<bucket-name>-backup"
34+
# (the "-backup" suffix is the BucketAccessClass name); the BucketInfo Secret
35+
# carries the same name.
36+
kubectl -n "$NAMESPACE" wait bucketaccesses.objectstorage.k8s.io "bucket-${BUCKET_NAME}-backup" --for=jsonpath='{.status.accessGranted}'=true --timeout=300s
37+
38+
log_substep "Reading bucket coordinates from BucketInfo Secret..."
39+
TMP=$(mktemp)
40+
trap 'rm -f "$TMP"' EXIT
41+
kubectl -n "$NAMESPACE" get secret "bucket-${BUCKET_NAME}-backup" -o jsonpath='{.data.BucketInfo}' | base64 -d > "$TMP"
42+
43+
S3_ACCESS_KEY=$(jq -r '.spec.secretS3.accessKeyID' "$TMP")
44+
S3_SECRET_KEY=$(jq -r '.spec.secretS3.accessSecretKey' "$TMP")
45+
S3_ENDPOINT=$(jq -r '.spec.secretS3.endpoint' "$TMP")
46+
S3_REGION=$(jq -r 'if (.spec.secretS3.region // "") == "" then "us-east-1" else .spec.secretS3.region end' "$TMP")
47+
S3_BUCKET=$(jq -r '.spec.bucketName' "$TMP")
48+
49+
# `jq -r` returns the literal string "null" for missing JSON paths; fail fast
50+
# here so a missing field surfaces at extraction time instead of as a confusing
51+
# curl/SigV4 error inside the backup Pod.
52+
for v in S3_ACCESS_KEY S3_SECRET_KEY S3_ENDPOINT S3_REGION S3_BUCKET; do
53+
[[ -n "${!v}" && "${!v}" != "null" ]] || { log_error "BucketInfo missing required field: ${v}"; exit 1; }
54+
done
55+
56+
# Persist for steps 04 / 07. The cache stores raw S3 credentials, so apply
57+
# restrictive perms before writing the body - umask alone could leave the file
58+
# group/world-readable.
59+
umask 077
60+
cat > "$SCRIPT_DIR/.bucket-info.env" <<ENV
61+
export S3_ACCESS_KEY=${S3_ACCESS_KEY}
62+
export S3_SECRET_KEY=${S3_SECRET_KEY}
63+
export S3_ENDPOINT=${S3_ENDPOINT}
64+
export S3_REGION=${S3_REGION}
65+
export S3_BUCKET=${S3_BUCKET}
66+
ENV
67+
chmod 600 "$SCRIPT_DIR/.bucket-info.env"
68+
69+
log_success "Bucket '${BUCKET_NAME}' ready; coordinates cached in $(basename "$SCRIPT_DIR")/.bucket-info.env."
70+
echo -e "\n${GREEN}${BOLD}Next:${NC} ./04-create-nats.sh"

0 commit comments

Comments
 (0)