|
| 1 | +#!/bin/bash |
| 2 | +# Shared helpers for the Kafka topic-data backup/restore demo. |
| 3 | +# Source this file in other scripts: source "$(dirname "$0")/00-helpers.sh" |
| 4 | + |
| 5 | +export RED='\033[0;31m' |
| 6 | +export GREEN='\033[0;32m' |
| 7 | +export YELLOW='\033[1;33m' |
| 8 | +export BLUE='\033[0;34m' |
| 9 | +export MAGENTA='\033[0;35m' |
| 10 | +export CYAN='\033[0;36m' |
| 11 | +export WHITE='\033[1;37m' |
| 12 | +export NC='\033[0m' |
| 13 | +export BOLD='\033[1m' |
| 14 | + |
| 15 | +# Default settings (override via environment). |
| 16 | +export NAMESPACE="${NAMESPACE:-tenant-test}" |
| 17 | +export KAFKA_NAME="${KAFKA_NAME:-kafka-test}" |
| 18 | +export KAFKA_RESTORE_NAME="${KAFKA_RESTORE_NAME:-kafka-restore}" |
| 19 | +export TOPIC="${TOPIC:-orders}" |
| 20 | +export PARTITIONS="${PARTITIONS:-3}" |
| 21 | +export MESSAGE_COUNT="${MESSAGE_COUNT:-30}" |
| 22 | +export BUCKET_NAME="${BUCKET_NAME:-kafka-backups}" |
| 23 | +export BACKUPCLASS_NAME="${BACKUPCLASS_NAME:-kafka-backup}" |
| 24 | +export STRATEGY_NAME="${STRATEGY_NAME:-kafka-job}" |
| 25 | +export BACKUPJOB_NAME="${BACKUPJOB_NAME:-kafka-backup-job}" |
| 26 | +export RESTOREJOB_INPLACE_NAME="${RESTOREJOB_INPLACE_NAME:-kafka-restore-inplace}" |
| 27 | +export RESTOREJOB_TOCOPY_NAME="${RESTOREJOB_TOCOPY_NAME:-kafka-restore-to-copy}" |
| 28 | +# The Strimzi Kafka image carries the full kafka-*.sh CLI plus bash, curl and |
| 29 | +# tar - everything the generic Job strategy and these host-side helpers need, |
| 30 | +# with no purpose-built backup image. Override KAFKA_IMAGE to match your |
| 31 | +# operator's Kafka image if it differs: |
| 32 | +# kubectl -n cozy-kafka-operator get deploy strimzi-cluster-operator \ |
| 33 | +# -o jsonpath='{range .spec.template.spec.containers[0].env[*]}{.name}={.value}{"\n"}{end}' | grep KAFKA_IMAGES |
| 34 | +export KAFKA_IMAGE="${KAFKA_IMAGE:-quay.io/strimzi/kafka:0.45.0-kafka-3.8.0}" |
| 35 | +export KAFKA_BIN="${KAFKA_BIN:-/opt/kafka/bin}" |
| 36 | + |
| 37 | +log_info() { echo -e "${BLUE}i${NC} $*" >&2; } |
| 38 | +log_success() { echo -e "${GREEN}OK${NC} $*" >&2; } |
| 39 | +log_warning() { echo -e "${YELLOW}!${NC} $*" >&2; } |
| 40 | +log_error() { echo -e "${RED}x${NC} $*" >&2; } |
| 41 | +log_step() { echo -e "\n${MAGENTA}${BOLD}> $*${NC}" >&2; } |
| 42 | +log_substep() { echo -e "${CYAN} -> $*${NC}" >&2; } |
| 43 | +log_command() { echo -e "${WHITE} $ $*${NC}" >&2; } |
| 44 | + |
| 45 | +separator() { |
| 46 | + echo -e "\n${CYAN}------------------------------------------------------------${NC}\n" >&2 |
| 47 | +} |
| 48 | + |
| 49 | +print_header() { |
| 50 | + local title="$1" |
| 51 | + echo -e "\n${MAGENTA}${BOLD}== $title ==${NC}\n" >&2 |
| 52 | +} |
| 53 | + |
| 54 | +# Wait until a JSONPath value on a resource matches the desired string. |
| 55 | +wait_for_field() { |
| 56 | + local resource_type="$1" |
| 57 | + local resource_name="$2" |
| 58 | + local jsonpath="$3" |
| 59 | + local desired="$4" |
| 60 | + local namespace="${5:-}" |
| 61 | + local timeout="${6:-300}" |
| 62 | + |
| 63 | + log_substep "Waiting for $resource_type/$resource_name $jsonpath to become '$desired'..." |
| 64 | + local elapsed=0 |
| 65 | + local ns_flag=() |
| 66 | + [[ -n "$namespace" ]] && ns_flag=(-n "$namespace") |
| 67 | + |
| 68 | + while true; do |
| 69 | + local current |
| 70 | + current=$(kubectl get "$resource_type" "$resource_name" "${ns_flag[@]}" -o jsonpath="$jsonpath" 2>/dev/null || true) |
| 71 | + if [[ "$current" == "$desired" ]]; then |
| 72 | + log_success "$resource_type/$resource_name reached '$desired'" |
| 73 | + return 0 |
| 74 | + fi |
| 75 | + if [[ $elapsed -ge $timeout ]]; then |
| 76 | + log_error "Timeout waiting for $resource_type/$resource_name (current: '$current', expected: '$desired')" |
| 77 | + return 1 |
| 78 | + fi |
| 79 | + sleep 5 |
| 80 | + elapsed=$((elapsed + 5)) |
| 81 | + done |
| 82 | +} |
| 83 | + |
| 84 | +# In-cluster bootstrap address for a Kafka application instance. The Cozystack |
| 85 | +# chart names the Strimzi Kafka cluster "kafka-<app>", and Strimzi names the |
| 86 | +# plaintext bootstrap Service "<cluster>-kafka-bootstrap" - so for an app named |
| 87 | +# <app> the Service is "kafka-<app>-kafka-bootstrap". The internal plain |
| 88 | +# listener on 9092 has no TLS or auth, so a bare host:port is all the CLI needs. |
| 89 | +kafka_bootstrap() { |
| 90 | + local app="$1" |
| 91 | + echo "kafka-${app}-kafka-bootstrap.${NAMESPACE}.svc:9092" |
| 92 | +} |
| 93 | + |
| 94 | +# Run a bash snippet in a throwaway Strimzi Kafka Pod. The snippet runs with |
| 95 | +# these variables pre-set, so it needs no nested shell quoting of its own: |
| 96 | +# $BOOT - the target app's plaintext bootstrap (host:port) |
| 97 | +# $BIN - the kafka CLI directory |
| 98 | +# $TOPIC, $PARTITIONS, $MESSAGE_COUNT - the demo knobs |
| 99 | +# This is the host-side analogue of the strategy Pod: same stock image, same |
| 100 | +# tools, no purpose-built backup container. Pass the snippet single-quoted so |
| 101 | +# its own $VAR references reach the Pod's bash unexpanded. |
| 102 | +kafka_run() { |
| 103 | + local app="$1"; shift |
| 104 | + local snippet="$1" |
| 105 | + local boot |
| 106 | + boot="$(kafka_bootstrap "$app")" |
| 107 | + kubectl -n "$NAMESPACE" run "kafka-cli-$RANDOM" \ |
| 108 | + --image="$KAFKA_IMAGE" --restart=Never --rm -i --quiet \ |
| 109 | + --command -- bash -c "set -eu |
| 110 | +BOOT=$(printf %q "$boot") |
| 111 | +BIN=$(printf %q "$KAFKA_BIN") |
| 112 | +TOPIC=$(printf %q "$TOPIC") |
| 113 | +PARTITIONS=$(printf %q "$PARTITIONS") |
| 114 | +MESSAGE_COUNT=$(printf %q "$MESSAGE_COUNT") |
| 115 | +$snippet" |
| 116 | +} |
| 117 | + |
| 118 | +# Create the demo topic (idempotent) and publish MESSAGE_COUNT keyed sentinel |
| 119 | +# messages spread across PARTITIONS. Keys ("k-<n>") make partition placement |
| 120 | +# deterministic on restore: re-producing the same key into a topic with the |
| 121 | +# same partition count lands it in the same partition via the default |
| 122 | +# (murmur2) partitioner - no per-partition produce needed for keyed records. |
| 123 | +seed_topic() { |
| 124 | + local app="$1" |
| 125 | + kafka_run "$app" ' |
| 126 | + "$BIN"/kafka-topics.sh --bootstrap-server "$BOOT" --create --if-not-exists \ |
| 127 | + --topic "$TOPIC" --partitions "$PARTITIONS" --replication-factor 1 |
| 128 | + i=1 |
| 129 | + while [ "$i" -le "$MESSAGE_COUNT" ]; do |
| 130 | + printf "k-%s\torder-%s\n" "$i" "$i" |
| 131 | + i=$((i + 1)) |
| 132 | + done | "$BIN"/kafka-console-producer.sh --bootstrap-server "$BOOT" \ |
| 133 | + --topic "$TOPIC" --property parse.key=true |
| 134 | + ' |
| 135 | +} |
| 136 | + |
| 137 | +# Total number of records currently stored in a topic, summed over partitions |
| 138 | +# as (end offset - begin offset). Prints a bare integer, or "" if the topic |
| 139 | +# does not exist / cannot be reached. Offsets, not a consumer, so it is exact |
| 140 | +# and cheap even for a compacted or truncated topic. kafka-get-offsets emits |
| 141 | +# "topic:partition:offset" lines; pure-shell parsing avoids awk quoting. |
| 142 | +topic_message_count() { |
| 143 | + local app="$1" |
| 144 | + kafka_run "$app" ' |
| 145 | + ends=$("$BIN"/kafka-get-offsets.sh --bootstrap-server "$BOOT" --topic "$TOPIC" --time -1 2>/dev/null) || exit 0 |
| 146 | + begins=$("$BIN"/kafka-get-offsets.sh --bootstrap-server "$BOOT" --topic "$TOPIC" --time -2 2>/dev/null) || exit 0 |
| 147 | + [ -n "$ends" ] || exit 0 |
| 148 | + total=0 |
| 149 | + for e in $ends; do |
| 150 | + ep=${e%:*}; ep=${ep##*:}; eo=${e##*:} |
| 151 | + for b in $begins; do |
| 152 | + bp=${b%:*}; bp=${bp##*:} |
| 153 | + if [ "$bp" = "$ep" ]; then total=$((total + eo - ${b##*:})); break; fi |
| 154 | + done |
| 155 | + done |
| 156 | + echo "$total" |
| 157 | + ' 2>/dev/null | tr -d '[:space:]' |
| 158 | +} |
| 159 | + |
| 160 | +# Create the "<app>-backup-s3" Secret the Job strategy Pod consumes, from the |
| 161 | +# bucket coordinates cached by 03-create-bucket.sh. The generic Job strategy - |
| 162 | +# unlike the app-specific drivers - has no chart support to emit this Secret, |
| 163 | +# so the tenant provides it. Called for the source app (step 04) and the |
| 164 | +# restore target (step 07). |
| 165 | +create_s3_secret() { |
| 166 | + local app="$1" |
| 167 | + local SCRIPT_DIR |
| 168 | + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" |
| 169 | + [[ -f "$SCRIPT_DIR/.bucket-info.env" ]] || { log_error "missing $SCRIPT_DIR/.bucket-info.env; run 03-create-bucket.sh first"; return 1; } |
| 170 | + # shellcheck disable=SC1091 |
| 171 | + source "$SCRIPT_DIR/.bucket-info.env" |
| 172 | + for v in S3_ACCESS_KEY S3_SECRET_KEY S3_ENDPOINT S3_REGION S3_BUCKET; do |
| 173 | + [[ -n "${!v:-}" ]] || { log_error "required variable is missing or empty: ${v}"; return 1; } |
| 174 | + done |
| 175 | + kubectl -n "$NAMESPACE" create secret generic "${app}-backup-s3" \ |
| 176 | + --from-literal=accessKey="$S3_ACCESS_KEY" \ |
| 177 | + --from-literal=secretKey="$S3_SECRET_KEY" \ |
| 178 | + --from-literal=endpoint="$S3_ENDPOINT" \ |
| 179 | + --from-literal=region="$S3_REGION" \ |
| 180 | + --from-literal=bucket="$S3_BUCKET" \ |
| 181 | + --dry-run=client -o yaml | kubectl apply -f - |
| 182 | +} |
0 commit comments