Skip to content

Commit bbd39c9

Browse files
themoriartithemoriarti
committed
feat(kubernetes-nodes): build the Proxmox worker template from the pool's Talos values
A proxmox pool clones its workers from a VM template the chart never sees, and nothing said which Talos image that template has to carry. The one an operator reaches for, the platform release's nocloud-amd64.raw.xz, is the bare-metal node image: its zfs extension's service waits for a /dev/zfs no VM has, the boot sequence never completes, and Talos reboots every worker each time the boot timeout runs out, about every 70 minutes on v1.13. It also follows the host nodes' Talos version, not the pool's talos.version that the installer in the worker machineconfig names. The chart's default schematic has a trap of its own: its qemu-guest-agent service waits the same way for the agent's virtio port, which Proxmox adds only to a VM with the agent enabled. hack/proxmox-talos-template.sh builds the template from the chart's talos.schematicID and talos.version in the nocloud flavour with the agent enabled, refuses a tag set another template already carries, because capmox then finds two and clones neither, and prints the templateTags that select it. Assisted-by: LLM Signed-off-by: Marian Koreniuk <[email protected]>
1 parent 08ee470 commit bbd39c9

8 files changed

Lines changed: 344 additions & 20 deletions

File tree

‎api/apps/v1alpha1/kubernetesnodes/types.go‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎hack/proxmox-talos-template.sh‎

Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
#!/bin/sh
2+
# Builds the Proxmox VM template a `substrate: proxmox` kubernetes-nodes pool
3+
# clones its workers from.
4+
#
5+
# The chart takes no image on that substrate: capmox clones whichever template
6+
# carries the pool's proxmox.templateTags, and the pool's talos.* values reach
7+
# only the installer written into the worker machineconfig. Nothing compares the
8+
# two, so the template is built here from the same talos.version and
9+
# talos.schematicID the chart renders, read from its values.yaml.
10+
#
11+
# Two images that look right are not:
12+
# - the platform release's nocloud-amd64.raw.xz is the bare-metal node image.
13+
# Its zfs extension starts a service that waits for /dev/zfs, which a VM never
14+
# has, so the Talos boot sequence never completes and the guest is rebooted
15+
# when the boot timeout runs out (about 70 minutes on v1.13), over and over.
16+
# - the openstack image the kubevirt substrate imports ignores the nocloud ISO
17+
# capmox hands the bootstrap data over on, and the worker never joins.
18+
#
19+
# The chart's default schematic carries qemu-guest-agent, whose service waits
20+
# the same way for the agent's virtio port. Proxmox adds that port only to a VM
21+
# with the agent enabled, so the template is created with it on.
22+
#
23+
# Usage:
24+
# hack/proxmox-talos-template.sh print <vmid> <storage>
25+
# hack/proxmox-talos-template.sh create <vmid> <storage>
26+
#
27+
# print writes the program that create runs, for review. create runs it over
28+
# ssh on COZY_PVE_SSH (user@host), or on this machine when that is unset, which
29+
# then has to be a Proxmox VE node. The template is created on that node.
30+
#
31+
# Optional overrides:
32+
# COZY_TALOS_VERSION, COZY_TALOS_SCHEMATIC_ID, COZY_TALOS_FACTORY_URL
33+
# default to talos.version, talos.schematicID and talos.imageFactoryURL of
34+
# packages/apps/kubernetes-nodes/values.yaml. A pool that overrides those
35+
# needs a template built with the same values.
36+
# COZY_TEMPLATE_TAGS comma-separated, default
37+
# cozystack,talos,talos-<version>,schematic-<8 chars>
38+
# COZY_TEMPLATE_DISK_SIZE default 20G. capmox clones the disk at the size the
39+
# template has, so this is every worker's system disk.
40+
# COZY_TEMPLATE_CPU default x86-64-v2-AES. Talos needs x86-64-v2, which
41+
# the Proxmox default kvm64 does not provide.
42+
# COZY_PVE_SSH_KEY default ~/.ssh/id_ed25519
43+
set -eu
44+
45+
VALUES="$(cd "$(dirname "$0")/.." && pwd)/packages/apps/kubernetes-nodes/values.yaml"
46+
47+
die() { printf 'proxmox-talos-template: %s\n' "$*" >&2; exit 1; }
48+
49+
usage() {
50+
die "usage: $0 print|create <vmid> <storage>"
51+
}
52+
53+
# The talos block of values.yaml is flat, two-space indented and quoted, so awk
54+
# reads it without making yq a requirement of an operator's workstation.
55+
chart_talos() {
56+
awk -v key="$1" '
57+
/^talos:/ { in_block = 1; next }
58+
in_block && /^[^ #]/ { exit }
59+
in_block && $1 == key ":" { v = $2; gsub(/"/, "", v); print v; exit }
60+
' "$VALUES"
61+
}
62+
63+
matches() { printf '%s' "$1" | grep -Eqx "$2"; }
64+
65+
[ $# -eq 3 ] || usage
66+
MODE=$1
67+
VMID=$2
68+
STORAGE=$3
69+
case "$MODE" in print|create) ;; *) usage ;; esac
70+
71+
VERSION=${COZY_TALOS_VERSION:-$(chart_talos version)}
72+
SCHEMATIC=${COZY_TALOS_SCHEMATIC_ID:-$(chart_talos schematicID)}
73+
FACTORY=${COZY_TALOS_FACTORY_URL:-$(chart_talos imageFactoryURL)}
74+
FACTORY=${FACTORY%/}
75+
DISK_SIZE=${COZY_TEMPLATE_DISK_SIZE:-20G}
76+
CPU=${COZY_TEMPLATE_CPU:-x86-64-v2-AES}
77+
TAGS=${COZY_TEMPLATE_TAGS:-cozystack,talos,talos-$VERSION,schematic-$(printf '%s' "$SCHEMATIC" | cut -c1-8)}
78+
79+
# Every value below is pasted into a program that runs as root on the
80+
# hypervisor, so each is held to the shape it is expected to have.
81+
matches "$VMID" '[1-9][0-9]{2,8}' || die "vmid $VMID is not a Proxmox VM id (100-999999999)"
82+
matches "$STORAGE" '[A-Za-z][A-Za-z0-9_.-]*' || die "storage $STORAGE is not a Proxmox storage id"
83+
matches "$VERSION" 'v[0-9]+\.[0-9]+\.[0-9]+' || die "Talos version '$VERSION' is not vMAJOR.MINOR.PATCH"
84+
matches "$SCHEMATIC" '[0-9a-f]{64}' || die "schematic '$SCHEMATIC' is not an Image Factory schematic id"
85+
matches "$FACTORY" 'https?://[A-Za-z0-9.-]+(:[0-9]+)?(/[A-Za-z0-9._~/-]*)?' || die "factory URL '$FACTORY' is not a plain http(s) URL"
86+
matches "$DISK_SIZE" '[1-9][0-9]*G' || die "disk size $DISK_SIZE is not a whole number of gigabytes such as 20G"
87+
matches "$CPU" '[A-Za-z0-9_.+-]+' || die "cpu type $CPU is not a Proxmox CPU model name"
88+
89+
# capmox lowercases the selector and compares it with the template's tags as a
90+
# set, so the tags are normalised the same way before they are checked and
91+
# written, and the selector printed at the end is exactly what will match.
92+
TAG_SET=$(printf '%s\n' "$TAGS" | tr ',' '\n' | tr '[:upper:]' '[:lower:]' | sed '/^$/d' | LC_ALL=C sort -u)
93+
[ -n "$TAG_SET" ] || die "the template needs at least one tag"
94+
bad=$(printf '%s\n' "$TAG_SET" | grep -Evx '[a-z0-9_][a-z0-9_+.-]*' || true)
95+
[ -z "$bad" ] || die "not a valid Proxmox tag: $(printf '%s' "$bad" | paste -sd ' ' -)"
96+
PVE_TAGS=$(printf '%s\n' "$TAG_SET" | paste -sd ';' -)
97+
98+
URL="$FACTORY/image/$SCHEMATIC/$VERSION/nocloud-amd64.raw.xz"
99+
NAME="talos-$VERSION-$(printf '%s' "$SCHEMATIC" | cut -c1-8)"
100+
101+
program() {
102+
cat <<EOF
103+
set -eu
104+
VMID=$VMID
105+
STORAGE=$STORAGE
106+
URL='$URL'
107+
NAME='$NAME'
108+
TAGS='$PVE_TAGS'
109+
DISK_SIZE=$DISK_SIZE
110+
CPU='$CPU'
111+
EOF
112+
cat <<'EOF'
113+
if ! command -v qm >/dev/null || ! command -v pvesh >/dev/null; then
114+
echo "this is not a Proxmox VE node: qm and pvesh are missing" >&2
115+
exit 1
116+
fi
117+
118+
# nextid answers with the id only when no VM in the cluster holds it.
119+
pvesh get /cluster/nextid --vmid "$VMID" >/dev/null
120+
121+
# capmox requires exactly one template with the selector's tag set across the
122+
# whole cluster and reports "found 2 VM templates" otherwise, so a second
123+
# template with the same set would stop every pool that selects it.
124+
same=$(pvesh get /cluster/resources --type vm --output-format json | perl -MJSON::PP -e '
125+
my $want = shift;
126+
local $/;
127+
for my $vm (@{ JSON::PP->new->decode(<STDIN>) }) {
128+
next unless $vm->{template} && defined $vm->{tags};
129+
my %t = map { lc($_) => 1 } grep { length } split /;/, $vm->{tags};
130+
print "$vm->{vmid} ($vm->{name}) on $vm->{node}\n" if join(";", sort keys %t) eq $want;
131+
}' "$TAGS")
132+
if [ -n "$same" ]; then
133+
echo "a template already carries the tags $TAGS: $same" >&2
134+
exit 1
135+
fi
136+
137+
work=$(mktemp -d "${TMPDIR:-/var/tmp}/talos-template.XXXXXX")
138+
trap 'rm -rf "$work"' EXIT
139+
curl -fL --retry 3 -o "$work/disk.raw.xz" "$URL"
140+
xz -d "$work/disk.raw.xz"
141+
142+
# The layout of the templates the Proxmox substrate was verified on. No NIC:
143+
# capmox writes net0 from the pool's proxmox.network on every clone.
144+
qm create "$VMID" --name "$NAME" --tags "$TAGS" --ostype l26 \
145+
--cpu "$CPU" --sockets 1 --cores 2 --memory 4096 \
146+
--bios ovmf --efidisk0 "$STORAGE:1,efitype=4m,pre-enrolled-keys=0" \
147+
--scsihw virtio-scsi-pci \
148+
--scsi0 "$STORAGE:0,import-from=$work/disk.raw,cache=none,discard=on,ssd=1" \
149+
--boot order=scsi0 --serial0 socket --vga std \
150+
--agent enabled=1
151+
qm disk resize "$VMID" scsi0 "$DISK_SIZE"
152+
qm template "$VMID"
153+
qm config "$VMID"
154+
EOF
155+
}
156+
157+
if [ "$MODE" = print ]; then
158+
program
159+
exit 0
160+
fi
161+
162+
if [ -n "${COZY_PVE_SSH:-}" ]; then
163+
program | ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new \
164+
-i "${COZY_PVE_SSH_KEY:-$HOME/.ssh/id_ed25519}" "$COZY_PVE_SSH" sh -s
165+
else
166+
program | sh -s
167+
fi
168+
169+
cat <<EOF
170+
171+
Template $VMID ($NAME) is ready. A pool clones it with:
172+
173+
talos:
174+
version: "$VERSION"
175+
schematicID: "$SCHEMATIC"
176+
proxmox:
177+
templateTags:
178+
EOF
179+
for tag in $TAG_SET; do
180+
printf ' - "%s"\n' "$tag"
181+
done

‎hack/proxmox-talos-template_test.bats‎

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
#!/usr/bin/env bats
2+
# hack/proxmox-talos-template.sh builds the VM template a proxmox kubernetes-nodes
3+
# pool clones. Nothing at render time can see what the template carries, so what
4+
# the script writes is pinned here: the image is the chart's own talos.* in its
5+
# nocloud flavour, the agent the default schematic waits for is enabled, and the
6+
# tag set capmox selects on is checked before anything is downloaded.
7+
# (hack/cozytest.sh runs these as POSIX sh: no `run`, no [[ ]].)
8+
9+
REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)"
10+
SCRIPT="$REPO_ROOT/hack/proxmox-talos-template.sh"
11+
VALUES="$REPO_ROOT/packages/apps/kubernetes-nodes/values.yaml"
12+
13+
chart() { sed -n "/^talos:/,/^[^ #]/s/^ $1: \"\(.*\)\"$/\1/p" "$VALUES"; }
14+
15+
# Stubs for the hypervisor side of the program. Each call to qm and curl is
16+
# logged, so a test can tell what ran and, as importantly, what did not.
17+
stub_pve() {
18+
stub="$1"
19+
printf '#!/bin/sh\necho "qm $*" >> "%s/calls"\n' "$stub" > "$stub/qm"
20+
cat > "$stub/pvesh" <<EOF
21+
#!/bin/sh
22+
case "\$2" in
23+
/cluster/nextid) echo "\$4" ;;
24+
/cluster/resources) cat "$stub/resources.json" ;;
25+
esac
26+
EOF
27+
cat > "$stub/curl" <<EOF
28+
#!/bin/sh
29+
echo "curl \$*" >> "$stub/calls"
30+
while [ \$# -gt 0 ]; do [ "\$1" = -o ] && : > "\$2"; shift; done
31+
EOF
32+
printf '#!/bin/sh\nmv "$2" "${2%%.xz}"\n' > "$stub/xz"
33+
chmod +x "$stub/qm" "$stub/pvesh" "$stub/curl" "$stub/xz"
34+
: > "$stub/calls"
35+
}
36+
37+
@test "the template image is the chart's talos schematic and version, in the nocloud flavour" {
38+
out="$("$SCRIPT" print 9002 main-pool)"
39+
url="$(chart imageFactoryURL)/image/$(chart schematicID)/$(chart version)/nocloud-amd64.raw.xz"
40+
printf '%s' "$out" | grep -qxF "URL='$url'"
41+
! printf '%s' "$out" | grep -q openstack
42+
}
43+
44+
@test "the template enables the guest agent whose port the default schematic waits for" {
45+
"$SCRIPT" print 9002 main-pool | grep -qE -- '--agent enabled=1$'
46+
}
47+
48+
@test "the default tags name the version and the schematic, in the order Proxmox stores them" {
49+
out="$("$SCRIPT" print 9002 main-pool)"
50+
want="TAGS='cozystack;schematic-$(chart schematicID | cut -c1-8);talos;talos-$(chart version)'"
51+
printf '%s' "$out" | grep -qxF "$want"
52+
}
53+
54+
@test "given tags are lowercased, deduplicated and sorted the way capmox compares them" {
55+
out="$(COZY_TEMPLATE_TAGS='Talos,cozystack,,talos,Pool-A' "$SCRIPT" print 9002 main-pool)"
56+
printf '%s' "$out" | grep -qxF "TAGS='cozystack;pool-a;talos'"
57+
}
58+
59+
@test "an override of the talos values reaches the image URL" {
60+
sid=376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba
61+
out="$(COZY_TALOS_VERSION=v1.14.2 COZY_TALOS_SCHEMATIC_ID=$sid COZY_TALOS_FACTORY_URL=https://factory.example:8080/ "$SCRIPT" print 9002 main-pool)"
62+
printf '%s' "$out" | grep -qxF "URL='https://factory.example:8080/image/$sid/v1.14.2/nocloud-amd64.raw.xz'"
63+
}
64+
65+
@test "values that would land in the root program unchecked are refused" {
66+
for pair in "9002;id|main-pool" "99|main-pool" "9002|main pool" "9002|main'pool"; do
67+
vmid="${pair%%|*}"
68+
storage="${pair#*|}"
69+
rc=0
70+
out="$("$SCRIPT" print "$vmid" "$storage" 2>&1)" || rc=$?
71+
[ "$rc" -ne 0 ] || { echo "accepted: $pair" >&2; exit 1; }
72+
case "$out" in *"set -eu"*) echo "printed a program for: $pair" >&2; exit 1 ;; esac
73+
done
74+
rc=0; COZY_TALOS_VERSION='v1.13.6;id' "$SCRIPT" print 9002 main-pool >/dev/null 2>&1 || rc=$?
75+
[ "$rc" -ne 0 ]
76+
rc=0; COZY_TALOS_SCHEMATIC_ID='ce4c9805' "$SCRIPT" print 9002 main-pool >/dev/null 2>&1 || rc=$?
77+
[ "$rc" -ne 0 ]
78+
rc=0; COZY_TEMPLATE_TAGS="talos,it's" "$SCRIPT" print 9002 main-pool >/dev/null 2>&1 || rc=$?
79+
[ "$rc" -ne 0 ]
80+
}
81+
82+
@test "a template already carrying the same tag set stops the program before the download" {
83+
stub="$(mktemp -d)"
84+
stub_pve "$stub"
85+
printf '[{"vmid":9000,"name":"old","node":"pve1","template":1,"tags":"talos-v1.13.6;Talos;schematic-%s;cozystack"}]' \
86+
"$(chart schematicID | cut -c1-8)" > "$stub/resources.json"
87+
rc=0
88+
out="$("$SCRIPT" print 9002 main-pool | PATH="$stub:$PATH" TMPDIR="$stub" sh 2>&1)" || rc=$?
89+
calls="$(cat "$stub/calls")"
90+
rm -rf "$stub"
91+
[ "$rc" -ne 0 ]
92+
case "$out" in *"already carries"*"9000 (old) on pve1"*) ;; *) echo "$out" >&2; exit 1 ;; esac
93+
[ -z "$calls" ] || { echo "$calls" >&2; exit 1; }
94+
}
95+
96+
@test "a template whose tags are a superset does not count, and the template is built" {
97+
stub="$(mktemp -d)"
98+
stub_pve "$stub"
99+
printf '[{"vmid":9000,"name":"old","node":"pve1","template":1,"tags":"cozystack;schematic-%s;talos;talos-v1.13.6;zfs"},{"vmid":101,"name":"vm","node":"pve1","template":0}]' \
100+
"$(chart schematicID | cut -c1-8)" > "$stub/resources.json"
101+
"$SCRIPT" print 9002 main-pool | PATH="$stub:$PATH" TMPDIR="$stub" sh >/dev/null
102+
calls="$(cat "$stub/calls")"
103+
rm -rf "$stub"
104+
printf '%s\n' "$calls" | grep -q '^curl .*/nocloud-amd64.raw.xz$'
105+
printf '%s\n' "$calls" | grep -qE '^qm create 9002 .*--scsi0 main-pool:0,import-from=[^ ]*/disk.raw,'
106+
printf '%s\n' "$calls" | grep -qx 'qm disk resize 9002 scsi0 20G'
107+
printf '%s\n' "$calls" | grep -qx 'qm template 9002'
108+
}
109+
110+
@test "create runs the printed program on the hypervisor over ssh and prints the pool's selector" {
111+
stub="$(mktemp -d)"
112+
printf '#!/bin/sh\necho "$*" > "%s/args"\ncat > "%s/stdin"\n' "$stub" "$stub" > "$stub/ssh"
113+
chmod +x "$stub/ssh"
114+
out="$(PATH="$stub:$PATH" COZY_PVE_SSH=root@pve "$SCRIPT" create 9002 main-pool)"
115+
args="$(cat "$stub/args")"
116+
sent="$(cat "$stub/stdin")"
117+
rm -rf "$stub"
118+
case "$args" in *"root@pve sh -s") ;; *) echo "$args" >&2; exit 1 ;; esac
119+
[ "$sent" = "$("$SCRIPT" print 9002 main-pool)" ]
120+
printf '%s' "$out" | grep -qxF ' - "talos-'"$(chart version)"'"'
121+
printf '%s' "$out" | grep -qxF ' schematicID: "'"$(chart schematicID)"'"'
122+
}
123+
124+
@test "create fails when the hypervisor program fails, and prints no selector" {
125+
stub="$(mktemp -d)"
126+
printf '#!/bin/sh\ncat >/dev/null\necho "ssh: connect to host pve: Connection refused" >&2\nexit 255\n' > "$stub/ssh"
127+
chmod +x "$stub/ssh"
128+
rc=0
129+
out="$(PATH="$stub:$PATH" COZY_PVE_SSH=root@pve "$SCRIPT" create 9002 main-pool 2>&1)" || rc=$?
130+
rm -rf "$stub"
131+
[ "$rc" -ne 0 ]
132+
case "$out" in *templateTags*) echo "$out" >&2; exit 1 ;; esac
133+
}

0 commit comments

Comments
 (0)