Skip to content

Commit 274ae9f

Browse files
committed
fix(clickhouse): address version-generator robustness notes
Follow-up on review notes for hack/update-versions.sh (all in the maintainer-run generator; the shipped chart is unchanged and files/versions.yaml / values.yaml regenerate byte-identically): - Refuse to change the default silently: if the current default's major is dropped from CH_SUPPORTED_MAJORS the generator now errors instead of promoting the newest major to default. - Freeze follows the default: the default major is pinned to the tag it already ships (read from the committed versions.yaml) rather than a hardcoded v24.9, so moving the default cannot leave it unpinned and patch-bumped on the next regeneration. - Bound the values.yaml splice: the version section ends at the first line that is not a recognised member, so a malformed input (enum header without a version: line) no longer swallows the rest of the file. Extends hack/clickhouse-update-versions_test.bats to cover all three. Signed-off-by: Alexey Artamonov <[email protected]>
1 parent 69cce57 commit 274ae9f

2 files changed

Lines changed: 112 additions & 58 deletions

File tree

‎hack/clickhouse-update-versions_test.bats‎

Lines changed: 70 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -10,37 +10,40 @@
1010
# 1. The server/keeper tag intersection must be computed with a byte
1111
# (lexicographic) collation, not a version collation. `comm` walks its
1212
# inputs assuming byte order; feeding it `sort -V` output silently drops
13-
# common tags at every X.9 -> X.10 boundary, so a whole major line can
14-
# vanish from the enum and existing tenant CRs pinned to it then fail
15-
# schema validation.
16-
# 2. The default line is patch-pinned and must be reproduced verbatim, so a
17-
# regeneration never bumps the image of existing default-version installs
18-
# (the "renders byte-for-byte identically" guarantee of the PR).
19-
# 3. A configured major with no tag common to both images is a hard error,
20-
# and the generator is atomic: on that error the committed files are left
21-
# untouched rather than half-rewritten.
13+
# common tags at every X.9 -> X.10 boundary.
14+
# 2. The default line is frozen to the exact tag it already ships (read from
15+
# the committed versions.yaml) so a regeneration never bumps the image of
16+
# existing default-version installs. The freeze follows whichever major is
17+
# the default, not a hardcoded one.
18+
# 3. Dropping the current default's major from the supported set is a hard
19+
# error, not a silent promotion of the newest major to default.
20+
# 4. A configured major with no tag common to both images is a hard error,
21+
# and the generator is atomic: on any error the committed files are left
22+
# untouched.
23+
# 5. The values.yaml splice is bounded: a malformed section (enum header
24+
# present, `version:` line absent) does not swallow the rest of the file.
2225
#
2326
# The generator is exercised offline: CH_SERVER_TAGS_FILE / CH_KEEPER_TAGS_FILE
2427
# inject the tag lists (no curl), and VALUES_FILE / VERSIONS_FILE stay in a
25-
# scratch dir so the real chart is never touched. Written in the portable
26-
# subset hack/cozytest.sh understands (no `run`/`$status`/`$lines`, no
27-
# setup/teardown, no standalone `}` inside a test body): each assertion exits
28-
# non-zero on failure and the body relies on `set -e`.
28+
# scratch dir. Written in the portable subset hack/cozytest.sh understands (no
29+
# `run`/`$status`/`$lines`, no setup/teardown, no standalone `}` inside a test
30+
# body): each assertion exits non-zero on failure and the body relies on
31+
# `set -e`.
2932

3033
REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)"
3134
GEN="$REPO_ROOT/packages/apps/clickhouse/hack/update-versions.sh"
3235

33-
# A minimal values.yaml carrying the version section the generator rewrites,
34-
# plus the section boundary it inserts before when the section is absent.
36+
# Minimal values.yaml with the version section the generator rewrites. $2 is
37+
# the default version written into both the enum and the `version:` line.
3538
seed_values() {
36-
cat > "$1" <<'EOF'
39+
cat > "$1" <<EOF
3740
## @param {string} storageClass - StorageClass used to store the data.
3841
storageClass: ""
3942
4043
## @enum {string} Version
41-
## @value v24.9
44+
## @value $2
4245
## @param {Version} version - ClickHouse major.minor version to deploy.
43-
version: v24.9
46+
version: $2
4447
4548
##
4649
## @section Application-specific parameters
@@ -50,7 +53,6 @@ logTTL: 15
5053
EOF
5154
}
5255

53-
# Per-test scratch state (no setup() — cozytest.sh does not call it).
5456
init_case() {
5557
WORK="$(mktemp -d)"
5658
SERVER="$WORK/server.txt"
@@ -59,30 +61,54 @@ init_case() {
5961
export VALUES_FILE="$WORK/values.yaml"
6062
export CH_SERVER_TAGS_FILE="$SERVER"
6163
export CH_KEEPER_TAGS_FILE="$KEEPER"
62-
seed_values "$VALUES_FILE"
6364
: > "$VERSIONS_FILE"
6465
}
6566

6667
@test "intersection keeps tags across the X.9 -> X.10 boundary" {
6768
init_case
69+
seed_values "$VALUES_FILE" v24.9
6870
printf '%s\n' 24.9.2.42 24.9.3.128 24.10.1.1 > "$SERVER"
6971
cp "$SERVER" "$KEEPER"
7072
CH_SUPPORTED_MAJORS="24.10 24.9" bash "$GEN"
7173
grep -q '^"v24.10": "24.10.1.1"$' "$VERSIONS_FILE" || { echo "24.10 dropped from intersection" >&2; exit 1; }
7274
grep -q '^"v24.9":' "$VERSIONS_FILE" || { echo "24.9 dropped from intersection" >&2; exit 1; }
7375
}
7476

75-
@test "the pinned default line is reproduced, not bumped to the latest patch" {
77+
@test "the default is frozen to its committed tag, not bumped to the latest patch" {
7678
init_case
79+
seed_values "$VALUES_FILE" v24.9
80+
printf '%s\n' '"v24.9": "24.9.2.42"' > "$VERSIONS_FILE"
7781
printf '%s\n' 24.9.2.42 24.9.3.128 > "$SERVER"
7882
cp "$SERVER" "$KEEPER"
7983
CH_SUPPORTED_MAJORS="24.9" bash "$GEN"
80-
grep -q '^"v24.9": "24.9.2.42"$' "$VERSIONS_FILE" || { echo "default not pinned to 24.9.2.42" >&2; exit 1; }
84+
grep -q '^"v24.9": "24.9.2.42"$' "$VERSIONS_FILE" || { echo "default not frozen to 24.9.2.42" >&2; exit 1; }
8185
if grep -q '24.9.3.128' "$VERSIONS_FILE"; then echo "default bumped to newer patch" >&2; exit 1; fi
8286
}
8387

88+
@test "the freeze follows the default major, not a hardcoded one" {
89+
init_case
90+
seed_values "$VALUES_FILE" v25.3
91+
printf '%s\n' '"v25.3": "25.3.14.14"' '"v24.9": "24.9.2.42"' > "$VERSIONS_FILE"
92+
printf '%s\n' 24.9.2.42 24.9.3.128 25.3.14.14 25.3.99.9 25.8.28.1 > "$SERVER"
93+
cp "$SERVER" "$KEEPER"
94+
CH_SUPPORTED_MAJORS="25.8 25.3 24.9" bash "$GEN"
95+
# v25.3 is the default -> frozen to its committed 25.3.14.14 despite 25.3.99.9
96+
grep -q '^"v25.3": "25.3.14.14"$' "$VERSIONS_FILE" || { echo "default v25.3 not frozen" >&2; exit 1; }
97+
# v24.9 is no longer the default -> free to move to its latest common patch
98+
grep -q '^"v24.9": "24.9.3.128"$' "$VERSIONS_FILE" || { echo "non-default v24.9 not on latest" >&2; exit 1; }
99+
}
100+
101+
@test "dropping the current default's major from the supported set is a hard error" {
102+
init_case
103+
seed_values "$VALUES_FILE" v24.9
104+
printf '%s\n' 25.3.14.14 25.8.28.1 > "$SERVER"
105+
cp "$SERVER" "$KEEPER"
106+
if CH_SUPPORTED_MAJORS="25.8 25.3" bash "$GEN" >/dev/null 2>&1; then echo "expected hard error on dropped default" >&2; exit 1; fi
107+
}
108+
84109
@test "non-default majors resolve to their latest common patch" {
85110
init_case
111+
seed_values "$VALUES_FILE" v25.8
86112
printf '%s\n' 25.3.14.14 25.3.15.1 25.8.28.1 > "$SERVER"
87113
cp "$SERVER" "$KEEPER"
88114
CH_SUPPORTED_MAJORS="25.8 25.3" bash "$GEN"
@@ -92,6 +118,7 @@ init_case() {
92118

93119
@test "a tag present in only one image is not selected" {
94120
init_case
121+
seed_values "$VALUES_FILE" v25.8
95122
printf '%s\n' 25.8.28.1 25.8.29.9 > "$SERVER"
96123
printf '%s\n' 25.8.28.1 > "$KEEPER"
97124
CH_SUPPORTED_MAJORS="25.8" bash "$GEN"
@@ -100,24 +127,29 @@ init_case() {
100127

101128
@test "an unresolvable configured major is a hard error" {
102129
init_case
130+
seed_values "$VALUES_FILE" v24.9
103131
printf '%s\n' 24.9.2.42 > "$SERVER"
104132
cp "$SERVER" "$KEEPER"
105133
if CH_SUPPORTED_MAJORS="26.99 24.9" bash "$GEN" >/dev/null 2>&1; then echo "expected non-zero exit" >&2; exit 1; fi
106134
}
107135

108136
@test "on error the committed files are left untouched (atomic)" {
109137
init_case
138+
seed_values "$VALUES_FILE" v24.9
110139
printf '%s\n' 24.9.2.42 > "$SERVER"
111140
cp "$SERVER" "$KEEPER"
141+
printf '%s\n' '"v24.9": "24.9.2.42"' > "$VERSIONS_FILE"
112142
cp "$VERSIONS_FILE" "$WORK/versions.before"
113143
cp "$VALUES_FILE" "$WORK/values.before"
114144
if CH_SUPPORTED_MAJORS="26.99" bash "$GEN" >/dev/null 2>&1; then echo "expected non-zero exit" >&2; exit 1; fi
115145
cmp -s "$VERSIONS_FILE" "$WORK/versions.before" || { echo "versions.yaml changed on failure" >&2; exit 1; }
116146
cmp -s "$VALUES_FILE" "$WORK/values.before" || { echo "values.yaml changed on failure" >&2; exit 1; }
117147
}
118148

119-
@test "values.yaml enum is rewritten newest-first with the pinned default" {
149+
@test "values.yaml enum is rewritten newest-first with the default preserved" {
120150
init_case
151+
seed_values "$VALUES_FILE" v24.9
152+
printf '%s\n' '"v24.9": "24.9.2.42"' > "$VERSIONS_FILE"
121153
printf '%s\n' 24.9.2.42 25.3.14.14 25.8.28.1 > "$SERVER"
122154
cp "$SERVER" "$KEEPER"
123155
CH_SUPPORTED_MAJORS="25.8 25.3 24.9" bash "$GEN"
@@ -127,3 +159,18 @@ init_case() {
127159
grep -q '^## @section Application-specific parameters$' "$VALUES_FILE" || { echo "section boundary lost" >&2; exit 1; }
128160
grep -q '^logTTL: 15$' "$VALUES_FILE" || { echo "trailing content lost" >&2; exit 1; }
129161
}
162+
163+
@test "a malformed version section does not swallow the rest of the file" {
164+
init_case
165+
cat > "$VALUES_FILE" <<'EOF'
166+
## @enum {string} Version
167+
## @value v99.9
168+
## @section Application-specific parameters
169+
SENTINEL_KEEP: yes
170+
EOF
171+
printf '%s\n' 25.8.28.1 > "$SERVER"
172+
cp "$SERVER" "$KEEPER"
173+
CH_SUPPORTED_MAJORS="25.8" bash "$GEN"
174+
grep -q '^SENTINEL_KEEP: yes$' "$VALUES_FILE" || { echo "content after a malformed section was swallowed" >&2; exit 1; }
175+
grep -q '^## @section Application-specific parameters$' "$VALUES_FILE" || { echo "section boundary swallowed" >&2; exit 1; }
176+
}

‎packages/apps/clickhouse/hack/update-versions.sh‎

Lines changed: 42 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -23,17 +23,6 @@ KEEPER_REPO="clickhouse/clickhouse-keeper"
2323
# Major.minor lines to publish, newest first. Overridable for tests.
2424
: "${CH_SUPPORTED_MAJORS:=25.8 25.3 24.9}"
2525

26-
# Patch pins: a major frozen at an exact patch regardless of newer tags, so a
27-
# regeneration never changes the image of existing installs on that line. The
28-
# default (v24.9) is pinned to the tag the chart shipped before the version
29-
# parameter existed, preserving byte-for-byte identical rendering.
30-
pin_for() {
31-
case "$1" in
32-
v24.9) printf '24.9.2.42' ;;
33-
*) printf '' ;;
34-
esac
35-
}
36-
3726
# Numeric sort of X.Y.Z.W tags — portable across BSD and GNU sort (no -V).
3827
version_sort() { sort -t. -k1,1n -k2,2n -k3,3n -k4,4n; }
3928

@@ -61,28 +50,51 @@ SERVER_TAGS="$(fetch_tags "$SERVER_REPO" CH_SERVER_TAGS_FILE)"
6150
KEEPER_TAGS="$(fetch_tags "$KEEPER_REPO" CH_KEEPER_TAGS_FILE)"
6251

6352
# Tags present in BOTH images. comm walks its inputs in byte order, so they
64-
# must be byte-collated (fetch_tags emits `LC_ALL=C sort -u`); a version
65-
# collation would silently drop common tags at every X.9 -> X.10 boundary.
53+
# must be byte-collated (fetch_tags emits `LC_ALL=C sort -u`, and comm itself
54+
# runs under LC_ALL=C); a version collation would silently drop common tags at
55+
# every X.9 -> X.10 boundary.
6656
COMMON_TAGS="$(LC_ALL=C comm -12 <(printf '%s\n' "$SERVER_TAGS") <(printf '%s\n' "$KEEPER_TAGS"))"
6757
[ -n "$COMMON_TAGS" ] || { echo "Error: no tag common to ${SERVER_REPO} and ${KEEPER_REPO}" >&2; exit 1; }
6858

69-
# Resolve each configured major to one tag: its pin if set (and available),
70-
# else the latest common patch. A configured major with no common tag is a
71-
# hard error raised BEFORE any file is written, so a partial Docker Hub result
72-
# never silently drops a supported version or leaves a half-updated tree.
59+
# Determine the default major from the current values.yaml. Dropping the
60+
# current default's major from CH_SUPPORTED_MAJORS would silently change the
61+
# default image, so it is a hard error rather than a fallback to the newest.
62+
# With no current default (first-time bootstrap) the newest major is used.
63+
CURRENT_DEFAULT="$(awk '/^version:[[:space:]]/ {print $2; exit}' "$VALUES_FILE" 2>/dev/null || true)"
64+
if [ -n "$CURRENT_DEFAULT" ]; then
65+
case " $CH_SUPPORTED_MAJORS " in
66+
*" ${CURRENT_DEFAULT#v} "*) DEFAULT_VERSION="$CURRENT_DEFAULT" ;;
67+
*) echo "Error: current default ${CURRENT_DEFAULT} is not in CH_SUPPORTED_MAJORS (${CH_SUPPORTED_MAJORS}); refusing to silently change the default" >&2; exit 1 ;;
68+
esac
69+
else
70+
DEFAULT_VERSION="v${CH_SUPPORTED_MAJORS%% *}"
71+
fi
72+
73+
# The default major is frozen to the exact tag it already ships, read from the
74+
# committed versions.yaml, so a regeneration never changes the default image.
75+
# The freeze follows whichever major is the default — there is no hardcoded
76+
# major. An empty result (bootstrap, or the default not yet mapped) means the
77+
# default takes the latest common patch like any other line.
78+
FROZEN_DEFAULT_TAG="$(grep -F "\"${DEFAULT_VERSION}\": " "$VERSIONS_FILE" 2>/dev/null | sed -E 's/.*: *"([^"]+)".*/\1/' | head -n1 || true)"
79+
80+
# Resolve each configured major to one tag: the default freezes to its shipped
81+
# tag (if still available), every other line takes its latest common patch. A
82+
# configured major with no common tag is a hard error raised BEFORE any file is
83+
# written, so a partial Docker Hub result never drops a supported version or
84+
# leaves a half-updated tree.
7385
MAJORS=()
7486
TAGS=()
7587
for mm in $CH_SUPPORTED_MAJORS; do
7688
key="v${mm}"
7789
mm_re="${mm//./\\.}"
7890
latest="$(printf '%s\n' "$COMMON_TAGS" | grep -E "^${mm_re}\.[0-9]+\.[0-9]+$" | version_sort | tail -n1 || true)"
79-
pin="$(pin_for "$key")"
80-
if [ -n "$pin" ]; then
81-
if printf '%s\n' "$COMMON_TAGS" | grep -qx "$pin"; then
82-
tag="$pin"
91+
if [ "$key" = "$DEFAULT_VERSION" ] && [ -n "$FROZEN_DEFAULT_TAG" ]; then
92+
if printf '%s\n' "$COMMON_TAGS" | grep -qx "$FROZEN_DEFAULT_TAG"; then
93+
tag="$FROZEN_DEFAULT_TAG"
8394
else
84-
echo "Error: pinned tag ${pin} for ${key} is not available in both images" >&2
85-
exit 1
95+
echo "Warning: frozen default tag ${FROZEN_DEFAULT_TAG} for ${key} is no longer available in both images; falling back to the latest ${mm} patch" >&2
96+
[ -n "$latest" ] || { echo "Error: no tag found for default ${mm} in both images" >&2; exit 1; }
97+
tag="$latest"
8698
fi
8799
elif [ -n "$latest" ]; then
88100
tag="$latest"
@@ -108,15 +120,6 @@ while [ "$i" -lt "${#MAJORS[@]}" ]; do
108120
i=$((i + 1))
109121
done
110122

111-
# Keep the current default if it is still supported, else fall back to newest.
112-
CURRENT_DEFAULT="$(awk '/^version:[[:space:]]/ {print $2; exit}' "$VALUES_FILE" 2>/dev/null || true)"
113-
DEFAULT_VERSION="${MAJORS[0]}"
114-
i=0
115-
while [ "$i" -lt "${#MAJORS[@]}" ]; do
116-
[ "${MAJORS[$i]}" = "$CURRENT_DEFAULT" ] && DEFAULT_VERSION="$CURRENT_DEFAULT"
117-
i=$((i + 1))
118-
done
119-
120123
# The values.yaml @enum/@param block, built in a file (avoids `awk -v` with
121124
# embedded newlines, which BSD awk rejects).
122125
{
@@ -131,8 +134,11 @@ done
131134
echo "version: ${DEFAULT_VERSION}"
132135
} > "$TMP_BLOCK"
133136

134-
# Splice the block into values.yaml with pure bash: replace an existing
135-
# version section, or insert before the Application-specific section.
137+
# Splice the block into values.yaml with pure bash: replace an existing version
138+
# section, or insert before the Application-specific section. The section is
139+
# bounded — it ends at the `version:` line, or at the first line that is not a
140+
# recognised section member — so a malformed input (enum header without a
141+
# `version:` line) does not swallow the rest of the file.
136142
replaced=0
137143
in_section=0
138144
while IFS= read -r line || [ -n "$line" ]; do
@@ -143,7 +149,8 @@ while IFS= read -r line || [ -n "$line" ]; do
143149
if [ "$in_section" -eq 1 ]; then
144150
case "$line" in
145151
version:*) in_section=0; continue ;;
146-
*) continue ;;
152+
"## @value"*|"## @param"*|"") continue ;;
153+
*) in_section=0 ;;
147154
esac
148155
fi
149156
if [ "$replaced" -eq 0 ] && [ "$line" = "## @section Application-specific parameters" ]; then

0 commit comments

Comments
 (0)