Skip to content

Commit 32d7f0f

Browse files
committed
fix(backups): restore Kafka 3.x topic configs that Kafka 4 rejects
Every Kafka now runs 4.x, and a topic-metadata backup taken on Kafka 3.8 or 3.9 can record message.format.version, message.downconversion.enable or message.timestamp.difference.max.ms, which Kafka 4 removed, or a segment.bytes below 1 MiB, the minimum Kafka 4 raised it to from 14 bytes. A 4.x broker rejects each of them, so the restore stopped at the first one, leaving that topic half-configured and every later topic unrestored, and a rerun failed on the same line. The restore now skips the removed keys and raises a smaller segment.bytes to the minimum, each with a warning, and applies the rest. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin <[email protected]>
1 parent 859fb08 commit 32d7f0f

2 files changed

Lines changed: 72 additions & 1 deletion

File tree

‎hack/kafka-backup-script.bats‎

Lines changed: 53 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,19 @@ if [ "\$mode" = describe ]; then
106106
[ -n "\${CFG_FAIL:-}" ] && exit 3
107107
cat "$STATE/cfg.\$name" 2>/dev/null || true; exit 0
108108
fi
109-
if [ "\$mode" = alter ]; then echo "\$name \$add" >> "$STATE/configs_applied"; exit 0; fi
109+
if [ "\$mode" = alter ]; then
110+
# Topic configs a Kafka 4 broker no longer knows.
111+
case "\${add%%=*}" in
112+
message.downconversion.enable|message.format.version|message.timestamp.difference.max.ms)
113+
echo "InvalidConfigurationException: Unknown topic config name: \${add%%=*}" >&2; exit 1 ;;
114+
esac
115+
# Kafka 4 raised the segment.bytes floor to 1 MiB.
116+
case "\$add" in
117+
segment.bytes=*)
118+
[ "\${add#*=}" -ge 1048576 ] || { echo "InvalidConfigurationException: Invalid value \${add#*=} for configuration segment.bytes: Value must be at least 1048576" >&2; exit 1; } ;;
119+
esac
120+
echo "\$name \$add" >> "$STATE/configs_applied"; exit 0
121+
fi
110122
exit 0
111123
STUB
112124

@@ -243,6 +255,46 @@ write_backup_object() {
243255
grep -qxF "orders cleanup.policy=[compact,delete]" "$STATE/configs_applied"
244256
}
245257

258+
@test "restore of a Kafka 3.x backup skips the topic configs Kafka 4 removed and restores the rest" {
259+
init_stubs
260+
{
261+
printf 'T\torders\t3\t1\n'
262+
printf 'C\torders\tmessage.format.version\t3.0-IV1\n'
263+
printf 'C\torders\tmessage.downconversion.enable\ttrue\n'
264+
printf 'C\torders\tmessage.timestamp.difference.max.ms\t60000\n'
265+
printf 'C\torders\tretention.ms\t86400000\n'
266+
printf 'T\tpayments\t2\t1\n'
267+
printf 'C\tpayments\tcleanup.policy\tcompact\n'
268+
} > "$STATE/s3_object"
269+
: > "$STATE/topics"
270+
out=$(MODE=restore S3_ENDPOINT="https://s3.example.org" bash "$SCRIPT" 2>&1)
271+
grep -qxF "orders retention.ms=86400000" "$STATE/configs_applied"
272+
grep -qxF "create payments 2" "$STATE/actions"
273+
grep -qxF "payments cleanup.policy=compact" "$STATE/configs_applied"
274+
for k in message.format.version message.downconversion.enable message.timestamp.difference.max.ms; do
275+
printf '%s\n' "$out" | grep -qF "orders: skipping $k"
276+
done
277+
}
278+
279+
@test "restore of a Kafka 3.x backup raises segment.bytes below the Kafka 4 floor and restores the rest" {
280+
init_stubs
281+
{
282+
printf 'T\torders\t3\t1\n'
283+
printf 'C\torders\tretention.ms\t86400000\n'
284+
printf 'C\torders\tsegment.bytes\t524288\n'
285+
printf 'T\tpayments\t2\t1\n'
286+
printf 'C\tpayments\tsegment.bytes\t1048576\n'
287+
} > "$STATE/s3_object"
288+
: > "$STATE/topics"
289+
out=$(MODE=restore S3_ENDPOINT="https://s3.example.org" bash "$SCRIPT" 2>&1)
290+
grep -qxF "orders retention.ms=86400000" "$STATE/configs_applied"
291+
grep -qxF "orders segment.bytes=1048576" "$STATE/configs_applied"
292+
grep -qxF "create payments 2" "$STATE/actions"
293+
grep -qxF "payments segment.bytes=1048576" "$STATE/configs_applied"
294+
printf '%s\n' "$out" | grep -qF "orders: raising segment.bytes from 524288 to 1048576"
295+
if printf '%s\n' "$out" | grep -qF "payments: raising"; then echo "warned about a value at the floor" >&2; return 1; fi
296+
}
297+
246298
@test "restore fails loudly on a replication-factor mismatch" {
247299
init_stubs
248300
printf 'T\torders\t3\t2\n' > "$STATE/s3_object" # backup wants RF 2

‎packages/system/backupstrategy-controller/files/kafka-backup.sh‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ set -euo pipefail
55
# kafka-topics.sh / kafka-configs.sh; the Job never sets it.
66
BIN="${BIN:-/opt/kafka/bin}"
77
TAB=$(printf '\t')
8+
# Kafka 4 rejects a smaller topic segment.bytes; Kafka 3.x accepted down to 14.
9+
SEGMENT_BYTES_MIN=1048576
810
# Overridable for the same reason as BIN: hack/kafka-backup-script.bats points it
911
# at a per-test path so runs cannot read back a previous test's records. The Job
1012
# never sets it.
@@ -197,6 +199,23 @@ else
197199
;;
198200
C)
199201
t=$a; k=$b; v=$c
202+
# A backup taken on Kafka 3.x can carry topic configs Kafka 4.2/4.3
203+
# reject, which would stop the restore halfway. The same limits are in
204+
# packages/apps/kafka/templates/topics.yaml,
205+
# packages/core/platform/templates/_helpers.tpl and
206+
# packages/core/platform/images/migrations/migrations/61.
207+
case "${k}" in
208+
message.downconversion.enable|message.format.version|message.timestamp.difference.max.ms)
209+
echo " config ${t}: skipping ${k}, removed in Kafka 4" >&2
210+
continue
211+
;;
212+
segment.bytes)
213+
if [[ "${v}" =~ ^-?[0-9]+$ ]] && [ "${v}" -lt "${SEGMENT_BYTES_MIN}" ]; then
214+
echo " config ${t}: raising segment.bytes from ${v} to ${SEGMENT_BYTES_MIN}, the Kafka 4 minimum" >&2
215+
v=${SEGMENT_BYTES_MIN}
216+
fi
217+
;;
218+
esac
200219
# Bracket-group only when the value contains a comma, so kafka-configs
201220
# does not split it into bogus key=val tokens.
202221
case "${v}" in

0 commit comments

Comments
 (0)