Skip to content

Commit 69c7044

Browse files
committed
fix(backups): skip topic configs Kafka 4 removed when restoring
Every Kafka now runs 4.x, and a topic-metadata backup taken on Kafka 3.8 or 3.9 can record message.format.version, message.downconversion.enable or message.timestamp.difference.max.ms. A 4.x broker rejects all three, so the restore stopped at the first one, leaving that topic half-configured and every later topic unrestored, and a rerun failed on the same line. The restore now skips those keys with a warning and applies the rest. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin <[email protected]>
1 parent abae46b commit 69c7044

2 files changed

Lines changed: 37 additions & 1 deletion

File tree

‎hack/kafka-backup-script.bats‎

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,14 @@ if [ "\$mode" = describe ]; then
106106
[ -n "\${CFG_FAIL:-}" ] && exit 3
107107
cat "$STATE/cfg.\$name" 2>/dev/null || true; exit 0
108108
fi
109-
if [ "\$mode" = alter ]; then echo "\$name \$add" >> "$STATE/configs_applied"; exit 0; fi
109+
if [ "\$mode" = alter ]; then
110+
# Topic configs a Kafka 4 broker no longer knows.
111+
case "\${add%%=*}" in
112+
message.downconversion.enable|message.format.version|message.timestamp.difference.max.ms)
113+
echo "InvalidConfigurationException: Unknown topic config name: \${add%%=*}" >&2; exit 1 ;;
114+
esac
115+
echo "\$name \$add" >> "$STATE/configs_applied"; exit 0
116+
fi
110117
exit 0
111118
STUB
112119

@@ -243,6 +250,27 @@ write_backup_object() {
243250
grep -qxF "orders cleanup.policy=[compact,delete]" "$STATE/configs_applied"
244251
}
245252

253+
@test "restore of a Kafka 3.x backup skips the topic configs Kafka 4 removed and restores the rest" {
254+
init_stubs
255+
{
256+
printf 'T\torders\t3\t1\n'
257+
printf 'C\torders\tmessage.format.version\t3.0-IV1\n'
258+
printf 'C\torders\tmessage.downconversion.enable\ttrue\n'
259+
printf 'C\torders\tmessage.timestamp.difference.max.ms\t60000\n'
260+
printf 'C\torders\tretention.ms\t86400000\n'
261+
printf 'T\tpayments\t2\t1\n'
262+
printf 'C\tpayments\tcleanup.policy\tcompact\n'
263+
} > "$STATE/s3_object"
264+
: > "$STATE/topics"
265+
out=$(MODE=restore S3_ENDPOINT="https://s3.example.org" bash "$SCRIPT" 2>&1)
266+
grep -qxF "orders retention.ms=86400000" "$STATE/configs_applied"
267+
grep -qxF "create payments 2" "$STATE/actions"
268+
grep -qxF "payments cleanup.policy=compact" "$STATE/configs_applied"
269+
for k in message.format.version message.downconversion.enable message.timestamp.difference.max.ms; do
270+
printf '%s\n' "$out" | grep -qF "orders: skipping $k"
271+
done
272+
}
273+
246274
@test "restore fails loudly on a replication-factor mismatch" {
247275
init_stubs
248276
printf 'T\torders\t3\t2\n' > "$STATE/s3_object" # backup wants RF 2

‎packages/system/backupstrategy-controller/files/kafka-backup.sh‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,14 @@ else
197197
;;
198198
C)
199199
t=$a; k=$b; v=$c
200+
# A backup taken on Kafka 3.x can carry topic configs Kafka 4 removed, and
201+
# the broker rejects them, which would stop the restore halfway.
202+
case "${k}" in
203+
message.downconversion.enable|message.format.version|message.timestamp.difference.max.ms)
204+
echo " config ${t}: skipping ${k}, removed in Kafka 4" >&2
205+
continue
206+
;;
207+
esac
200208
# Bracket-group only when the value contains a comma, so kafka-configs
201209
# does not split it into bogus key=val tokens.
202210
case "${v}" in

0 commit comments

Comments
 (0)