Skip to content

Commit 65c183e

Browse files
committed
fix(e2e): keep backup preflight failure evidence across attempts
The backup access preflight printed its port-forward capture only when the port-forward never came up, and truncated its mc log at the top of every attempt. On the path that fails in practice, a healthy port-forward and an S3 round trip that never completes, the output was the tail of the last attempt with nothing to say which stage it died at, which reads as a successful upload followed by a hung download. Print the port-forward capture alongside the mc log on the timeout path, append the mc log across attempts, and record which of PUT, GET, compare or DELETE each attempt reached. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin <[email protected]>
1 parent 79602e9 commit 65c183e

2 files changed

Lines changed: 126 additions & 9 deletions

File tree

‎hack/backup-access-preflight.bats‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,113 @@
130130
fi
131131
grep -q 'granted but did not pass S3 PUT/GET/DELETE within 0s' "$out"
132132
grep -q 's3-preflight: AccessDenied' "$out"
133+
grep -q 's3-preflight: attempt 1 failed at PUT' "$out"
134+
}
135+
136+
@test "preflight timeout reports the port-forward and how every attempt ended" {
137+
. hack/e2e-chainsaw/_lib/backup-access-preflight.sh
138+
out=$(mktemp)
139+
now=$(mktemp)
140+
pf_ready=$(mktemp)
141+
attempts=$(mktemp)
142+
uploaded_source=$(mktemp)
143+
printf '100\n' > "$now"
144+
printf '0\n' > "$attempts"
145+
146+
kubectl() {
147+
case "$*" in
148+
*" get secret bucket-demo-backup "*)
149+
printf '%s' '{"spec":{"secretS3":{"accessKeyID":"access","accessSecretKey":"secret"},"bucketName":"bucket-real"}}' | base64 | tr -d '\n'
150+
;;
151+
*" port-forward "*)
152+
echo 'Forwarding from 127.0.0.1:18333 -> 8333'
153+
printf 'ready\n' > "$pf_ready"
154+
while :; do command sleep 1; done
155+
;;
156+
*) return 1 ;;
157+
esac
158+
}
159+
# A stubbed clock keeps the retry budget independent of how fast the
160+
# stubs run: only a completed PUT advances it, so the loop ends after a
161+
# known number of attempts.
162+
date() {
163+
if [ "${1:-}" = +%s ]; then sed -n '1p' "$now"; else command date "$@"; fi
164+
}
165+
timeout() {
166+
shift
167+
if [ "$1" = sh ]; then
168+
waited=0
169+
while [ ! -s "$pf_ready" ] && [ "$waited" -lt 100 ]; do
170+
waited=$(( waited + 1 ))
171+
command sleep 0.1
172+
done
173+
return 0
174+
fi
175+
"$@"
176+
}
177+
sleep() { :; }
178+
# The first three attempts break at a later stage than the one before, so
179+
# the stages that follow a completed upload are told apart rather than
180+
# merged.
181+
mc() {
182+
case "$1" in
183+
alias) return 0 ;;
184+
cp)
185+
case "$3" in
186+
backup-preflight/*)
187+
case "$(sed -n '1p' "$attempts")" in
188+
1)
189+
echo 'mc: <ERROR> Unable to download: connection reset by peer' >&2
190+
return 1
191+
;;
192+
2) printf 'truncated\n' > "$4" ;;
193+
*) command cp "$(sed -n '1p' "$uploaded_source")" "$4" ;;
194+
esac
195+
;;
196+
*)
197+
printf '%s\n' "$3" > "$uploaded_source"
198+
count=$(sed -n '1p' "$attempts")
199+
printf '%s\n' "$(( count + 1 ))" > "$attempts"
200+
stamp=$(sed -n '1p' "$now")
201+
printf '%s\n' "$(( stamp + 10 ))" > "$now"
202+
# Do not tidy this arm away into a clean ascending
203+
# ladder. A ladder where every attempt fails later than
204+
# the last can never catch a missing per-iteration
205+
# stage reset, because the stale value is overwritten
206+
# before anything reads it.
207+
if [ "$count" -eq 3 ]; then
208+
echo 'mc: <ERROR> Unable to upload: connection reset by peer' >&2
209+
return 1
210+
fi
211+
echo 'source -> backup-preflight/bucket-real/object'
212+
;;
213+
esac
214+
;;
215+
rm) return 1 ;;
216+
*) return 0 ;;
217+
esac
218+
}
219+
220+
if cozy_backup_access_preflight tenant-test bucket-demo-backup 35 >"$out" 2>&1; then
221+
echo "an S3 round trip that never completed passed the preflight" >&2
222+
exit 1
223+
fi
224+
grep -q 'port-forward: Forwarding from 127.0.0.1:18333' "$out" || {
225+
echo "timeout path dropped the port-forward log" >&2
226+
cat "$out" >&2
227+
exit 1
228+
}
229+
for expected in \
230+
'attempt 1 failed at GET' \
231+
'attempt 2 failed at compare' \
232+
'attempt 3 failed at DELETE' \
233+
'attempt 4 failed at PUT'; do
234+
grep -q "s3-preflight: $expected" "$out" || {
235+
echo "timeout path lost '$expected'" >&2
236+
cat "$out" >&2
237+
exit 1
238+
}
239+
done
133240
}
134241

135242
@test "all active database backup flows gate immediately after accessGranted" {

‎hack/e2e-chainsaw/_lib/backup-access-preflight.sh‎

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ cozy_backup_access_preflight() (
2020
bucket=''
2121
deadline=''
2222
attempt=''
23+
stage=''
2324
object=''
2425

2526
case "${timeout_seconds}:${local_port}" in
@@ -102,23 +103,32 @@ cozy_backup_access_preflight() (
102103
attempt=0
103104
while :; do
104105
attempt=$(( attempt + 1 ))
105-
: > "${workdir}/mc.log"
106+
stage=PUT
106107
if timeout 15 mc cp --insecure "${workdir}/source" \
107-
"backup-preflight/${bucket}/${object}" >"${workdir}/mc.log" 2>&1; then
108+
"backup-preflight/${bucket}/${object}" >>"${workdir}/mc.log" 2>&1; then
108109
uploaded=true
110+
stage=GET
109111
if timeout 15 mc cp --insecure "backup-preflight/${bucket}/${object}" \
110-
"${workdir}/download" >>"${workdir}/mc.log" 2>&1 \
111-
&& cmp "${workdir}/source" "${workdir}/download" \
112-
&& timeout 15 mc rm --insecure "backup-preflight/${bucket}/${object}" \
113-
>>"${workdir}/mc.log" 2>&1; then
114-
uploaded=false
115-
echo "BucketAccess ${namespace}/${access_name} passed PUT, GET, compare, and DELETE on attempt ${attempt}."
116-
return 0
112+
"${workdir}/download" >>"${workdir}/mc.log" 2>&1; then
113+
stage=compare
114+
if cmp "${workdir}/source" "${workdir}/download" >>"${workdir}/mc.log" 2>&1; then
115+
stage=DELETE
116+
if timeout 15 mc rm --insecure "backup-preflight/${bucket}/${object}" \
117+
>>"${workdir}/mc.log" 2>&1; then
118+
uploaded=false
119+
echo "BucketAccess ${namespace}/${access_name} passed PUT, GET, compare, and DELETE on attempt ${attempt}."
120+
return 0
121+
fi
122+
fi
117123
fi
118124
fi
125+
# One log carries every attempt, so this line is all that tells a failed
126+
# upload from a failed download.
127+
printf 'attempt %s failed at %s\n' "${attempt}" "${stage}" >>"${workdir}/mc.log"
119128

120129
if [ "$(date +%s)" -ge "${deadline}" ]; then
121130
echo "BucketAccess ${namespace}/${access_name} was granted but did not pass S3 PUT/GET/DELETE within ${timeout_seconds}s" >&2
131+
sed 's/^/ port-forward: /' "${workdir}/port-forward.log" >&2
122132
sed 's/^/ s3-preflight: /' "${workdir}/mc.log" >&2
123133
return 1
124134
fi

0 commit comments

Comments
 (0)