Skip to content

Commit d6394a3

Browse files
committed
fix(seaweedfs): grant Object Lock actions to existing S3 identities
A patched COSI driver only fixes new grants. Every identity issued earlier holds Read and Write alone and loses Object Lock access the moment SeaweedFS starts checking the dedicated actions. A hook grants each identity the dedicated actions its Read or Write grant covered, with the same bucket suffix, through the filer IAM API. It runs from the seaweedfs-system chart rather than as a platform migration so it reaches every tenant on every platform variant, and after the release is applied so the new driver is already the one granting. It runs on install as well as upgrade: Flux remediates a failed install by reinstalling, and the filer keeps its identities across that. The older SeaweedFS ignores the extra actions, so a cluster that runs this before the version bump keeps old keys working through the bump; one that takes both in a single upgrade has them refused until the hook finishes. A failure of the grant ends in a Warning Event on the HelmRelease instead of a failed hook: Flux remediates a failed upgrade by rolling back, which would undo the bump this exists for. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin <[email protected]>
1 parent ab5676b commit d6394a3

4 files changed

Lines changed: 707 additions & 0 deletions

File tree

Lines changed: 240 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,240 @@
1+
#!/usr/bin/env bats
2+
# Unit tests for packages/system/seaweedfs/files/object-lock-actions.sh, the
3+
# post-upgrade hook that grants the dedicated Object Lock actions SeaweedFS 4.48
4+
# checks in place of Read and Write. Drives it against a fake `weed shell` that
5+
# keeps the IAM configuration in a JSON file, and a fake kubectl.
6+
7+
load test_helper
8+
9+
SCRIPT="$PWD/packages/system/seaweedfs/files/object-lock-actions.sh"
10+
11+
setup() {
12+
strict_setup
13+
BIN="$BATS_TEST_TMPDIR/bin"
14+
mkdir -p "$BIN"
15+
export STATE="$BATS_TEST_TMPDIR/iam.json"
16+
export CALLS="$BATS_TEST_TMPDIR/calls"
17+
export KUBECTL_LOG="$BATS_TEST_TMPDIR/kubectl"
18+
: > "$CALLS"
19+
: > "$KUBECTL_LOG"
20+
cat > "$BIN/weed" <<'EOF'
21+
#!/bin/bash
22+
# A weed shell session over $STATE. WEED_SLEEP hangs it, WEED_NOOP_APPLY drops
23+
# every apply, WEED_DROP_SESSIONS drops the applies of that many sessions,
24+
# WEED_READS_OK lets only that many listings succeed, and
25+
# WEED_UNRESOLVED names a user that GetUser cannot find, the
26+
# way s3.configure sees one: a fresh identity with no keys, which the apply
27+
# then stores over the listed one.
28+
set -eu
29+
[ "$1" = shell ] || exit 2
30+
[ -n "${WEED_SLEEP:-}" ] && sleep "$WEED_SLEEP"
31+
drop=false
32+
if [ -n "${WEED_DROP_SESSIONS:-}" ]; then
33+
n=$(cat "$STATE.sessions" 2>/dev/null || echo 0)
34+
# Only sessions that apply count.
35+
input=$(cat)
36+
case $input in *" -apply"*)
37+
n=$((n + 1)); echo "$n" > "$STATE.sessions"
38+
[ "$n" -le "$WEED_DROP_SESSIONS" ] && drop=true ;;
39+
esac
40+
exec <<< "$input"
41+
fi
42+
while read -r line; do
43+
case $line in
44+
"s3.configure")
45+
if [ -n "${WEED_READS_OK:-}" ]; then
46+
reads=$(cat "$STATE.reads" 2>/dev/null || echo 0)
47+
echo $((reads + 1)) > "$STATE.reads"
48+
[ "$reads" -lt "$WEED_READS_OK" ] || exit 1
49+
fi
50+
cat "$STATE"
51+
;;
52+
"s3.configure -user="*)
53+
echo "$line" >> "$CALLS"
54+
user=${line#s3.configure -user=}; user=${user%% *}
55+
rest=${line#* -actions=}
56+
actions=${rest%% *}
57+
apply=false
58+
case $rest in *" -apply") apply=true ;; esac
59+
if [ "$user" = "${WEED_UNRESOLVED:-}" ]; then
60+
current='{"name":"'"$user"'","credentials":[],"actions":[]}'
61+
else
62+
current=$(jq -c --arg u "$user" '.identities[] | select(.name == $u)' "$STATE")
63+
fi
64+
updated=$(jq -c --arg a "$actions" '.actions = reduce ($a | split(","))[] as $x (.actions; if index([$x]) then . else . + [$x] end)' <<< "$current")
65+
if $apply; then
66+
{ [ -n "${WEED_NOOP_APPLY:-}" ] || $drop; } && continue
67+
jq --arg u "$user" --argjson id "$updated" \
68+
'.identities |= map(if .name == $u then $id else . end)' "$STATE" > "$STATE.new"
69+
mv "$STATE.new" "$STATE"
70+
echo "$updated"
71+
else
72+
echo "$updated"
73+
echo 'Running in simulation mode. Use "-apply" option to apply the changes.'
74+
fi
75+
;;
76+
*)
77+
exit 3
78+
;;
79+
esac
80+
done
81+
EOF
82+
cat > "$BIN/kubectl" <<'EOF'
83+
#!/bin/sh
84+
case $1 in
85+
get) echo "ARGS: $*" >> "$KUBECTL_LOG"; printf 0b5e1a7e-0000-4000-8000-000000000001 ;;
86+
*) { echo "ARGS: $*"; cat; } >> "$KUBECTL_LOG" ;;
87+
esac
88+
EOF
89+
chmod +x "$BIN/weed" "$BIN/kubectl"
90+
export PATH="$BIN:$PATH"
91+
export MASTER=seaweedfs-master:9333 FILER=seaweedfs-filer-client:8888
92+
export NAMESPACE=tenant-foo RELEASE=seaweedfs-system
93+
export ATTEMPTS=2 RETRY_DELAY=0 SHELL_TIMEOUT=5
94+
}
95+
96+
identity() {
97+
# identity <name> <actions-json-array> [isStatic]
98+
printf '{"name":"%s","credentials":[{"accessKey":"AK-%s","secretKey":"SECRET-%s"}],"actions":%s,"isStatic":%s}' \
99+
"$1" "$1" "$1" "$2" "${3:-false}"
100+
}
101+
102+
state() {
103+
printf '{"identities":[%s]}\n' "$(IFS=,; echo "$*")" > "$STATE"
104+
}
105+
106+
actions_of() {
107+
jq -c --arg u "$1" '.identities[] | select(.name == $u) | .actions | sort' "$STATE"
108+
}
109+
110+
@test "readwrite identity gains the Get and Put Object Lock actions for its bucket" {
111+
state "$(identity rw '["Read:b1","Write:b1","List:b1","Tagging:b1"]')"
112+
"$SCRIPT"
113+
[ "$(actions_of rw)" = '["GetBucketObjectLockConfiguration:b1","GetObjectLegalHold:b1","GetObjectRetention:b1","List:b1","PutBucketObjectLockConfiguration:b1","PutObjectLegalHold:b1","PutObjectRetention:b1","Read:b1","Tagging:b1","Write:b1"]' ]
114+
}
115+
116+
@test "readonly identity gains only the Get Object Lock actions" {
117+
state "$(identity ro '["Read:b1","List:b1"]')"
118+
"$SCRIPT"
119+
[ "$(actions_of ro)" = '["GetBucketObjectLockConfiguration:b1","GetObjectLegalHold:b1","GetObjectRetention:b1","List:b1","Read:b1"]' ]
120+
}
121+
122+
@test "the suffix of each grant is carried over verbatim, bare grants stay bare" {
123+
state "$(identity w '["Write","Read:logs/*"]')"
124+
"$SCRIPT"
125+
[ "$(actions_of w)" = '["GetBucketObjectLockConfiguration:logs/*","GetObjectLegalHold:logs/*","GetObjectRetention:logs/*","PutBucketObjectLockConfiguration","PutObjectLegalHold","PutObjectRetention","Read:logs/*","Write"]' ]
126+
}
127+
128+
@test "actions that only start with Read or Write imply nothing" {
129+
state "$(identity acp '["ReadAcp:b1","WriteAcp:b1","List:b1"]')"
130+
"$SCRIPT"
131+
[ "$(actions_of acp)" = '["List:b1","ReadAcp:b1","WriteAcp:b1"]' ]
132+
[ ! -s "$CALLS" ]
133+
}
134+
135+
@test "admin and static identities are left alone" {
136+
state "$(identity admin '["Admin"]')" "$(identity static '["Read","Write"]' true)"
137+
"$SCRIPT"
138+
[ "$(actions_of admin)" = '["Admin"]' ]
139+
[ "$(actions_of static)" = '["Read","Write"]' ]
140+
[ ! -s "$CALLS" ]
141+
}
142+
143+
@test "a second run changes nothing" {
144+
state "$(identity rw '["Read:b1","Write:b1"]')" "$(identity ro '["Read:b2"]')"
145+
"$SCRIPT"
146+
[ "$(grep -c -- ' -apply$' "$CALLS")" -eq 2 ]
147+
: > "$CALLS"
148+
"$SCRIPT"
149+
[ ! -s "$CALLS" ]
150+
}
151+
152+
@test "secret keys never reach the output" {
153+
state "$(identity rw '["Read:b1","Write:b1"]')"
154+
out=$("$SCRIPT" 2>&1)
155+
case $out in
156+
*SECRET-*) echo "FAIL: secret key printed: $out"; false ;;
157+
esac
158+
case $out in
159+
*rw*) ;;
160+
*) echo "FAIL: the identity it changed is not named: $out"; false ;;
161+
esac
162+
}
163+
164+
@test "a grant that does not stick ends in a Warning Event on the HelmRelease and exit 0" {
165+
state "$(identity rw '["Read:b1","Write:b1"]')"
166+
WEED_NOOP_APPLY=1 "$SCRIPT"
167+
[ "$(grep -c -- ' -apply$' "$CALLS")" -eq 2 ]
168+
grep -q '^ARGS: create --filename -$' "$KUBECTL_LOG"
169+
grep -q '^type: Warning$' "$KUBECTL_LOG"
170+
grep -q '^ kind: HelmRelease$' "$KUBECTL_LOG"
171+
grep -q '^ name: seaweedfs-system$' "$KUBECTL_LOG"
172+
grep -q '^ namespace: tenant-foo$' "$KUBECTL_LOG"
173+
grep -q '^ uid: "0b5e1a7e-0000-4000-8000-000000000001"$' "$KUBECTL_LOG"
174+
grep -q 'rw' "$KUBECTL_LOG"
175+
}
176+
177+
@test "an unreachable filer ends in a Warning Event and exit 0" {
178+
rm -f "$STATE"
179+
"$SCRIPT"
180+
grep -q '^type: Warning$' "$KUBECTL_LOG"
181+
grep -q 'could not be read' "$KUBECTL_LOG"
182+
}
183+
184+
@test "a weed shell that never returns is cut off and still ends in an Event" {
185+
state "$(identity rw '["Read:b1","Write:b1"]')"
186+
SHELL_TIMEOUT=1 WEED_SLEEP=30 "$SCRIPT"
187+
grep -q '^type: Warning$' "$KUBECTL_LOG"
188+
}
189+
190+
@test "an identity s3.configure cannot resolve is never applied over" {
191+
state "$(identity rw '["Read:b1","Write:b1"]')" "$(identity lost '["Read:b2","Write:b2"]')"
192+
WEED_UNRESOLVED=lost "$SCRIPT"
193+
[ "$(jq -c '.identities[] | select(.name == "lost")' "$STATE")" = '{"name":"lost","credentials":[{"accessKey":"AK-lost","secretKey":"SECRET-lost"}],"actions":["Read:b2","Write:b2"],"isStatic":false}' ]
194+
if grep -q -- '-user=lost .* -apply$' "$CALLS"; then echo "FAIL: applied over an unresolved identity"; false; fi
195+
[ "$(actions_of rw)" = '["GetBucketObjectLockConfiguration:b1","GetObjectLegalHold:b1","GetObjectRetention:b1","PutBucketObjectLockConfiguration:b1","PutObjectLegalHold:b1","PutObjectRetention:b1","Read:b1","Write:b1"]' ]
196+
grep -q '^type: Warning$' "$KUBECTL_LOG"
197+
grep -q 'lost' "$KUBECTL_LOG"
198+
}
199+
200+
@test "output without an IAM configuration counts as a failed read" {
201+
echo 'error: connection refused' > "$STATE"
202+
"$SCRIPT"
203+
grep -q 'could not be read' "$KUBECTL_LOG"
204+
}
205+
206+
@test "no Event is written when every grant sticks" {
207+
state "$(identity rw '["Read:b1","Write:b1"]')"
208+
"$SCRIPT"
209+
[ ! -s "$KUBECTL_LOG" ]
210+
}
211+
212+
@test "JSON that is not an IAM configuration counts as a failed read" {
213+
echo '{"error":"not ready"}' > "$STATE"
214+
"$SCRIPT"
215+
grep -q 'could not be read' "$KUBECTL_LOG"
216+
}
217+
218+
@test "grants that stick only on the last attempt raise no Event" {
219+
state "$(identity rw '["Read:b1","Write:b1"]')"
220+
WEED_DROP_SESSIONS=1 "$SCRIPT"
221+
[ "$(grep -c -- ' -apply$' "$CALLS")" -eq 2 ]
222+
[ ! -s "$KUBECTL_LOG" ]
223+
}
224+
225+
@test "a name or action weed shell would split never reaches a command line and is reported" {
226+
state "$(identity rw '["Read:b1","Write:b1"]')" "$(identity 'x;s3.configure -user=rw -actions=Admin -apply' '["Read:b2"]')" "$(identity y '["Read:b3;s3.configure -user=rw -actions=Admin"]')"
227+
"$SCRIPT"
228+
if grep -q -- 'x;' "$CALLS"; then echo "FAIL: unsafe name reached weed shell"; false; fi
229+
if grep -q -- 'Admin' "$CALLS"; then echo "FAIL: injected action reached weed shell"; false; fi
230+
[ "$(actions_of rw)" = '["GetBucketObjectLockConfiguration:b1","GetObjectLegalHold:b1","GetObjectRetention:b1","PutBucketObjectLockConfiguration:b1","PutObjectLegalHold:b1","PutObjectRetention:b1","Read:b1","Write:b1"]' ]
231+
grep -q '^type: Warning$' "$KUBECTL_LOG"
232+
}
233+
234+
@test "a final read that fails reports the outcome as unverified, not the last plan" {
235+
state "$(identity rw '["Read:b1","Write:b1"]')"
236+
WEED_READS_OK=1 "$SCRIPT"
237+
[ "$(actions_of rw)" = '["GetBucketObjectLockConfiguration:b1","GetObjectLegalHold:b1","GetObjectRetention:b1","PutBucketObjectLockConfiguration:b1","PutObjectLegalHold:b1","PutObjectRetention:b1","Read:b1","Write:b1"]' ]
238+
grep -q 'could not be read' "$KUBECTL_LOG"
239+
if grep -q 'missing them' "$KUBECTL_LOG"; then echo "FAIL: reported a stale plan"; false; fi
240+
}
Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
#!/bin/sh
2+
# SeaweedFS 4.48 checks Object Lock requests against dedicated actions instead
3+
# of Read and Write. Identities granted before that keep only Read and Write,
4+
# so this grants each of them the dedicated actions its Read or Write grant
5+
# covered until then, with the same bucket suffix. Governance bypass is not
6+
# among them: it was a separate action before 4.48 too.
7+
#
8+
# The IAM configuration carries secret keys and the apply echoes the whole
9+
# identity back, so neither is printed.
10+
#
11+
# A failure must not fail the release: Flux rolls a failed upgrade back, which
12+
# would take SeaweedFS back to the version without the split. It ends in a
13+
# Warning Event on the HelmRelease instead.
14+
set -eu
15+
16+
: "${MASTER:?}" "${FILER:?}" "${NAMESPACE:?}" "${RELEASE:?}"
17+
JOB=${JOB:-$RELEASE-object-lock-actions}
18+
# A weed shell waits for the master without a deadline, and a hook still
19+
# running at the release timeout fails the upgrade, so every call is bounded
20+
# and the worst case stays well inside the HelmRelease timeout.
21+
ATTEMPTS=${ATTEMPTS:-3}
22+
RETRY_DELAY=${RETRY_DELAY:-20}
23+
SHELL_TIMEOUT=${SHELL_TIMEOUT:-30}
24+
25+
# Runs the commands on stdin in one session.
26+
weed_shell() {
27+
timeout "$SHELL_TIMEOUT" weed shell -master="$MASTER" -filer="$FILER"
28+
}
29+
30+
# The plan: every identity that lacks an action its grants imply, with the
31+
# actions it holds now. Static identities come from a config file the IAM API
32+
# cannot change.
33+
plan_from_config() {
34+
sed -n '/^{/,$p' | jq -c '
35+
def implied:
36+
if test("^Write(:|$)") then
37+
sub("^Write"; "") as $s
38+
| ["PutObjectRetention", "PutObjectLegalHold", "PutBucketObjectLockConfiguration"] | map(. + $s)
39+
elif test("^Read(:|$)") then
40+
sub("^Read"; "") as $s
41+
| ["GetObjectRetention", "GetObjectLegalHold", "GetBucketObjectLockConfiguration"] | map(. + $s)
42+
else [] end;
43+
if has("identities") | not then error("no IAM configuration in the output") else . end
44+
| [.identities[]?
45+
| select(.isStatic != true)
46+
| (.actions // []) as $have
47+
| {name, have: $have, missing: ([$have[] | implied[]] | unique - $have)}
48+
| select(.missing | length > 0)]'
49+
}
50+
51+
# s3.configure resolves the user on its own, and creates a fresh identity
52+
# with no keys when it cannot, which would then replace the one listed. The
53+
# dry run shows what it resolved; only an identity that still carries every
54+
# action the listing showed is applied.
55+
safe_commands() {
56+
sed '/^Running in simulation mode/d' | jq -rs --argjson plan "$1" '
57+
(map({key: .name, value: (.actions // [])}) | from_entries) as $resolved
58+
| $plan[]
59+
| select(($resolved[.name] // []) as $r | .have - $r | length == 0)
60+
| "s3.configure -user=\(.name) -actions=\(.missing | join(",")) -apply"'
61+
}
62+
63+
# weed shell splits a line on ";" and on whitespace, so an identity whose name
64+
# or actions carry anything else is never put on a command line. It stays in
65+
# the plan and is reported.
66+
dry_run_commands() {
67+
jq -r '.[]
68+
| select(.name | test("^[A-Za-z0-9._@+=-]+$"))
69+
| select(.missing | all(test("^[A-Za-z0-9._@+=:/*?-]+$")))
70+
| "s3.configure -user=\(.name) -actions=\(.missing | join(","))"'
71+
}
72+
73+
# Each attempt reads, then grants; the read after the last attempt is what
74+
# decides the outcome. A failed read keeps the previous plan for the next
75+
# grant, but the outcome is never reported from it.
76+
plan='[]'
77+
attempt=0
78+
while :; do
79+
read_ok=false
80+
if config=$(echo s3.configure | weed_shell) && fresh=$(printf '%s\n' "$config" | plan_from_config) && [ -n "$fresh" ]; then
81+
read_ok=true
82+
plan=$fresh
83+
if [ "$plan" = '[]' ]; then
84+
echo "every identity carries the Object Lock actions its grants imply"
85+
exit 0
86+
fi
87+
fi
88+
[ "$attempt" -ge "$ATTEMPTS" ] && break
89+
attempt=$((attempt + 1))
90+
if [ -n "$plan" ] && dry=$(printf '%s\n' "$plan" | dry_run_commands) && [ -n "$dry" ] &&
91+
commands=$(printf '%s\n' "$dry" | weed_shell | safe_commands "$plan") && [ -n "$commands" ]; then
92+
printf '%s\n' "$commands" | sed 's/^s3.configure -user=\([^ ]*\) -actions=\([^ ]*\) -apply$/granting \2 to \1/'
93+
printf '%s\n' "$commands" | weed_shell >/dev/null || true
94+
fi
95+
sleep "$RETRY_DELAY"
96+
done
97+
98+
if $read_ok; then
99+
pending=$(printf '%s\n' "$plan" | jq -r '[.[].name] | join(" ")')
100+
message="Object Lock actions not granted after $ATTEMPTS attempts; identities still missing them: $pending. See the logs of Job $JOB."
101+
else
102+
message="Object Lock actions unverified after $ATTEMPTS attempts: the IAM configuration could not be read. See the logs of Job $JOB."
103+
fi
104+
message=$(printf '%s' "$message" | tr -d '"\134')
105+
echo "$message" >&2
106+
now=$(date -u +%Y-%m-%dT%H:%M:%SZ)
107+
# kubectl describe matches events by uid as well as name.
108+
uid=$(kubectl get helmrelease "$RELEASE" --namespace "$NAMESPACE" --output jsonpath='{.metadata.uid}') || uid=
109+
kubectl create --filename - <<EOF || echo "could not record the Warning Event" >&2
110+
apiVersion: v1
111+
kind: Event
112+
metadata:
113+
generateName: $JOB-
114+
namespace: $NAMESPACE
115+
involvedObject:
116+
apiVersion: helm.toolkit.fluxcd.io/v2
117+
kind: HelmRelease
118+
name: $RELEASE
119+
namespace: $NAMESPACE
120+
uid: "$uid"
121+
type: Warning
122+
reason: ObjectLockActionsNotGranted
123+
message: "$message"
124+
source:
125+
component: $JOB
126+
firstTimestamp: "$now"
127+
lastTimestamp: "$now"
128+
count: 1
129+
EOF

0 commit comments

Comments
 (0)