|
| 1 | +# Exercises the sentinel model against the live API server: the writer policy |
| 2 | +# that gates TenantProjection, and the CA-extraction controller that reconciles |
| 3 | +# one. |
| 4 | +# |
| 5 | +# Two things need a real cluster that unit tests cannot reach: |
| 6 | +# - the ValidatingAdmissionPolicy, whose whole protection is one CEL expression |
| 7 | +# the API server evaluates on admission. helm-unittest pins the expression and |
| 8 | +# the bats check pins the writer literal, but only a live deny/allow probe |
| 9 | +# proves the API server actually refuses a forger and admits flux. |
| 10 | +# - the controller wiring end to end: a sentinel a flux identity creates, an |
| 11 | +# absent source surfaced as Ready=False/SourceNotFound, and a real source |
| 12 | +# lifted into a key-free projection. |
| 13 | +# |
| 14 | +# Impersonation and error-message greps cannot be expressed by Chainsaw `apply` |
| 15 | +# — and the writer policy denies the admin identity Chainsaw applies as, so a |
| 16 | +# sentinel can only be created by impersonating flux — so the cases stay scripts, |
| 17 | +# mirroring hack/e2e-chainsaw/gateway. Each case does its create, its assertions |
| 18 | +# and its cleanup inside ONE script step with a self-contained trap, which is the |
| 19 | +# pattern docs/agents/e2e-testing.md sanctions (no test-level EXIT trap). |
| 20 | +# |
| 21 | +# The tenant-visibility leg (a projection reaching the tenant through |
| 22 | +# core.cozystack.io/tenantsecrets) is NOT exercised here: it needs a full |
| 23 | +# application so the lineage webhook can resolve the projection to an app whose |
| 24 | +# spec.secrets selects it, which is the postgres app suite's job. This suite |
| 25 | +# covers the parts that stand alone — the policy and the controller. |
| 26 | +apiVersion: chainsaw.kyverno.io/v1alpha1 |
| 27 | +kind: Test |
| 28 | +metadata: |
| 29 | + name: tenant-projection |
| 30 | +spec: |
| 31 | + steps: |
| 32 | + - name: policy-binding-present |
| 33 | + try: |
| 34 | + - assert: |
| 35 | + timeout: 1m |
| 36 | + resource: |
| 37 | + apiVersion: admissionregistration.k8s.io/v1 |
| 38 | + kind: ValidatingAdmissionPolicyBinding |
| 39 | + metadata: |
| 40 | + name: cozystack-tenant-projection-writer-policy |
| 41 | + |
| 42 | + - name: forge-denied-flux-allowed |
| 43 | + try: |
| 44 | + - script: |
| 45 | + content: | |
| 46 | + set -eu |
| 47 | + ns="$NAMESPACE" |
| 48 | + flux="system:serviceaccount:cozy-fluxcd:flux" |
| 49 | +
|
| 50 | + cleanup() { |
| 51 | + # This suite runs against the shared, pre-existing tenant-test |
| 52 | + # namespace (hack/e2e-chainsaw/.chainsaw.yaml). Chainsaw reclaims only |
| 53 | + # what IT applied; everything here is created by kubectl inside this |
| 54 | + # script, so it must be pruned by hand. The trap is armed BEFORE the |
| 55 | + # RBAC apply so a partial apply cannot leak the privileged Role. The |
| 56 | + # sentinel a regression would create (vap-tp-forged) is deleted too — |
| 57 | + # it must not outlive the run. Deletes are not masked with `|| true` |
| 58 | + # (docs/agents/e2e-testing.md forbids it): each runs even if an earlier |
| 59 | + # errors, the test's own exit status is preserved, and a genuine |
| 60 | + # cleanup failure fails the step. |
| 61 | + status=$? |
| 62 | + rc=0 |
| 63 | + kubectl -n "$ns" delete tenantprojection vap-tp-forged --ignore-not-found || rc=1 |
| 64 | + kubectl -n "$ns" delete tenantprojection vap-tp-flux --ignore-not-found || rc=1 |
| 65 | + kubectl -n "$ns" delete rolebinding vap-probe-tp --ignore-not-found || rc=1 |
| 66 | + kubectl -n "$ns" delete role vap-probe-tp --ignore-not-found || rc=1 |
| 67 | + if [ "$status" -ne 0 ]; then exit "$status"; fi |
| 68 | + exit "$rc" |
| 69 | + } |
| 70 | + trap cleanup EXIT |
| 71 | +
|
| 72 | + as_tenant="kubectl --as=system:serviceaccount:${ns}:default \ |
| 73 | + --as-group=system:serviceaccounts \ |
| 74 | + --as-group=system:serviceaccounts:${ns}" |
| 75 | +
|
| 76 | + # 1. RBAC is the PRIMARY boundary: the tenant SA holds no verb on |
| 77 | + # internal.cozystack.io, so its create stops at authorization before |
| 78 | + # admission is even reached. |
| 79 | + out=$($as_tenant apply -f - 2>&1 <<EOF |
| 80 | + apiVersion: internal.cozystack.io/v1alpha1 |
| 81 | + kind: TenantProjection |
| 82 | + metadata: |
| 83 | + name: vap-tp-forged |
| 84 | + namespace: ${ns} |
| 85 | + spec: |
| 86 | + projections: |
| 87 | + - type: CACert |
| 88 | + sourceSecretName: vap-tp-forged-src |
| 89 | + sourceKey: ca.crt |
| 90 | + EOF |
| 91 | + ) && rc=0 || rc=$? |
| 92 | + echo "rbac-deny rc=$rc out=$out" |
| 93 | + [ "$rc" -ne 0 ] |
| 94 | + echo "$out" | grep -qi "forbidden" |
| 95 | + echo "$out" | grep -qi "tenantprojections" |
| 96 | +
|
| 97 | + # 2. Grant the tenant SA the verb, so the next create reaches ADMISSION |
| 98 | + # (the VAP) instead of stopping at RBAC — otherwise an RBAC Forbidden |
| 99 | + # would masquerade as the policy's denial. |
| 100 | + kubectl apply -f - <<EOF |
| 101 | + apiVersion: rbac.authorization.k8s.io/v1 |
| 102 | + kind: Role |
| 103 | + metadata: |
| 104 | + name: vap-probe-tp |
| 105 | + namespace: ${ns} |
| 106 | + rules: |
| 107 | + - apiGroups: ["internal.cozystack.io"] |
| 108 | + resources: ["tenantprojections"] |
| 109 | + verbs: ["create", "update", "patch", "get", "delete"] |
| 110 | + --- |
| 111 | + apiVersion: rbac.authorization.k8s.io/v1 |
| 112 | + kind: RoleBinding |
| 113 | + metadata: |
| 114 | + name: vap-probe-tp |
| 115 | + namespace: ${ns} |
| 116 | + roleRef: |
| 117 | + apiGroup: rbac.authorization.k8s.io |
| 118 | + kind: Role |
| 119 | + name: vap-probe-tp |
| 120 | + subjects: |
| 121 | + - kind: ServiceAccount |
| 122 | + name: default |
| 123 | + namespace: ${ns} |
| 124 | + EOF |
| 125 | +
|
| 126 | + # 3. FORGE: the tenant SA, now RBAC-authorized, tries to declare a trust |
| 127 | + # anchor. The VAP refuses it because the writer is not flux. |
| 128 | + out=$($as_tenant apply -f - 2>&1 <<EOF |
| 129 | + apiVersion: internal.cozystack.io/v1alpha1 |
| 130 | + kind: TenantProjection |
| 131 | + metadata: |
| 132 | + name: vap-tp-forged |
| 133 | + namespace: ${ns} |
| 134 | + spec: |
| 135 | + projections: |
| 136 | + - type: CACert |
| 137 | + sourceSecretName: vap-tp-forged-src |
| 138 | + sourceKey: ca.crt |
| 139 | + EOF |
| 140 | + ) && rc=0 || rc=$? |
| 141 | + echo "vap-deny rc=$rc out=$out" |
| 142 | + [ "$rc" -ne 0 ] |
| 143 | + echo "$out" | grep -qi "ValidatingAdmissionPolicy" |
| 144 | + echo "$out" | grep -q "may only be written by the flux service account" |
| 145 | +
|
| 146 | + # 4. flux IS allowed. Also the positive control — a policy that denied |
| 147 | + # everyone (a mistyped writer literal) fails HERE, not silently above. |
| 148 | + kubectl --as="$flux" apply -f - <<EOF |
| 149 | + apiVersion: internal.cozystack.io/v1alpha1 |
| 150 | + kind: TenantProjection |
| 151 | + metadata: |
| 152 | + name: vap-tp-flux |
| 153 | + namespace: ${ns} |
| 154 | + spec: |
| 155 | + projections: |
| 156 | + - type: CACert |
| 157 | + sourceSecretName: vap-tp-flux-src |
| 158 | + sourceKey: ca.crt |
| 159 | + EOF |
| 160 | + check: |
| 161 | + ($error == null): true |
| 162 | + |
| 163 | + - name: controller-projects-a-key-free-anchor |
| 164 | + try: |
| 165 | + - script: |
| 166 | + content: | |
| 167 | + set -eu |
| 168 | + ns="$NAMESPACE" |
| 169 | + flux="system:serviceaccount:cozy-fluxcd:flux" |
| 170 | + release="cacert-e2e" |
| 171 | + sentinel="${release}-ca" |
| 172 | + source="${release}-ca-src" |
| 173 | + proj="${release}.tenant-ca" |
| 174 | + tmp="$(mktemp -d)" |
| 175 | +
|
| 176 | + cleanup() { |
| 177 | + status=$? |
| 178 | + rc=0 |
| 179 | + # Deleting the sentinel garbage-collects the projection through its |
| 180 | + # owner reference; delete the projection explicitly too so a |
| 181 | + # regression that mis-owns it cannot leak into the shared namespace. |
| 182 | + kubectl -n "$ns" delete tenantprojection "$sentinel" --ignore-not-found || rc=1 |
| 183 | + kubectl -n "$ns" delete secret "$proj" --ignore-not-found || rc=1 |
| 184 | + kubectl -n "$ns" delete secret "$source" --ignore-not-found || rc=1 |
| 185 | + rm -rf "$tmp" || rc=1 |
| 186 | + if [ "$status" -ne 0 ]; then exit "$status"; fi |
| 187 | + exit "$rc" |
| 188 | + } |
| 189 | + trap cleanup EXIT |
| 190 | +
|
| 191 | + # The sentinel carries the helm.toolkit.fluxcd.io/name label Flux stamps |
| 192 | + # in production; the controller derives the canonical projection name |
| 193 | + # from it. flux must create it — the writer policy denies everyone else. |
| 194 | + kubectl --as="$flux" apply -f - <<EOF |
| 195 | + apiVersion: internal.cozystack.io/v1alpha1 |
| 196 | + kind: TenantProjection |
| 197 | + metadata: |
| 198 | + name: ${sentinel} |
| 199 | + namespace: ${ns} |
| 200 | + labels: |
| 201 | + helm.toolkit.fluxcd.io/name: ${release} |
| 202 | + spec: |
| 203 | + projections: |
| 204 | + - type: CACert |
| 205 | + sourceSecretName: ${source} |
| 206 | + sourceKey: ca.crt |
| 207 | + EOF |
| 208 | +
|
| 209 | + # The source does not exist yet: the sentinel's Ready condition must say |
| 210 | + # so, in the open, rather than the controller retrying in silence. |
| 211 | + # TODO(e2e-replace-fixed-timeouts): a script poll because the sentinel is |
| 212 | + # impersonation-created, which Chainsaw's declarative assert cannot do. |
| 213 | + ok="" |
| 214 | + for _ in $(seq 1 30); do |
| 215 | + reason=$(kubectl -n "$ns" get tenantprojection "$sentinel" \ |
| 216 | + -o jsonpath="{.status.conditions[?(@.type=='Ready')].reason}" 2>/dev/null || true) |
| 217 | + status=$(kubectl -n "$ns" get tenantprojection "$sentinel" \ |
| 218 | + -o jsonpath="{.status.conditions[?(@.type=='Ready')].status}" 2>/dev/null || true) |
| 219 | + if [ "$status" = "False" ] && [ "$reason" = "SourceNotFound" ]; then ok=1; break; fi |
| 220 | + sleep 2 |
| 221 | + done |
| 222 | + [ -n "$ok" ] || { echo "sentinel never reported SourceNotFound (status=$status reason=$reason)"; exit 1; } |
| 223 | +
|
| 224 | + # Now the operator mints its CA: a real certificate NEXT TO the CA |
| 225 | + # private key, exactly the shape the controller exists to break apart. |
| 226 | + openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:P-256 -nodes \ |
| 227 | + -keyout "$tmp/ca.key" -out "$tmp/ca.crt" -days 1 \ |
| 228 | + -subj "/CN=cozystack-cacert-e2e" >/dev/null 2>&1 |
| 229 | + kubectl -n "$ns" create secret generic "$source" \ |
| 230 | + --from-file=ca.crt="$tmp/ca.crt" --from-file=ca.key="$tmp/ca.key" |
| 231 | +
|
| 232 | + # The projection must appear, key-free — and this is the guard for the |
| 233 | + # source WATCH, not just the projection. The source Secret carries no |
| 234 | + # label, so nothing but the metadata Secret watch reacts to its creation; |
| 235 | + # a sentinel change is the only other trigger and there is none here. So |
| 236 | + # the 60s budget below (30x2s) is deliberately far under resyncInterval |
| 237 | + # (5m): if the watch is filtered or dead (e.g. wired onto the manager's |
| 238 | + # wildcard-scoped Secret cache), the projection only lands on the resync |
| 239 | + # and this times out. It passes only when the dedicated metadata cache |
| 240 | + # delivers the source-creation event promptly. |
| 241 | + # TODO(e2e-replace-fixed-timeouts): a script poll because the sentinel is |
| 242 | + # impersonation-created, which Chainsaw's declarative assert cannot do. |
| 243 | + ok="" |
| 244 | + for _ in $(seq 1 30); do |
| 245 | + if kubectl -n "$ns" get secret "$proj" >/dev/null 2>&1; then ok=1; break; fi |
| 246 | + sleep 2 |
| 247 | + done |
| 248 | + [ -n "$ok" ] || { echo "projection $proj was never created"; exit 1; } |
| 249 | +
|
| 250 | + # It is Opaque, carries ONLY ca.crt (no ca.key), matches the source |
| 251 | + # certificate, is labelled for tenant visibility, and is owned by the |
| 252 | + # sentinel — the chain the lineage webhook and the GC both rely on. |
| 253 | + [ "$(kubectl -n "$ns" get secret "$proj" -o jsonpath='{.type}')" = "Opaque" ] |
| 254 | + [ "$(kubectl -n "$ns" get secret "$proj" -o jsonpath='{.metadata.labels.internal\.cozystack\.io/tenant-ca}')" = "true" ] |
| 255 | + [ "$(kubectl -n "$ns" get secret "$proj" -o jsonpath='{.metadata.ownerReferences[0].kind}')" = "TenantProjection" ] |
| 256 | + [ "$(kubectl -n "$ns" get secret "$proj" -o jsonpath='{.metadata.ownerReferences[0].name}')" = "$sentinel" ] |
| 257 | + kubectl -n "$ns" get secret "$proj" -o jsonpath='{.data.ca\.crt}' | grep -q . |
| 258 | + if kubectl -n "$ns" get secret "$proj" -o jsonpath='{.data.ca\.key}' | grep -q .; then |
| 259 | + echo "the projection leaked ca.key — the whole point of the controller failed" |
| 260 | + exit 1 |
| 261 | + fi |
| 262 | + # Compare the parsed certificate, not the raw PEM: the controller |
| 263 | + # re-encodes ca.crt from the DER it parsed, so a header line or a |
| 264 | + # column-width difference from openssl's output would fail a byte |
| 265 | + # comparison of a correct projection. The fingerprint is stable across |
| 266 | + # any re-encoding. |
| 267 | + proj_fp=$(kubectl -n "$ns" get secret "$proj" -o jsonpath='{.data.ca\.crt}' | base64 -d | openssl x509 -noout -fingerprint -sha256) |
| 268 | + src_fp=$(openssl x509 -in "$tmp/ca.crt" -noout -fingerprint -sha256) |
| 269 | + [ "$proj_fp" = "$src_fp" ] || { echo "projected ca.crt does not match the source certificate"; exit 1; } |
| 270 | +
|
| 271 | + # And the sentinel now reports Ready. |
| 272 | + # TODO(e2e-replace-fixed-timeouts): same reason as the polls above. |
| 273 | + ok="" |
| 274 | + for _ in $(seq 1 30); do |
| 275 | + status=$(kubectl -n "$ns" get tenantprojection "$sentinel" \ |
| 276 | + -o jsonpath="{.status.conditions[?(@.type=='Ready')].status}" 2>/dev/null || true) |
| 277 | + if [ "$status" = "True" ]; then ok=1; break; fi |
| 278 | + sleep 2 |
| 279 | + done |
| 280 | + [ -n "$ok" ] || { echo "sentinel never reported Ready=True (status=$status)"; exit 1; } |
| 281 | + check: |
| 282 | + ($error == null): true |
0 commit comments