|
| 1 | +# Namespace logs through the observability proxy, as the console reaches |
| 2 | +# them: every request goes through the console nginx (a port-forward to its |
| 3 | +# Service), which routes the logs root to the proxy, which authenticates the |
| 4 | +# bearer token, asks a SubjectAccessReview and forwards a rebuilt query to |
| 5 | +# the tenant's log store. |
| 6 | +# |
| 7 | +# Two sibling tenants, A (logsa) and B (logsb), both under tenant-test, so |
| 8 | +# both resolve to the root log store. A viewer of A reads A's logs and is |
| 9 | +# refused B's; parameters and headers a client adds cannot widen the scope |
| 10 | +# (the proxy forces the namespace filter and tenant 0:0); a live tail stays |
| 11 | +# open and keeps delivering lines past 60s. |
| 12 | +# |
| 13 | +# Not covered here: the OIDC gatekeeper (oauth2-proxy) in front of nginx, |
| 14 | +# because driving a Keycloak login needs a browser; and Fluent Bit writing |
| 15 | +# real pod logs, because the root log store is off in e2e and this suite |
| 16 | +# seeds its own (log-store.yaml). |
| 17 | +apiVersion: chainsaw.kyverno.io/v1alpha1 |
| 18 | +kind: Test |
| 19 | +metadata: |
| 20 | + name: observability-proxy-logs |
| 21 | +spec: |
| 22 | + catch: |
| 23 | + - events: {} |
| 24 | + - podLogs: |
| 25 | + namespace: cozy-observability-proxy |
| 26 | + selector: app.kubernetes.io/name=observability-proxy |
| 27 | + tail: 100 |
| 28 | + - podLogs: |
| 29 | + namespace: tenant-root |
| 30 | + selector: app.kubernetes.io/name=e2e-observability-log-store |
| 31 | + tail: 100 |
| 32 | + - podLogs: |
| 33 | + namespace: cozy-dashboard |
| 34 | + selector: app.kubernetes.io/name=console |
| 35 | + tail: 100 |
| 36 | + steps: |
| 37 | + - name: proxy-available |
| 38 | + try: |
| 39 | + - assert: |
| 40 | + resource: |
| 41 | + apiVersion: apps/v1 |
| 42 | + kind: Deployment |
| 43 | + metadata: |
| 44 | + name: observability-proxy |
| 45 | + namespace: cozy-observability-proxy |
| 46 | + status: |
| 47 | + (conditions[?type == 'Available']): |
| 48 | + - status: "True" |
| 49 | + |
| 50 | + - name: tenants |
| 51 | + try: |
| 52 | + - apply: |
| 53 | + resource: |
| 54 | + apiVersion: apps.cozystack.io/v1alpha1 |
| 55 | + kind: Tenant |
| 56 | + metadata: |
| 57 | + name: logsa |
| 58 | + spec: {} |
| 59 | + - apply: |
| 60 | + resource: |
| 61 | + apiVersion: apps.cozystack.io/v1alpha1 |
| 62 | + kind: Tenant |
| 63 | + metadata: |
| 64 | + name: logsb |
| 65 | + spec: {} |
| 66 | + # 16m: tenant-rd's 15m install timeout plus the 60s start margin; see |
| 67 | + # "Sizing an HR-Ready budget" in docs/agents/e2e-testing.md. Waited so |
| 68 | + # that cleanup does not uninstall a release whose install still runs. |
| 69 | + - assert: |
| 70 | + timeout: 16m |
| 71 | + resource: |
| 72 | + apiVersion: helm.toolkit.fluxcd.io/v2 |
| 73 | + kind: HelmRelease |
| 74 | + metadata: |
| 75 | + name: tenant-logsa |
| 76 | + status: |
| 77 | + (conditions[?type == 'Ready']): |
| 78 | + - status: "True" |
| 79 | + - assert: |
| 80 | + timeout: 16m |
| 81 | + resource: |
| 82 | + apiVersion: helm.toolkit.fluxcd.io/v2 |
| 83 | + kind: HelmRelease |
| 84 | + metadata: |
| 85 | + name: tenant-logsb |
| 86 | + status: |
| 87 | + (conditions[?type == 'Ready']): |
| 88 | + - status: "True" |
| 89 | + # The label names the store the proxy reads; the seeded store below |
| 90 | + # stands in for the root one, so both namespaces must resolve to it. |
| 91 | + - assert: |
| 92 | + resource: |
| 93 | + apiVersion: v1 |
| 94 | + kind: Namespace |
| 95 | + metadata: |
| 96 | + name: tenant-test-logsa |
| 97 | + labels: |
| 98 | + namespace.cozystack.io/monitoring: tenant-root |
| 99 | + - assert: |
| 100 | + resource: |
| 101 | + apiVersion: v1 |
| 102 | + kind: Namespace |
| 103 | + metadata: |
| 104 | + name: tenant-test-logsb |
| 105 | + labels: |
| 106 | + namespace.cozystack.io/monitoring: tenant-root |
| 107 | + |
| 108 | + - name: viewer-and-store |
| 109 | + try: |
| 110 | + # Applying over a real root store would rewrite its Service; refuse to. |
| 111 | + - error: |
| 112 | + resource: |
| 113 | + apiVersion: v1 |
| 114 | + kind: Service |
| 115 | + metadata: |
| 116 | + name: vlselect-generic |
| 117 | + namespace: tenant-root |
| 118 | + - apply: |
| 119 | + file: viewer.yaml |
| 120 | + - apply: |
| 121 | + file: log-store.yaml |
| 122 | + - assert: |
| 123 | + timeout: 3m |
| 124 | + resource: |
| 125 | + apiVersion: apps/v1 |
| 126 | + kind: Deployment |
| 127 | + metadata: |
| 128 | + name: e2e-observability-log-store |
| 129 | + namespace: tenant-root |
| 130 | + status: |
| 131 | + (conditions[?type == 'Available']): |
| 132 | + - status: "True" |
| 133 | + |
| 134 | + - name: seed-logs |
| 135 | + try: |
| 136 | + - script: |
| 137 | + # 30s forward + 4 x 15s ingests + the 30s visibility poll, each |
| 138 | + # capped on its own, under a 3m op. |
| 139 | + timeout: 3m |
| 140 | + content: | |
| 141 | + set -eu |
| 142 | + . ./lib.sh |
| 143 | + trap 'for p in ${pf_pids:-}; do kill "$p" 2>/dev/null || true; done; rm -f "$body"' EXIT |
| 144 | + forward tenant-root vlselect-generic 19471:9471 |
| 145 | + ingest tenant-test-logsa e2e-marker-a |
| 146 | + ingest tenant-test-logsb e2e-marker-b |
| 147 | + # Same namespace, another VictoriaLogs tenant: reachable only if a |
| 148 | + # client AccountID header got through. |
| 149 | + ingest tenant-test-logsa e2e-marker-account7 7 |
| 150 | + timeout 30 sh -ec "until curl --silent --fail '$STORE/select/logsql/query?query=e2e-marker-a' | grep -q e2e-marker-a; do sleep 1; done" |
| 151 | +
|
| 152 | + - name: namespace-scope |
| 153 | + try: |
| 154 | + - script: |
| 155 | + # 2 x 30s forwards + one 30s token request + 11 calls capped at 30s. |
| 156 | + timeout: 8m |
| 157 | + content: | |
| 158 | + set -eu |
| 159 | + . ./lib.sh |
| 160 | + trap 'for p in ${pf_pids:-}; do kill "$p" 2>/dev/null || true; done; rm -f "$body"' EXIT |
| 161 | + forward cozy-dashboard cozy-dashboard-console 18080:8080 |
| 162 | + token=$(k -n tenant-test-logsa create token logs-viewer --duration=10m) |
| 163 | + auth="Authorization: Bearer $token" |
| 164 | +
|
| 165 | + code=$(call "$LOGS_A/query?query=*") |
| 166 | + expect 401 "$code" "no token" |
| 167 | +
|
| 168 | + code=$(call "$LOGS_A/query?query=*" --header "$auth") |
| 169 | + expect 200 "$code" "viewer reads A" |
| 170 | + grep -q e2e-marker-a "$body" || { echo "FAIL: A's own line is missing" >&2; cat "$body" >&2; exit 1; } |
| 171 | + absent e2e-marker-b "viewer reads A" |
| 172 | + absent e2e-marker-account7 "viewer reads A" |
| 173 | +
|
| 174 | + code=$(call "$LOGS_B/query?query=*" --header "$auth") |
| 175 | + expect 403 "$code" "viewer reads B" |
| 176 | +
|
| 177 | + code=$(call "$LOGS_A/query" --header "$auth" --get \ |
| 178 | + --data-urlencode 'query=*' \ |
| 179 | + --data-urlencode 'extra_stream_filters={kubernetes_namespace_name="tenant-test-logsb"}' \ |
| 180 | + --data-urlencode 'extra_filters=kubernetes_namespace_name:tenant-test-logsb') |
| 181 | + expect 200 "$code" "client extra filters in the query string" |
| 182 | + absent e2e-marker-b "client extra filters in the query string" |
| 183 | +
|
| 184 | + code=$(call "$LOGS_A/query" --header "$auth" \ |
| 185 | + --data-urlencode 'query=*' \ |
| 186 | + --data-urlencode 'extra_stream_filters={kubernetes_namespace_name="tenant-test-logsb"}') |
| 187 | + expect 200 "$code" "client extra filters in a POST form" |
| 188 | + absent e2e-marker-b "client extra filters in a POST form" |
| 189 | +
|
| 190 | + code=$(call "$LOGS_A/query?query=*" --header "$auth" --header "AccountID: 7" --header "ProjectID: 0") |
| 191 | + expect 200 "$code" "client AccountID header" |
| 192 | + absent e2e-marker-account7 "client AccountID header" |
| 193 | +
|
| 194 | + code=$(call "$LOGS_A/query" --header "$auth" --get \ |
| 195 | + --data-urlencode 'query={kubernetes_namespace_name="tenant-test-logsb"}') |
| 196 | + expect 200 "$code" "B's stream named in the query" |
| 197 | + absent e2e-marker-b "B's stream named in the query" |
| 198 | +
|
| 199 | + code=$(call "$LOGS_A/query" --header "$auth" --get \ |
| 200 | + --data-urlencode 'query=options(ignore_global_time_filter=true) *') |
| 201 | + expect 400 "$code" "query options" |
| 202 | +
|
| 203 | + # nginx routes on the normalised path, which is B's logs root, and |
| 204 | + # forwards the raw one; the proxy must refuse it, not read B. |
| 205 | + code=$(call "$LOGS_A/../../../../tenant-test-logsb/logs/select/logsql/query?query=*" --path-as-is --header "$auth") |
| 206 | + expect 400 "$code" "dot segments toward B" |
| 207 | +
|
| 208 | + code=$(call "$LOGS_A/delete?query=*" --header "$auth") |
| 209 | + expect 404 "$code" "an endpoint outside the allowlist" |
| 210 | +
|
| 211 | + - name: tail-outlives-a-minute |
| 212 | + try: |
| 213 | + - script: |
| 214 | + # 2 x 30s forwards + 30s token + 3 x 15s ingests + the 70s hold + |
| 215 | + # the 30s delivery poll; the tail itself is cut at 150s. |
| 216 | + timeout: 6m |
| 217 | + content: | |
| 218 | + set -eu |
| 219 | + . ./lib.sh |
| 220 | + tail_out=$(mktemp) |
| 221 | + trap 'kill "${tail_pid:-}" 2>/dev/null || true; for p in ${pf_pids:-}; do kill "$p" 2>/dev/null || true; done; rm -f "$body" "$tail_out"' EXIT |
| 222 | + forward tenant-root vlselect-generic 19471:9471 |
| 223 | + forward cozy-dashboard cozy-dashboard-console 18080:8080 |
| 224 | + token=$(k -n tenant-test-logsa create token logs-viewer --duration=10m) |
| 225 | +
|
| 226 | + timeout 150 curl --silent --no-buffer --header "Authorization: Bearer $token" \ |
| 227 | + "$CONSOLE$LOGS_A/tail?query=*" > "$tail_out" & |
| 228 | + tail_pid=$! |
| 229 | + ingest tenant-test-logsa e2e-tail-early |
| 230 | + ingest tenant-test-logsb e2e-tail-b |
| 231 | + # TODO(e2e-replace-fixed-timeouts): the hold is the property under |
| 232 | + # test, not a wait for a condition: a tail must survive a minute |
| 233 | + # without the client sending anything. No event can replace it. |
| 234 | + sleep 70 |
| 235 | + if ! kill -0 "$tail_pid" 2>/dev/null; then |
| 236 | + echo "FAIL: the tail closed before 70s" >&2 |
| 237 | + cat "$tail_out" >&2 |
| 238 | + exit 1 |
| 239 | + fi |
| 240 | + ingest tenant-test-logsa e2e-tail-late |
| 241 | + # vlselect re-reads every second, five seconds behind now. |
| 242 | + timeout 30 sh -ec "until grep -q e2e-tail-late '$tail_out'; do sleep 1; done" || { |
| 243 | + echo "FAIL: a line written at 70s did not reach the open tail" >&2 |
| 244 | + cat "$tail_out" >&2 |
| 245 | + exit 1 |
| 246 | + } |
| 247 | + grep -q e2e-tail-early "$tail_out" || { echo "FAIL: the early line is missing from the tail" >&2; exit 1; } |
| 248 | + if grep -q e2e-tail-b "$tail_out"; then |
| 249 | + echo "FAIL: the tail of A delivered B's line" >&2 |
| 250 | + exit 1 |
| 251 | + fi |
0 commit comments