Skip to content

Commit aa72a23

Browse files
feat(cozyreport): collect Flux/cert-manager/host context + summary.txt
`hack/cozyreport.sh` now collects materially more signal so a triage artifact can show what is broken rather than just dumping pod logs: - Flux controller logs (helm-controller, source-controller, notification-controller, kustomize-controller, last 2000 lines) - Flux source resources (HelmRepository, OCIRepository, GitRepository, ExternalArtifact) - Decoded Helm storage secrets for non-Ready HRs (`base64 -d | base64 -d | gzip -d`) - Cluster events (all + warning-only filter) - cert-manager Certificate / CertificateRequest / Order / Challenge resources + cert-manager logs - `cozystack-operator` deployment logs (current + previous) - `Application` / `ApplicationDefinition` / `Tenant` resources - Sandbox host context per node: df, free, ps, dmesg, talosctl logs/dmesg/kubelet/containerd New `hack/cozyreport-summary.sh` writes a `summary.txt` at the archive root listing what is broken right now — first thing to read when triaging an artifact. Signed-off-by: Myasnikov Daniil <[email protected]> Assisted-By: Claude <[email protected]>
1 parent 5786afe commit aa72a23

2 files changed

Lines changed: 161 additions & 1 deletion

File tree

‎hack/cozyreport-summary.sh‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
#!/bin/sh
2+
# Emit a human-readable summary of "what is broken" to a single file.
3+
# Reads the live cluster (not the report dir) so it can use kubectl JSONPath.
4+
# Usage: cozyreport-summary.sh > summary.txt
5+
set -eu
6+
7+
echo "# Cozystack E2E Diagnostic Summary"
8+
echo "Generated: $(date -Iseconds)"
9+
echo
10+
11+
echo "## HelmReleases not Ready"
12+
echo
13+
kubectl get hr -A --no-headers 2>/dev/null \
14+
| awk '$4 != "True" {printf " %s/%s — %s\n", $1, $2, $5}' \
15+
| head -40
16+
echo
17+
18+
echo "## Pods not Running/Succeeded"
19+
echo
20+
kubectl get pod -A --no-headers 2>/dev/null \
21+
| awk '$4 !~ /Running|Succeeded|Completed/ {printf " %s/%s — %s (restarts=%s, age=%s)\n", $1, $2, $4, $5, $6}' \
22+
| head -40
23+
echo
24+
25+
echo "## ImagePullBackOff / ErrImagePull"
26+
echo
27+
kubectl get pod -A --no-headers 2>/dev/null \
28+
| awk '$4 ~ /ImagePullBackOff|ErrImagePull/ {printf " %s/%s — %s\n", $1, $2, $4}'
29+
echo
30+
31+
echo "## OOMKilled in last 30 min"
32+
echo
33+
kubectl get events -A --field-selector reason=OOMKilling --sort-by=.lastTimestamp 2>/dev/null \
34+
| tail -20
35+
echo
36+
37+
echo "## Recent Warning events (top 30)"
38+
echo
39+
kubectl get events -A --field-selector type=Warning --sort-by=.lastTimestamp 2>/dev/null \
40+
| tail -30
41+
echo
42+
43+
echo "## cert-manager: Certificates not Ready"
44+
echo
45+
if kubectl get crd certificates.cert-manager.io >/dev/null 2>&1; then
46+
kubectl get certificates.cert-manager.io -A --no-headers 2>/dev/null \
47+
| awk '$3 != "True" {printf " %s/%s — Ready=%s\n", $1, $2, $3}'
48+
fi
49+
echo
50+
51+
echo "## Flux Sources not Ready"
52+
echo
53+
for kind in helmrepositories.source.toolkit.fluxcd.io ocirepositories.source.toolkit.fluxcd.io gitrepositories.source.toolkit.fluxcd.io; do
54+
kubectl get $kind -A --no-headers 2>/dev/null \
55+
| awk -v k="${kind%%.*}" '$4 != "True" {printf " %s %s/%s — Ready=%s\n", k, $1, $2, $4}'
56+
done
57+
echo
58+
59+
echo "## Storage: PVCs not Bound, PVs not Bound"
60+
echo
61+
kubectl get pvc -A --no-headers 2>/dev/null | awk '$3 != "Bound" {printf " PVC %s/%s — %s\n", $1, $2, $3}'
62+
kubectl get pv --no-headers 2>/dev/null | awk '$5 != "Bound" {printf " PV %s — %s\n", $1, $5}'
63+
echo
64+
65+
echo "## Node Conditions"
66+
kubectl get nodes -o custom-columns=NAME:.metadata.name,READY:.status.conditions[?\(@.type==\"Ready\"\)].status,DISK:.status.conditions[?\(@.type==\"DiskPressure\"\)].status,MEM:.status.conditions[?\(@.type==\"MemoryPressure\"\)].status 2>/dev/null

‎hack/cozyreport.sh‎

Lines changed: 95 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,17 +9,60 @@ command -V kubectl >/dev/null || exit $?
99
command -V tar >/dev/null || exit $?
1010

1111
# -- cozystack module
12-
1312
echo "Collecting Cozystack information..."
1413
mkdir -p $REPORT_DIR/cozystack
1514
kubectl get deploy -n cozy-system cozystack -o jsonpath='{.spec.template.spec.containers[0].image}' > $REPORT_DIR/cozystack/image.txt 2>&1
15+
if kubectl get deploy -n cozy-system cozystack-operator >/dev/null 2>&1; then
16+
kubectl logs -n cozy-system deploy/cozystack-operator --tail=2000 > $REPORT_DIR/cozystack/operator.log 2>&1
17+
kubectl logs -n cozy-system deploy/cozystack-operator --tail=2000 --previous > $REPORT_DIR/cozystack/operator-previous.log 2>&1 || true
18+
fi
1619
kubectl get cm -n cozy-system --no-headers | awk '$1 ~ /^cozystack/' |
1720
while read NAME _; do
1821
DIR=$REPORT_DIR/cozystack/configs
1922
mkdir -p $DIR
2023
kubectl get cm -n cozy-system $NAME -o yaml > $DIR/$NAME.yaml 2>&1
2124
done
2225

26+
# -- flux module
27+
echo "Collecting Flux controller state..."
28+
mkdir -p $REPORT_DIR/flux
29+
for ctrl in helm-controller source-controller notification-controller kustomize-controller; do
30+
if kubectl get deploy -n cozy-fluxcd $ctrl >/dev/null 2>&1; then
31+
kubectl logs -n cozy-fluxcd deploy/$ctrl --tail=2000 > $REPORT_DIR/flux/$ctrl.log 2>&1
32+
kubectl logs -n cozy-fluxcd deploy/$ctrl --tail=2000 --previous > $REPORT_DIR/flux/$ctrl-previous.log 2>&1 || true
33+
fi
34+
done
35+
36+
echo "Collecting Flux sources..."
37+
for kind in helmrepositories.source.toolkit.fluxcd.io ocirepositories.source.toolkit.fluxcd.io gitrepositories.source.toolkit.fluxcd.io externalartifacts.source.toolkit.fluxcd.io; do
38+
short=${kind%%.*}
39+
kubectl get $kind -A > $REPORT_DIR/flux/$short.txt 2>&1
40+
kubectl get $kind -A -o yaml > $REPORT_DIR/flux/$short.yaml 2>&1
41+
done
42+
43+
# -- cert-manager module
44+
if kubectl get crd certificates.cert-manager.io >/dev/null 2>&1; then
45+
echo "Collecting cert-manager state..."
46+
DIR=$REPORT_DIR/cert-manager
47+
mkdir -p $DIR
48+
kubectl get certificates.cert-manager.io -A > $DIR/certificates.txt 2>&1
49+
kubectl get certificaterequests.cert-manager.io -A > $DIR/certificaterequests.txt 2>&1
50+
kubectl get orders.acme.cert-manager.io -A > $DIR/orders.txt 2>&1
51+
kubectl get challenges.acme.cert-manager.io -A > $DIR/challenges.txt 2>&1
52+
# Per non-Ready cert: full yaml + describe
53+
kubectl get certificates.cert-manager.io -A --no-headers 2>/dev/null | awk '$3 != "True"' | \
54+
while read NAMESPACE NAME _; do
55+
cdir=$DIR/certificates/$NAMESPACE/$NAME
56+
mkdir -p $cdir
57+
kubectl get certificates.cert-manager.io -n $NAMESPACE $NAME -o yaml > $cdir/cert.yaml 2>&1
58+
kubectl describe certificates.cert-manager.io -n $NAMESPACE $NAME > $cdir/describe.txt 2>&1
59+
done
60+
if kubectl get deploy -n cozy-cert-manager cert-manager >/dev/null 2>&1; then
61+
kubectl logs -n cozy-cert-manager deploy/cert-manager --tail=2000 > $DIR/cert-manager.log 2>&1
62+
kubectl logs -n cozy-cert-manager deploy/cert-manager-webhook --tail=2000 > $DIR/cert-manager-webhook.log 2>&1
63+
fi
64+
fi
65+
2366
# -- kubernetes module
2467

2568
echo "Collecting Kubernetes information..."
@@ -46,6 +89,13 @@ kubectl get ns --no-headers | awk '$2 != "Active"' |
4689
kubectl describe ns $NAME > $DIR/describe.txt 2>&1
4790
done
4891

92+
echo "Collecting events..."
93+
kubectl get events -A --sort-by=.lastTimestamp > $REPORT_DIR/kubernetes/events.txt 2>&1
94+
# Filter to warning-class and recent for quick triage
95+
kubectl get events -A --sort-by=.lastTimestamp \
96+
-o jsonpath='{range .items[?(@.type!="Normal")]}{.lastTimestamp}{"\t"}{.involvedObject.namespace}/{.involvedObject.kind}/{.involvedObject.name}{"\t"}{.reason}{"\t"}{.message}{"\n"}{end}' \
97+
> $REPORT_DIR/kubernetes/events-warnings.txt 2>&1
98+
4999
echo "Collecting helmreleases..."
50100
kubectl get hr -A > $REPORT_DIR/kubernetes/helmreleases.txt 2>&1
51101
kubectl get hr -A --no-headers | awk '$4 != "True"' | \
@@ -54,6 +104,13 @@ kubectl get hr -A --no-headers | awk '$4 != "True"' | \
54104
mkdir -p $DIR
55105
kubectl get hr -n $NAMESPACE $NAME -o yaml > $DIR/hr.yaml 2>&1
56106
kubectl describe hr -n $NAMESPACE $NAME > $DIR/describe.txt 2>&1
107+
# Helm storage secrets: latest revision per release.
108+
kubectl get secret -n $NAMESPACE -l owner=helm,name=$NAME --sort-by='.metadata.creationTimestamp' --no-headers 2>/dev/null | \
109+
tail -1 | awk '{print $1}' | while read SECRET; do
110+
[ -z "$SECRET" ] && continue
111+
kubectl get secret -n $NAMESPACE $SECRET -o jsonpath='{.data.release}' 2>/dev/null \
112+
| base64 -d | base64 -d | gzip -d > $DIR/helm-release.json 2>&1 || true
113+
done
57114
done
58115

59116
echo "Collecting packages..."
@@ -76,6 +133,23 @@ kubectl get packagesources --no-headers | awk '$3 != "True"' | \
76133
kubectl describe packagesource $NAME > $DIR/describe.txt 2>&1
77134
done
78135

136+
echo "Collecting cozystack apps..."
137+
DIR=$REPORT_DIR/cozystack-apps
138+
mkdir -p $DIR
139+
for kind in applications.apps.cozystack.io applicationdefinitions.apps.cozystack.io tenants.apps.cozystack.io; do
140+
short=${kind%%.*}
141+
if kubectl get crd $kind >/dev/null 2>&1; then
142+
kubectl get $kind -A > $DIR/$short.txt 2>&1
143+
kubectl get $kind -A --no-headers 2>/dev/null | awk 'NF >= 3 && $NF != "True" && $NF != "Ready"' | \
144+
while read NAMESPACE NAME _; do
145+
d=$DIR/$short/$NAMESPACE/$NAME
146+
mkdir -p $d
147+
kubectl get $kind -n $NAMESPACE $NAME -o yaml > $d/$short.yaml 2>&1
148+
kubectl describe $kind -n $NAMESPACE $NAME > $d/describe.txt 2>&1
149+
done
150+
fi
151+
done
152+
79153
echo "Collecting pods..."
80154
kubectl get pod -A -o wide > $REPORT_DIR/kubernetes/pods.txt 2>&1
81155
kubectl get pod -A --no-headers | awk '$4 !~ /Running|Succeeded|Completed/' |
@@ -157,8 +231,28 @@ if kubectl get deploy -n cozy-linstor linstor-controller >/dev/null 2>&1; then
157231
kubectl exec -n cozy-linstor deploy/linstor-controller -- linstor --no-color r l > $DIR/resources.txt 2>&1
158232
fi
159233

234+
# -- sandbox-host module
235+
236+
echo "Collecting sandbox host context..."
237+
DIR=$REPORT_DIR/sandbox-host
238+
mkdir -p $DIR
239+
df -h > $DIR/df.txt 2>&1
240+
free -m > $DIR/free.txt 2>&1
241+
ps auxww > $DIR/ps.txt 2>&1
242+
dmesg | tail -200 > $DIR/dmesg.txt 2>&1 || true
243+
if [ -f /workspace/talosconfig ]; then
244+
for node in 192.168.123.11 192.168.123.12 192.168.123.13; do
245+
talosctl --talosconfig /workspace/talosconfig -n $node dmesg --tail=200 > $DIR/talos-$node-dmesg.txt 2>&1 || true
246+
talosctl --talosconfig /workspace/talosconfig -n $node logs kubelet --tail=500 > $DIR/talos-$node-kubelet.log 2>&1 || true
247+
talosctl --talosconfig /workspace/talosconfig -n $node logs containerd --tail=500 > $DIR/talos-$node-containerd.log 2>&1 || true
248+
done
249+
fi
250+
160251
# -- finalization
161252

253+
echo "Generating summary..."
254+
hack/cozyreport-summary.sh > $REPORT_DIR/summary.txt 2>&1 || true
255+
162256
echo "Creating archive..."
163257
tar -czf $REPORT_NAME.tgz -C $REPORT_PDIR .
164258
echo "Report created: $REPORT_NAME.tgz"

0 commit comments

Comments
 (0)