|
| 1 | +#!/usr/bin/env bash |
| 2 | +# |
| 3 | +# Run one CI command under a wall-clock deadline, and when it blows the |
| 4 | +# deadline, record what is still alive before killing it (#544). |
| 5 | +# |
| 6 | +# scripts/ci/hang-watchdog.sh <seconds> <command> [args...] |
| 7 | +# |
| 8 | +# A step-level `timeout-minutes` already stops a hang, but it does so silently: |
| 9 | +# the log ends, then "The operation was canceled", and the runner reaps the |
| 10 | +# leftover process during job cleanup without saying what it was doing. That is |
| 11 | +# all #544 had to go on — `npm run ant:download` printed its last line, called |
| 12 | +# `process.exit(0)`, and sat for 19 minutes. This wrapper fails the step at the |
| 13 | +# deadline instead, after printing: |
| 14 | +# |
| 15 | +# - the process table (`ps`), so the log names what is still running — |
| 16 | +# npm, node, a tar/unzip child, something else holding the step's pipe; |
| 17 | +# - a stack sample of every process in the command's process tree: `sample` |
| 18 | +# on macOS (it ships with the OS), the kernel wait channel and stack on |
| 19 | +# Linux. A native hang (a thread join, a blocked syscall) is invisible |
| 20 | +# from JavaScript, so the stack is the only place it shows. |
| 21 | +# |
| 22 | +# Exit status: the command's own status, or 124 (as `timeout` uses) when the |
| 23 | +# deadline expired. The command runs in its own process group so the whole |
| 24 | +# tree, not just the top process, is signalled. |
| 25 | +# |
| 26 | +# Pair it with a step `timeout-minutes` a little above the deadline, as a |
| 27 | +# backstop for a watchdog that cannot itself run (e.g. the runner losing the |
| 28 | +# machine). |
| 29 | + |
| 30 | +set -uo pipefail |
| 31 | + |
| 32 | +if [ "$#" -lt 2 ] || ! [[ "$1" =~ ^[0-9]+$ ]]; then |
| 33 | + echo "usage: $0 <seconds> <command> [args...]" >&2 |
| 34 | + exit 2 |
| 35 | +fi |
| 36 | + |
| 37 | +deadline="$1" |
| 38 | +shift |
| 39 | + |
| 40 | +# `set -m` puts the background command in a process group of its own, whose id |
| 41 | +# is its pid, so `kill -- -<pid>` reaches every descendant. |
| 42 | +set -m |
| 43 | +"$@" & |
| 44 | +pid=$! |
| 45 | +set +m |
| 46 | + |
| 47 | +# Every pid whose ancestry leads back to $1 (including $1 itself). |
| 48 | +descendants() { |
| 49 | + local root="$1" |
| 50 | + ps -axo pid=,ppid= 2>/dev/null | awk -v root="$root" ' |
| 51 | + { parent[$1] = $2; pids[NR] = $1 } |
| 52 | + END { |
| 53 | + for (i in pids) { |
| 54 | + p = pids[i] |
| 55 | + for (q = p; q != "" && q != 0 && q != 1; q = parent[q]) { |
| 56 | + if (q == root) { print p; break } |
| 57 | + } |
| 58 | + } |
| 59 | + }' |
| 60 | +} |
| 61 | + |
| 62 | +diagnose() { |
| 63 | + echo "::group::hang-watchdog: process table" |
| 64 | + ps -axo pid,ppid,pgid,etime,stat,%cpu,command 2>/dev/null || ps -ef || ps |
| 65 | + echo "::endgroup::" |
| 66 | + local p |
| 67 | + for p in $(descendants "$pid"); do |
| 68 | + echo "::group::hang-watchdog: stack of pid $p ($(ps -o command= -p "$p" 2>/dev/null))" |
| 69 | + if command -v sample >/dev/null 2>&1; then |
| 70 | + # macOS: 2 s of samples, every thread, symbolicated. Node's release |
| 71 | + # binaries use the hardened runtime, which can refuse an unprivileged |
| 72 | + # sampler; hosted runners have passwordless sudo for the retry. |
| 73 | + { sample "$p" 2 -mayDie 2>&1 || sudo -n sample "$p" 2 -mayDie 2>&1; } | sed -n '1,400p' |
| 74 | + elif [ -r "/proc/$p/status" ]; then |
| 75 | + grep -E '^(State|Threads):' "/proc/$p/status" |
| 76 | + for t in /proc/"$p"/task/*; do |
| 77 | + echo "thread ${t##*/}: wchan=$(cat "$t/wchan" 2>/dev/null) $(cat "$t/stat" 2>/dev/null | awk '{print "state=" $3}')" |
| 78 | + cat "$t/stack" 2>/dev/null | sed 's/^/ /' |
| 79 | + done |
| 80 | + else |
| 81 | + echo "(no stack sampler on this platform)" |
| 82 | + fi |
| 83 | + echo "::endgroup::" |
| 84 | + done |
| 85 | +} |
| 86 | + |
| 87 | +start=$SECONDS |
| 88 | +while kill -0 "$pid" 2>/dev/null; do |
| 89 | + if [ $((SECONDS - start)) -ge "$deadline" ]; then |
| 90 | + echo "::error title=hang-watchdog::'$*' still running after ${deadline}s; dumping processes and killing it (see #544)" |
| 91 | + diagnose |
| 92 | + kill -TERM -- "-$pid" 2>/dev/null || kill -TERM "$pid" 2>/dev/null |
| 93 | + sleep 5 |
| 94 | + kill -KILL -- "-$pid" 2>/dev/null || kill -KILL "$pid" 2>/dev/null |
| 95 | + wait "$pid" 2>/dev/null |
| 96 | + exit 124 |
| 97 | + fi |
| 98 | + sleep 1 |
| 99 | +done |
| 100 | + |
| 101 | +wait "$pid" |
0 commit comments