|
| 1 | +#!/bin/sh |
| 2 | +# Fail unless every image the packages tree pins is an index serving both |
| 3 | +# linux/amd64 and linux/arm64. |
| 4 | +# |
| 5 | +# Usage: hack/verify-multiarch.sh [--report] [--allowlist <file>] <packages-root> |
| 6 | +# hack/verify-multiarch.sh [--report] [--allowlist <file>] --refs-file <file> |
| 7 | +# --report print the same list but exit 0, for a tree that is not being |
| 8 | +# released (the nightly arm64 build runs it against main) |
| 9 | +# --allowlist repositories allowed to stay single-arch; defaults to |
| 10 | +# hack/multiarch-allowlist |
| 11 | +# --refs-file check the refs listed in <file>, one per line, instead of a |
| 12 | +# tree: what an e2e run's nodes actually pulled, which includes |
| 13 | +# images the tree does not pin by digest. A ref without a digest |
| 14 | +# is resolved by its tag. |
| 15 | +# |
| 16 | +# A release ships one ref per image to clusters of either architecture, so a |
| 17 | +# ref that resolves to a single-platform manifest, or to an index missing one of |
| 18 | +# the two, installs on one of them and fails to pull on the other. First- and |
| 19 | +# third-party refs are checked alike: an arm64 node pulls both. |
| 20 | +# |
| 21 | +# The allowlist holds one repository per line with a mandatory `# reason`, and |
| 22 | +# no tag or digest, so bumping an allowed image needs no edit. Allowed refs are |
| 23 | +# still listed, under their own heading, and in tree mode an entry that |
| 24 | +# matches no ref that failed the check is named as stale. |
| 25 | +# |
| 26 | +# A registry that answers a lookup with a rate limit or a 5xx is asked again, |
| 27 | +# up to three times, VERIFY_MULTIARCH_RETRY_DELAY seconds (default 10) times |
| 28 | +# the attempt apart; a ref still refused fails as rate-limited, never as |
| 29 | +# single-arch. |
| 30 | +# |
| 31 | +# Exit: 0 all refs pass (or --report), 1 a ref failed, 2 usage or allowlist |
| 32 | +# error. Requires skopeo, jq, and yq (mikefarah) to read a tree. |
| 33 | +set -eu |
| 34 | + |
| 35 | +usage() { echo "usage: verify-multiarch.sh [--report] [--allowlist <file>] <packages-root> | --refs-file <file>" >&2; exit 2; } |
| 36 | + |
| 37 | +report=0 |
| 38 | +allowlist="$(dirname "$0")/multiarch-allowlist" |
| 39 | +refs_file="" |
| 40 | +while [ "$#" -gt 0 ]; do |
| 41 | + case "$1" in |
| 42 | + --report) report=1; shift ;; |
| 43 | + --allowlist) [ "$#" -ge 2 ] || usage; allowlist="$2"; shift 2 ;; |
| 44 | + --refs-file) [ "$#" -ge 2 ] || usage; refs_file="$2"; shift 2 ;; |
| 45 | + -*) usage ;; |
| 46 | + *) break ;; |
| 47 | + esac |
| 48 | +done |
| 49 | +if [ -n "$refs_file" ]; then |
| 50 | + [ "$#" -eq 0 ] || usage |
| 51 | + [ -f "$refs_file" ] || { echo "refs file '$refs_file' does not exist" >&2; exit 2; } |
| 52 | + tools="skopeo jq" |
| 53 | +else |
| 54 | + [ "$#" -eq 1 ] || usage |
| 55 | + ROOT="${1%/}" |
| 56 | + [ -d "$ROOT" ] || { echo "packages-root '$ROOT' is not a directory" >&2; exit 2; } |
| 57 | + tools="skopeo jq yq" |
| 58 | +fi |
| 59 | +[ -f "$allowlist" ] || { echo "allowlist '$allowlist' does not exist" >&2; exit 2; } |
| 60 | +for _tool in $tools; do |
| 61 | + command -v "$_tool" >/dev/null || { echo "$_tool is required" >&2; exit 2; } |
| 62 | +done |
| 63 | + |
| 64 | +# shellcheck source=hack/lib/image-refs.sh |
| 65 | +. "$(dirname "$0")/lib/image-refs.sh" |
| 66 | + |
| 67 | +# "<repo> <reason>" per entry. |
| 68 | +allowed="$(sed -E '/^[[:space:]]*(#|$)/d' "$allowlist" | while IFS= read -r _line; do |
| 69 | + _repo="$(printf '%s' "$_line" | sed -E 's/[[:space:]]*#.*//')" |
| 70 | + _why="$(printf '%s' "$_line" | sed -nE 's/^[^#]*#[[:space:]]*//p')" |
| 71 | + if [ -z "$_why" ] || [ -z "$_repo" ]; then |
| 72 | + echo "::error::allowlist entry '$_line' needs a '# reason'" >&2 |
| 73 | + exit 2 |
| 74 | + fi |
| 75 | + printf '%s %s\n' "$_repo" "$_why" |
| 76 | +done)" || exit 2 |
| 77 | +
|
| 78 | +# A ref can sit inside a container argument, such as kamaji's |
| 79 | +# --migrate-image=<ref>. The packages artifact is an OCI artifact with no |
| 80 | +# platform, so it is left out. The enumeration also emits the bare |
| 81 | +# "<tag>@<digest>" or "<repo>@<digest>" half of a split map next to the full |
| 82 | +# ref; such a fragment is dropped when a ref with a host carries its digest, and |
| 83 | +# kept otherwise, so a ref the gate cannot resolve still fails it. |
| 84 | +if [ -n "$refs_file" ]; then |
| 85 | + refs="$(grep -vE '^[[:space:]]*(#|$)' "$refs_file" | sort -u)" || true |
| 86 | + [ -n "$refs" ] || { echo "no image refs in ${refs_file}" >&2; exit 2; } |
| 87 | +else |
| 88 | + refs="$(collect_image_refs "$ROOT" \ |
| 89 | + | sed -E 's/^--[A-Za-z0-9-]+=//' \ |
| 90 | + | grep -vE '/cozystack-packages(:[^@]*)?@' \ |
| 91 | + | sort -u \ |
| 92 | + | awk -F@ '{ ref[NR] = $0; dig[NR] = $2; if ($1 ~ /\//) full[$2] = 1 } |
| 93 | + END { for (i = 1; i <= NR; i++) if (ref[i] ~ /^[^@]*\// || !(dig[i] in full)) print ref[i] }')" || true |
| 94 | + [ -n "$refs" ] || { echo "no image refs found under ${ROOT}" >&2; exit 2; } |
| 95 | +fi |
| 96 | +
|
| 97 | +bad="" |
| 98 | +ok_single="" |
| 99 | +used="" |
| 100 | +nl=' |
| 101 | +' |
| 102 | +while IFS= read -r ref; do |
| 103 | + name="${ref%@*}" |
| 104 | + last="${name##*/}" |
| 105 | + repo="${name%/*}/${last%%:*}" |
| 106 | + case "$name" in */*) ;; *) repo="${last%%:*}" ;; esac |
| 107 | + # A registry client rejects a ref carrying both a tag and a digest, so a |
| 108 | + # digest ref drops its tag; a tag-only ref resolves as written. |
| 109 | + case "$ref" in |
| 110 | + *@*) target="${repo}@${ref##*@}" ;; |
| 111 | + *) target="$ref" ;; |
| 112 | + esac |
| 113 | + # A rate limit or a 5xx says nothing about the image, so that answer alone is |
| 114 | + # retried, a bounded number of times; any other error is deterministic and |
| 115 | + # fails at once. |
| 116 | + err="$(mktemp)" |
| 117 | + try=1 |
| 118 | + while :; do |
| 119 | + if raw="$(skopeo inspect --raw "docker://${target}" 2>"$err")"; then |
| 120 | + missing="$(printf '%s' "$raw" | jq -r ' |
| 121 | + if .manifests then |
| 122 | + [.manifests[].platform | select(.os == "linux") | .architecture] as $a |
| 123 | + | ["amd64", "arm64"] - $a | map("index without linux/" + .) | join(", ") |
| 124 | + else "single-platform manifest" end')" |
| 125 | + break |
| 126 | + fi |
| 127 | + if ! grep -qiE 'toomanyrequests|too many requests|status:? (429|5[0-9][0-9])|status code (429|5[0-9][0-9])' "$err"; then |
| 128 | + missing="cannot read the manifest: $(head -n 1 "$err")" |
| 129 | + break |
| 130 | + fi |
| 131 | + if [ "$try" -ge 3 ]; then |
| 132 | + missing="rate-limited or unavailable after 3 tries, not checked: $(head -n 1 "$err")" |
| 133 | + break |
| 134 | + fi |
| 135 | + sleep $((try * ${VERIFY_MULTIARCH_RETRY_DELAY:-10})) |
| 136 | + try=$((try + 1)) |
| 137 | + done |
| 138 | + rm -f "$err" |
| 139 | + [ -n "$missing" ] || continue |
| 140 | + # A ref of an allowlisted repository that failed the check, whatever the |
| 141 | + # reason, keeps the entry from being stale; the entry excuses a platform |
| 142 | + # verdict only, since a ref that cannot be read or stays rate-limited is no |
| 143 | + # more usable on amd64 than on arm64. |
| 144 | + entry="$(printf '%s\n' "$allowed" | awk -v r="$repo" '$1 == r { $1 = ""; sub(/^ /, ""); print; exit }')" |
| 145 | + [ -z "$entry" ] || used="${used}${repo}${nl}" |
| 146 | + why="" |
| 147 | + case "$missing" in |
| 148 | + "single-platform manifest"|"index without"*) why="$entry" ;; |
| 149 | + esac |
| 150 | + if [ -n "$why" ]; then |
| 151 | + ok_single="${ok_single} ${ref}: ${missing} (${why})${nl}" |
| 152 | + else |
| 153 | + bad="${bad} ${ref}: ${missing}${nl}" |
| 154 | + fi |
| 155 | +done <<EOF |
| 156 | +$refs |
| 157 | +EOF |
| 158 | +
|
| 159 | +# Only a tree says an entry is dead. A list of pulled images leaves out every |
| 160 | +# image the run did not pull, allowed ones included. |
| 161 | +stale="" |
| 162 | +if [ -z "$refs_file" ]; then |
| 163 | + stale="$(printf '%s\n' "$allowed" | awk 'NF { print $1 }' | while IFS= read -r _repo; do |
| 164 | + printf '%s' "$used" | grep -qxF "$_repo" || printf ' %s\n' "$_repo" |
| 165 | + done)" || true |
| 166 | +fi |
| 167 | +
|
| 168 | +n_bad="$(printf '%s' "$bad" | grep -c . || true)" |
| 169 | +echo "refs that failed (${n_bad}):" |
| 170 | +printf '%s\n' "$bad" |
| 171 | +if [ -n "$ok_single" ]; then |
| 172 | + echo "allowed single-arch:" |
| 173 | + printf '%s\n' "$ok_single" |
| 174 | +fi |
| 175 | +if [ -n "$stale" ]; then |
| 176 | + echo "stale allowlist entries, matching no ref that failed the check:" |
| 177 | + printf '%s\n\n' "$stale" |
| 178 | +fi |
| 179 | +
|
| 180 | +[ "$n_bad" -eq 0 ] || [ "$report" -eq 1 ] || exit 1 |
0 commit comments