Change to the NetBSD archive URL rather than the CDN

tgross35 · cuviper · commit c822bf248649 · 2024-07-05T23:00:06.000-07:00
The CDN has been down for a few hours. Switch to an alternative for the time being so we can unblock CI. It appears that the CDN is quite a bit faster, so we will likely want to switch back when available. (cherry picked from commit 42ac903)
diff --git a/src/ci/docker/host-x86_64/dist-x86_64-netbsd/build-netbsd-toolchain.sh b/src/ci/docker/host-x86_64/dist-x86_64-netbsd/build-netbsd-toolchain.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 # ignore-tidy-linelength
 
-set -ex
+set -eux
 
 hide_output() {
   set +x
@@ -20,24 +20,54 @@ exit 1
   set -x
 }
 
+# Download, verify SHA512, and remove the downloaded file
+# Usage: <file name> <url> <file sha> <full tar command using fname>
+download() {
+  fname="$1"
+  shift
+  url="$1"
+  shift
+  sha="$1"
+  shift
+
+  curl "$url" -o "$fname"
+  echo "$sha  $fname" | shasum -a 512 --check || exit 1
+  "$@"
+  rm "$fname"
+}
+
 mkdir netbsd
 cd netbsd
 
 mkdir -p /x-tools/x86_64-unknown-netbsd/sysroot
 
 # URL=https://ci-mirrors.rust-lang.org/rustc
 
-SOURCE_URL=https://cdn.netbsd.org/pub/NetBSD/NetBSD-9.0/source/sets
-curl $SOURCE_URL/src.tgz | tar xzf -
-curl $SOURCE_URL/gnusrc.tgz | tar xzf -
-curl $SOURCE_URL/sharesrc.tgz | tar xzf -
-curl $SOURCE_URL/syssrc.tgz | tar xzf -
-
-BINARY_URL=https://cdn.netbsd.org/pub/NetBSD/NetBSD-9.0/amd64/binary/sets
-curl $BINARY_URL/base.tar.xz | \
-  tar xJf - -C /x-tools/x86_64-unknown-netbsd/sysroot ./usr/include ./usr/lib ./lib
-curl $BINARY_URL/comp.tar.xz | \
-  tar xJf - -C /x-tools/x86_64-unknown-netbsd/sysroot ./usr/include ./usr/lib
+# Hashes come from https://cdn.netbsd.org/pub/NetBSD/security/hashes/NetBSD-9.0_hashes.asc
+SRC_SHA=2c791ae009a6929c6fc893ec5df7e62910ee8207e0b2159d6937309c03efe175b6ae1e445829a13d041b6851334ad35c521f2fa03c97675d4a05f1fafe58ede0
+GNUSRC_SHA=3710085a73feecf6a843415271ec794c90146b03f6bbd30f07c9e0c79febf8995d557e40194f1e05db655e4f5ef2fae97563f8456fceaae65d4ea98857a83b1c
+SHARESRC_SHA=f080776ed82c3ac5d6272dee39746f87897d8e6984996caf5bf6d87bf11d9c9e0c1ad5c437c21258bd278bb6fd76974946e878f548517885f71c556096231369
+SYSSRC_SHA=60b9ddf4cc6402256473e2e1eefeabd9001aa4e205208715ecc6d6fc3f5b400e469944580077271b8e80562a4c2f601249e69e07a504f46744e0c50335f1cbf1
+BASE_SHA=b5926b107cebf40c3c19b4f6cd039b610987dd7f819e7cdde3bd1e5230a856906e7930b15ab242d52ced9f0bda01d574be59488b8dbb95fa5df2987d0a70995f
+COMP_SHA=38ea54f30d5fc2afea87e5096f06873e00182789e8ad9cec0cb3e9f7c538c1aa4779e63fd401a36ba02676158e83fa5c95e8e87898db59c1914fb206aecd82d2
+
+# FIXME: the archive URL is being used temporarily while the CDN is down.
+# We should serve this from our own CDN
+# SOURCE_URL=https://cdn.netbsd.org/pub/NetBSD/NetBSD-9.0/source/sets
+SOURCE_URL=http://archive.netbsd.org/pub/NetBSD-archive/NetBSD-9.0/source/sets
+download src.tgz "$SOURCE_URL/src.tgz" "$SRC_SHA" tar xzf src.tgz
+download gnusrc.tgz "$SOURCE_URL/gnusrc.tgz" "$GNUSRC_SHA" tar xzf gnusrc.tgz
+download sharesrc.tgz "$SOURCE_URL/sharesrc.tgz" "$SHARESRC_SHA" tar xzf sharesrc.tgz
+download syssrc.tgz "$SOURCE_URL/syssrc.tgz" "$SYSSRC_SHA" tar xzf syssrc.tgz
+
+# FIXME: the archive URL is being used temporarily while the CDN is down.
+# We should serve this from our own CDN
+# BINARY_URL=https://cdn.netbsd.org/pub/NetBSD/NetBSD-9.0/amd64/binary/sets
+BINARY_URL=http://archive.netbsd.org/pub/NetBSD-archive/NetBSD-9.0/amd64/binary/sets
+download base.tar.xz "$BINARY_URL/base.tar.xz" "$BASE_SHA" \
+  tar xJf base.tar.xz -C /x-tools/x86_64-unknown-netbsd/sysroot ./usr/include ./usr/lib ./lib
+download comp.tar.xz "$BINARY_URL/comp.tar.xz" "$COMP_SHA" \
+  tar xJf comp.tar.xz -C /x-tools/x86_64-unknown-netbsd/sysroot ./usr/include ./usr/lib
 
 cd usr/src
 
