Skip to content

Commit f663d56

Browse files
am11am11
authored
Add curl support in build-rootfs.sh (#14744)
There are systems which come preinstalled with wget, curl, both or none. To make things smoother, this PR adds curl support side-by-side wget. We have this flexibility in other scripts as well.
1 parent 9fc042c commit f663d56

1 file changed

Lines changed: 102 additions & 20 deletions

File tree

‎eng/common/cross/build-rootfs.sh‎

Lines changed: 102 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -451,13 +451,32 @@ fi
451451
mkdir -p "$__RootfsDir"
452452
__RootfsDir="$( cd "$__RootfsDir" && pwd )"
453453

454+
__hasWget=
455+
ensureDownloadTool()
456+
{
457+
if command -v wget &> /dev/null; then
458+
__hasWget=1
459+
elif command -v curl &> /dev/null; then
460+
__hasWget=0
461+
else
462+
>&2 echo "ERROR: either wget or curl is required by this script."
463+
exit 1
464+
fi
465+
}
466+
454467
if [[ "$__CodeName" == "alpine" ]]; then
455468
__ApkToolsVersion=2.12.11
456469
__ApkToolsDir="$(mktemp -d)"
457470
__ApkKeysDir="$(mktemp -d)"
458-
459471
arch="$(uname -m)"
460-
wget "https://gitlab.alpinelinux.org/api/v4/projects/5/packages/generic/v$__ApkToolsVersion/$arch/apk.static" -P "$__ApkToolsDir"
472+
473+
ensureDownloadTool()
474+
475+
if [[ "$__hasWget" == 1 ]]; then
476+
wget -P "$__ApkToolsDir" "https://gitlab.alpinelinux.org/api/v4/projects/5/packages/generic/v$__ApkToolsVersion/$arch/apk.static"
477+
else
478+
curl -SLO --create-dirs --output-dir "$__ApkToolsDir" "https://gitlab.alpinelinux.org/api/v4/projects/5/packages/generic/v$__ApkToolsVersion/$arch/apk.static"
479+
fi
461480
if [[ "$arch" == "x86_64" ]]; then
462481
__ApkToolsSHA512SUM="53e57b49230da07ef44ee0765b9592580308c407a8d4da7125550957bb72cb59638e04f8892a18b584451c8d841d1c7cb0f0ab680cc323a3015776affaa3be33"
463482
elif [[ "$arch" == "aarch64" ]]; then
@@ -520,12 +539,23 @@ if [[ "$__CodeName" == "alpine" ]]; then
520539
elif [[ "$__CodeName" == "freebsd" ]]; then
521540
mkdir -p "$__RootfsDir"/usr/local/etc
522541
JOBS=${MAXJOBS:="$(getconf _NPROCESSORS_ONLN)"}
523-
wget -O - "https://download.freebsd.org/ftp/releases/${__FreeBSDArch}/${__FreeBSDMachineArch}/${__FreeBSDBase}/base.txz" | tar -C "$__RootfsDir" -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version
542+
543+
ensureDownloadTool()
544+
545+
if [[ "$__hasWget" == 1 ]]; then
546+
wget -O- "https://download.freebsd.org/ftp/releases/${__FreeBSDArch}/${__FreeBSDMachineArch}/${__FreeBSDBase}/base.txz" | tar -C "$__RootfsDir" -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version
547+
else
548+
curl -SL "https://download.freebsd.org/ftp/releases/${__FreeBSDArch}/${__FreeBSDMachineArch}/${__FreeBSDBase}/base.txz" | tar -C "$__RootfsDir" -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version
549+
fi
524550
echo "ABI = \"FreeBSD:${__FreeBSDABI}:${__FreeBSDMachineArch}\"; FINGERPRINTS = \"${__RootfsDir}/usr/share/keys\"; REPOS_DIR = [\"${__RootfsDir}/etc/pkg\"]; REPO_AUTOUPDATE = NO; RUN_SCRIPTS = NO;" > "${__RootfsDir}"/usr/local/etc/pkg.conf
525551
echo "FreeBSD: { url: \"pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly\", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"${__RootfsDir}/usr/share/keys/pkg\", enabled: yes }" > "${__RootfsDir}"/etc/pkg/FreeBSD.conf
526552
mkdir -p "$__RootfsDir"/tmp
527553
# get and build package manager
528-
wget -O - "https://github.com/freebsd/pkg/archive/${__FreeBSDPkg}.tar.gz" | tar -C "$__RootfsDir"/tmp -zxf -
554+
if [[ "$__hasWget" == 1 ]]; then
555+
wget -O- "https://github.com/freebsd/pkg/archive/${__FreeBSDPkg}.tar.gz" | tar -C "$__RootfsDir"/tmp -zxf -
556+
else
557+
curl -SL "https://github.com/freebsd/pkg/archive/${__FreeBSDPkg}.tar.gz" | tar -C "$__RootfsDir"/tmp -zxf -
558+
fi
529559
cd "$__RootfsDir/tmp/pkg-${__FreeBSDPkg}"
530560
# needed for install to succeed
531561
mkdir -p "$__RootfsDir"/host/etc
@@ -539,15 +569,30 @@ elif [[ "$__CodeName" == "illumos" ]]; then
539569
mkdir "$__RootfsDir/tmp"
540570
pushd "$__RootfsDir/tmp"
541571
JOBS=${MAXJOBS:="$(getconf _NPROCESSORS_ONLN)"}
572+
573+
ensureDownloadTool()
574+
542575
echo "Downloading sysroot."
543-
wget -O - https://github.com/illumos/sysroot/releases/download/20181213-de6af22ae73b-v1/illumos-sysroot-i386-20181213-de6af22ae73b-v1.tar.gz | tar -C "$__RootfsDir" -xzf -
576+
if [[ "$__hasWget" == 1 ]]; then
577+
wget -O- https://github.com/illumos/sysroot/releases/download/20181213-de6af22ae73b-v1/illumos-sysroot-i386-20181213-de6af22ae73b-v1.tar.gz | tar -C "$__RootfsDir" -xzf -
578+
else
579+
curl -SL https://github.com/illumos/sysroot/releases/download/20181213-de6af22ae73b-v1/illumos-sysroot-i386-20181213-de6af22ae73b-v1.tar.gz | tar -C "$__RootfsDir" -xzf -
580+
fi
544581
echo "Building binutils. Please wait.."
545-
wget -O - https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.bz2 | tar -xjf -
582+
if [[ "$__hasWget" == 1 ]]; then
583+
wget -O- https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.bz2 | tar -xjf -
584+
else
585+
curl -SL https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.bz2 | tar -xjf -
586+
fi
546587
mkdir build-binutils && cd build-binutils
547588
../binutils-2.33.1/configure --prefix="$__RootfsDir" --target="${__illumosArch}-sun-solaris2.10" --program-prefix="${__illumosArch}-illumos-" --with-sysroot="$__RootfsDir"
548589
make -j "$JOBS" && make install && cd ..
549590
echo "Building gcc. Please wait.."
550-
wget -O - https://ftp.gnu.org/gnu/gcc/gcc-8.4.0/gcc-8.4.0.tar.xz | tar -xJf -
591+
if [[ "$__hasWget" == 1 ]]; then
592+
wget -O- https://ftp.gnu.org/gnu/gcc/gcc-8.4.0/gcc-8.4.0.tar.xz | tar -xJf -
593+
else
594+
curl -SL https://ftp.gnu.org/gnu/gcc/gcc-8.4.0/gcc-8.4.0.tar.xz | tar -xJf -
595+
fi
551596
CFLAGS="-fPIC"
552597
CXXFLAGS="-fPIC"
553598
CXXFLAGS_FOR_TARGET="-fPIC"
@@ -564,15 +609,23 @@ elif [[ "$__CodeName" == "illumos" ]]; then
564609
fi
565610
BaseUrl="$BaseUrl/packages/SmartOS/trunk/${__illumosArch}/All"
566611
echo "Downloading manifest"
567-
wget "$BaseUrl"
612+
if [[ "$__hasWget" == 1 ]]; then
613+
wget "$BaseUrl"
614+
else
615+
curl -SLO "$BaseUrl"
616+
fi
568617
echo "Downloading dependencies."
569618
read -ra array <<<"$__IllumosPackages"
570619
for package in "${array[@]}"; do
571620
echo "Installing '$package'"
572621
# find last occurrence of package in listing and extract its name
573622
package="$(sed -En '/.*href="('"$package"'-[0-9].*).tgz".*/h;$!d;g;s//\1/p' All)"
574623
echo "Resolved name '$package'"
575-
wget "$BaseUrl"/"$package".tgz
624+
if [[ "$__hasWget" == 1 ]]; then
625+
wget "$BaseUrl"/"$package".tgz
626+
else
627+
curl -SLO "$BaseUrl"/"$package".tgz
628+
fi
576629
ar -x "$package".tgz
577630
tar --skip-old-files -xzf "$package".tmp.tg* -C "$__RootfsDir" 2>/dev/null
578631
done
@@ -581,10 +634,17 @@ elif [[ "$__CodeName" == "illumos" ]]; then
581634
rm -rf "$__RootfsDir"/{tmp,+*}
582635
mkdir -p "$__RootfsDir"/usr/include/net
583636
mkdir -p "$__RootfsDir"/usr/include/netpacket
584-
wget -P "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/bpf.h
585-
wget -P "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/dlt.h
586-
wget -P "$__RootfsDir"/usr/include/netpacket https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/inet/sockmods/netpacket/packet.h
587-
wget -P "$__RootfsDir"/usr/include/sys https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/sys/sdt.h
637+
if [[ "$__hasWget" == 1 ]]; then
638+
wget -P "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/bpf.h
639+
wget -P "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/dlt.h
640+
wget -P "$__RootfsDir"/usr/include/netpacket https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/inet/sockmods/netpacket/packet.h
641+
wget -P "$__RootfsDir"/usr/include/sys https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/sys/sdt.h
642+
else
643+
curl -SLO --create-dirs --output-dir "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/bpf.h
644+
curl -SLO --create-dirs --output-dir "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/dlt.h
645+
curl -SLO --create-dirs --output-dir "$__RootfsDir"/usr/include/netpacket https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/inet/sockmods/netpacket/packet.h
646+
curl -SLO --create-dirs --output-dir "$__RootfsDir"/usr/include/sys https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/sys/sdt.h
647+
fi
588648
elif [[ "$__CodeName" == "haiku" ]]; then
589649
JOBS=${MAXJOBS:="$(getconf _NPROCESSORS_ONLN)"}
590650

@@ -594,9 +654,16 @@ elif [[ "$__CodeName" == "haiku" ]]; then
594654

595655
mkdir "$__RootfsDir/tmp/download"
596656

657+
ensureDownloadTool()
658+
597659
echo "Downloading Haiku package tool"
598660
git clone https://github.com/haiku/haiku-toolchains-ubuntu --depth 1 "$__RootfsDir/tmp/script"
599-
wget -O "$__RootfsDir/tmp/download/hosttools.zip" "$("$__RootfsDir/tmp/script/fetch.sh" --hosttools)"
661+
if [[ "$__hasWget" == 1 ]]; then
662+
wget -O "$__RootfsDir/tmp/download/hosttools.zip" "$("$__RootfsDir/tmp/script/fetch.sh" --hosttools)"
663+
else
664+
curl -SLo "$__RootfsDir/tmp/download/hosttools.zip" "$("$__RootfsDir/tmp/script/fetch.sh" --hosttools)"
665+
fi
666+
600667
unzip -o "$__RootfsDir/tmp/download/hosttools.zip" -d "$__RootfsDir/tmp/bin"
601668

602669
DepotBaseUrl="https://depot.haiku-os.org/__api/v2/pkg/get-pkg"
@@ -609,14 +676,25 @@ elif [[ "$__CodeName" == "haiku" ]]; then
609676
echo "Downloading $package..."
610677
# API documented here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L60
611678
# The schema here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L598
612-
hpkgDownloadUrl="$(wget -qO- --post-data='{"name":"'"$package"'","repositorySourceCode":"haikuports_'$__HaikuArch'","versionType":"LATEST","naturalLanguageCode":"en"}' \
613-
--header='Content-Type:application/json' "$DepotBaseUrl" | jq -r '.result.versions[].hpkgDownloadURL')"
614-
wget -P "$__RootfsDir/tmp/download" "$hpkgDownloadUrl"
679+
if [[ "$__hasWget" == 1 ]]; then
680+
hpkgDownloadUrl="$(wget -qO- --post-data '{"name":"'"$package"'","repositorySourceCode":"haikuports_'$__HaikuArch'","versionType":"LATEST","naturalLanguageCode":"en"}' \
681+
--header 'Content-Type:application/json' "$DepotBaseUrl" | jq -r '.result.versions[].hpkgDownloadURL')"
682+
wget -P "$__RootfsDir/tmp/download" "$hpkgDownloadUrl"
683+
else
684+
hpkgDownloadUrl="$(curl -sSL -XPOST --data '{"name":"'"$package"'","repositorySourceCode":"haikuports_'$__HaikuArch'","versionType":"LATEST","naturalLanguageCode":"en"}' \
685+
--header 'Content-Type:application/json' "$DepotBaseUrl" | jq -r '.result.versions[].hpkgDownloadURL')"
686+
curl -SLO --create-dirs --output-dir "$__RootfsDir/tmp/download" "$hpkgDownloadUrl"
687+
fi
615688
done
616689
for package in haiku haiku_devel; do
617690
echo "Downloading $package..."
618-
hpkgVersion="$(wget -qO- $HpkgBaseUrl | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')"
619-
wget -P "$__RootfsDir/tmp/download" "$HpkgBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg"
691+
if [[ "$__hasWget" == 1 ]]; then
692+
hpkgVersion="$(wget -qO- "$HpkgBaseUrl" | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')"
693+
wget -P "$__RootfsDir/tmp/download" "$HpkgBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg"
694+
else
695+
hpkgVersion="$(curl -sSL "$HpkgBaseUrl" | sed -n 's/^.*version: "\([^"]*\)".*$/\1/p')"
696+
curl -SLO --create-dirs --output-dir "$__RootfsDir/tmp/download" "$HpkgBaseUrl/packages/$package-$hpkgVersion-1-$__HaikuArch.hpkg"
697+
fi
620698
done
621699

622700
# Set up the sysroot
@@ -629,7 +707,11 @@ elif [[ "$__CodeName" == "haiku" ]]; then
629707

630708
# Download buildtools
631709
echo "Downloading Haiku buildtools"
632-
wget -O "$__RootfsDir/tmp/download/buildtools.zip" "$("$__RootfsDir/tmp/script/fetch.sh" --buildtools --arch=$__HaikuArch)"
710+
if [[ "$__hasWget" == 1 ]]; then
711+
wget -O "$__RootfsDir/tmp/download/buildtools.zip" "$("$__RootfsDir/tmp/script/fetch.sh" --buildtools --arch=$__HaikuArch)"
712+
else
713+
curl -SLo "$__RootfsDir/tmp/download/buildtools.zip" "$("$__RootfsDir/tmp/script/fetch.sh" --buildtools --arch=$__HaikuArch)"
714+
fi
633715
unzip -o "$__RootfsDir/tmp/download/buildtools.zip" -d "$__RootfsDir"
634716

635717
# Cleaning up temporary files

0 commit comments

Comments
 (0)