From fc304cb7499e52cfe2ee4fd68f3119a300bc8a64 Mon Sep 17 00:00:00 2001 From: Valentin Volkl Date: Thu, 12 Dec 2024 22:35:19 +0100 Subject: [PATCH] makedist: update cvmfs package repo urls with fallback Adds a fallback URL for the cvmfs package download. makedist currently looks for the latest package in the listing returned by package servers; since the s3 bucket we are mainly using (and which is the most reliable) does not support listings, this mechanism now relies on reading a small file .latest_cvmfs_version_prod (https://cvmrepo.web.cern.ch/cvmrepo/.latest_cvmfs_version_prod) containing a string with the latest version. I'll add this to our testing to ensure it is kept up to date. Obviously this won't work if you wanted to add mirrors maintained by a third party - one possibility would be to hardcode the latest version in the script. The downloads for the non-default config packages is kept the same, except for a small fix for the egi download. The ecsft.cern.ch url is removed alltogether, as this is the most likely one to cause issues. --- makedist | 95 +++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 63 insertions(+), 32 deletions(-) diff --git a/makedist b/makedist index 834eefe..508aec4 100755 --- a/makedist +++ b/makedist @@ -5,7 +5,8 @@ # Written by Dave Dykstra 17 April 2019 # -SUPPORTEDTYPES="rhel7-x86_64 rhel8-aarch64 rhel8-x86_64 rhel8-ppc64le rhel9-aarch64 rhel9-x86_64 rhel9-ppc64le suse15-x86_64" +#TODO: re-add support for suse15-x86_64 +SUPPORTEDTYPES="rhel7-x86_64 rhel8-aarch64 rhel8-x86_64 rhel8-ppc64le rhel9-aarch64 rhel9-x86_64 rhel9-ppc64le" supportedtypes() { typeset LASTDISTRO="" @@ -75,6 +76,17 @@ distroname() { fi } +# return the distroname as encountered in yum package repo URLS, +# i.e https://cvmrepo.web.cern.ch/cvmrepo/yum/cvmfs/EL/9/ +distroname_yum() { + case " $1 " in + *" rhel "*) echo EL;; + *" suse "*) echo suse;; + *) echo "Distro name $1 not supported" >&2 + exit 2;; + esac +} + distroversion() { if [ -n "$MACHTYPE" ]; then echo "$MACHTYPE"|sed 's/^[^0-9]*\([0-9]*\)-.*/\1/' @@ -100,6 +112,7 @@ distroarch() { } DISTRO="`distroname`" +DISTRO_YUM="`distroname_yum $DISTRO`" VERS="`distroversion`" ARCH="`distroarch`" @@ -183,17 +196,28 @@ case $1 in REL=23-main fi REPO=release - BASEURL="https://repo.opensciencegrid.org/osg/$REL/el$EL/$REPO/x86_64";; + BASEURL="https://repo.opensciencegrid.org/osg/$REL/el$EL/$REPO/x86_64" + BASELIST="`curl -Ls $BASEURL/|grep "cvmfs-"|sed 's/.*href="//;s/".*//'`";; egi) OS=centos$EL - BASEURL="https://repository.egi.eu/sw/production/umd/4/$OS/x86_64/updates";; + if [ $EL -gt 8 ]; then + BASEURL="https://repository.egi.eu/sw/production/umd/5/al9/release/x86_64/" + elif [ $EL -eq 7 ]; then + BASEURL="https://repository.egi.eu/sw/production/umd/4/centos7/x86_64/updates" + else + echo "ERROR: unsupported OS for egi! only centos7, almalinux9 supported." + exit 1 + fi + BASELIST="`curl -Ls $BASEURL/|grep "cvmfs-"|sed 's/.*href="//;s/".*//'`";; default|none) INCLUDEHELPER=false - BASEURL="https://cvmrepo.web.cern.ch/cvmrepo/yum/cvmfs/EL/$EL/x86_64";; + BASEURL="https://cvmrepo.web.cern.ch/cvmrepo/yum/cvmfs/EL/$EL/x86_64" + BASEURL_MIRROR1="https://cvmrepo.web.cern.ch/cvmrepo/yum/cvmfs/EL/$EL/x86_64";; *) usage;; esac DISTTYPE=$1 + if [ -d $DIST ]; then echo "$DIST already exists" >&2 exit 1 @@ -232,24 +256,32 @@ if [ "$ARCH" != "x86_64" ]; then # we're looking at x86_64 version repositories for the config rpm INCLUDEHELPER=false fi - -URLS="" -BASELIST="`curl -Ls $BASEURL/|grep "cvmfs-"|sed 's/.*href="//;s/".*//'`" -CVMFSPKG="`echo "$BASELIST"|grep "^cvmfs-[0-9]"|tail -1`" -if [ -z "$CVMFSPKG" ]; then - if [ "$DISTTYPE" = egi ] && [ "$EL" = 8 ]; then - echo "egi's UMD does not yet support rhel8" 2>&1 - else - echo "No cvmfs package found from $BASEURL" >&2 - fi +if [ "$DISTTYPE" = egi ] && [ "$EL" = 8 ]; then + echo "egi's UMD does not yet support rhel8" 2>&1 exit 1 fi +CVMFS_BASEURL_MIRROR1="https://cvmrepo.s3.cern.ch/cvmrepo" +CVMFS_BASEURL_MIRROR2="https://cvmrepo.web.cern.ch/cvmrepo" +CVMFS_BASEURL=$CVMFS_BASEURL_MIRROR1 +CVMFSVERSION="`curl -Ls $CVMFS_BASE_URL/.latest_cvmfs_version_prod`" +if [ -z "CVMFSVERSION" ]; then + echo "WARN: Couldn't find .latest_cvmfs_version_prod in mirror $CVMFS_BASEURL, switching over.." + CVMFS_BASEURL="$CVMFS_BASE_URL_MIRROR2" +fi +CVMFSVERSION="`curl -Ls $CVMFS_BASEURL/.latest_cvmfs_version_prod`" +if [ -z "CVMFSVERSION" ]; then + echo "ERROR: Couldn't find .latest_cvmfs_version_prod in mirror $CVMFS_BASEURL, and no mirrors left! " + exit 1 +fi + +URLS="" CVMFSURL="" +CVMFSRPMURLS="" if [ "$ARCH" = "ppc64le" ]; then # Grab cvmfs package from copr, using yumdownloader to calculate # the URL - CVMFSRPMURL="$(getcoprurl cvmfs)" + CVMFSRPMURLS="$(getcoprurl cvmfs)" if [ -z "$CVMFSRPMURL" ]; then echo "Failed to get $ARCH cvmfs rpm from copr" >&2 exit 1 @@ -258,32 +290,30 @@ if [ "$ARCH" = "ppc64le" ]; then CVMFSRPMNAME="$(basename $CVMFSRPMURL)" CVMFSVERSION="`echo "$CVMFSRPMNAME" 's/.*cvmfs-\([^-]*\)-.*/\1/'`" else - # Now that we can figure out the latest cvmfs version, download the - # corresponding cvmfs packages from CERN instead - CVMFSVERSION="`echo "$CVMFSPKG"|sed 's/.*cvmfs-\([^-]*\)-.*/\1/'`" - CVMFSURL="https://ecsft.cern.ch/dist/cvmfs/cvmfs-$CVMFSVERSION" - CVMFSRPMNAME="`curl -Ls $CVMFSURL/|grep "cvmfs-$CVMFSVERSION-.*$MACH.*\.$ARCH"|sed 's/.*href="//;s/".*//'|tail -1`" - if [ -z "$CVMFSRPMNAME" ]; then - echo "No matching cvmfs rpm found at $CVMFSURL" >&2 - exit 1 - fi - CVMFSRPMURL="$CVMFSURL/$CVMFSRPMNAME" + CVMFSURL="$CVMFS_BASEURL/yum/cvmfs/$DISTRO_YUM/$VERS/$ARCH" + CVMFSRPMNAME="cvmfs-$CVMFSVERSION-1.$MACH.$ARCH.rpm" + CVMFSRPMURLS="$CVMFSURL/$CVMFSRPMNAME" fi -URLS="$CVMFSRPMURL" MINORVERSION="`echo "$CVMFSVERSION"|cut -d. -f2`" if [ "$MINORVERSION" -ge 10 ]; then # include cvmfs-libs - URLS="$URLS $CVMFSURL/`echo $CVMFSRPMNAME|sed 's/cvmfs-/cvmfs-libs-/'`" + CVMFSRPMURLS="$CVMFSRPMURLS $CVMFSURL/`echo $CVMFSRPMNAME|sed 's/cvmfs-/cvmfs-libs-/'`" fi +URLS="$URLS $CVMFSRPMURLS" + if [ "$DISTTYPE" != "none" ]; then - CONFIGREPO="`echo "$BASELIST"|grep "^cvmfs-config-$DISTTYPE-[0-9]"|tail -1`" - if [ -z "$CONFIGREPO" ]; then - echo "cvmfs-config not found at $BASEURL!" >&2 - exit 1 + if [ "$DISTTYPE" != "default" ]; then + CONFIGREPO="`echo "$BASELIST"|grep "^cvmfs-config-$DISTTYPE-[0-9]"|tail -1`" + if [ -z "$CONFIGREPO" ]; then + echo "cvmfs-config not found at $BASEURL!" >&2 + exit 1 + fi + URLS="$URLS $BASEURL/$CONFIGREPO" + else + URLS="$URLS $CVMFS_BASEURL/yum/cvmfs-config-default-latest.noarch.rpm" fi - URLS="$URLS $BASEURL/$CONFIGREPO" fi # return the url of the latest version of a package given @@ -325,6 +355,7 @@ fi FUSESUBDIR=false CVMFSFUSE3URL="$CVMFSURL/`echo $CVMFSRPMNAME|sed 's/cvmfs-/cvmfs-fuse3-/'`" +CVMFSRPMURLS="$CVMFSRPMURLS $CVMFSFUSE3URL" if $SING; then URLS="$URLS $CVMFSFUSE3URL" if [ "$EL" -lt 8 ]; then