makedist: update cvmfs package repo urls with fallback

Adds a fallback URL for the cvmfs package download.
makedist currently looks for the latest package in the listing returned by package servers;
since the s3 bucket we are mainly using (and which is the most reliable) does not support listings,
this mechanism now relies on reading a small file .latest_cvmfs_version_prod (https://cvmrepo.web.cern.ch/cvmrepo/.latest_cvmfs_version_prod) containing a string with the latest version.
I'll add this to our testing to ensure it is kept up to date.

Obviously this won't work if you wanted to add mirrors maintained by a third party - one possibility would be to hardcode the latest version in the script.

The downloads for the non-default config packages is kept the same, except for a small fix for the egi download.

The ecsft.cern.ch url is removed alltogether, as this is the most likely one to cause issues.

makedist

@@ -5,7 +5,8 @@
 # Written by Dave Dykstra 17 April 2019
 #
 
-SUPPORTEDTYPES="rhel7-x86_64 rhel8-aarch64 rhel8-x86_64 rhel8-ppc64le rhel9-aarch64 rhel9-x86_64 rhel9-ppc64le suse15-x86_64"
+#TODO: re-add support for suse15-x86_64
+SUPPORTEDTYPES="rhel7-x86_64 rhel8-aarch64 rhel8-x86_64 rhel8-ppc64le rhel9-aarch64 rhel9-x86_64 rhel9-ppc64le"
 
 supportedtypes() {
     typeset LASTDISTRO=""
@@ -75,6 +76,17 @@ distroname() {
     fi
 }
 
+# return the distroname as encountered in yum package repo URLS,
+# i.e  https://cvmrepo.web.cern.ch/cvmrepo/yum/cvmfs/EL/9/
+distroname_yum() {
+    case " $1 " in
+        *" rhel "*) echo EL;;
+        *" suse "*) echo suse;;
+        *)  echo "Distro name $1 not supported" >&2
+            exit 2;;
+    esac
+}
+
 distroversion() {
     if [ -n "$MACHTYPE" ]; then
         echo "$MACHTYPE"|sed 's/^[^0-9]*\([0-9]*\)-.*/\1/'
@@ -100,6 +112,7 @@ distroarch() {
 }
 
 DISTRO="`distroname`"
+DISTRO_YUM="`distroname_yum $DISTRO`"
 VERS="`distroversion`"
 ARCH="`distroarch`"
 
@@ -183,17 +196,28 @@ case $1 in
             REL=23-main
         fi
         REPO=release
-        BASEURL="https://repo.opensciencegrid.org/osg/$REL/el$EL/$REPO/x86_64";;
+        BASEURL="https://repo.opensciencegrid.org/osg/$REL/el$EL/$REPO/x86_64"
+        BASELIST="`curl -Ls $BASEURL/|grep "cvmfs-"|sed 's/.*href="//;s/".*//'`";;
     egi)
         OS=centos$EL
-        BASEURL="https://repository.egi.eu/sw/production/umd/4/$OS/x86_64/updates";;
+        if [ $EL -gt 8 ]; then
+             BASEURL="https://repository.egi.eu/sw/production/umd/5/al9/release/x86_64/"
+        elif [ $EL -eq 7 ]; then
+            BASEURL="https://repository.egi.eu/sw/production/umd/4/centos7/x86_64/updates"
+        else
+          echo "ERROR: unsupported OS for egi! only centos7, almalinux9 supported."
+          exit 1
+        fi 
+        BASELIST="`curl -Ls $BASEURL/|grep "cvmfs-"|sed 's/.*href="//;s/".*//'`";;
     default|none)
         INCLUDEHELPER=false
-        BASEURL="https://cvmrepo.web.cern.ch/cvmrepo/yum/cvmfs/EL/$EL/x86_64";;
+        BASEURL="https://cvmrepo.web.cern.ch/cvmrepo/yum/cvmfs/EL/$EL/x86_64"
+        BASEURL_MIRROR1="https://cvmrepo.web.cern.ch/cvmrepo/yum/cvmfs/EL/$EL/x86_64";;
     *) usage;;
 esac
 DISTTYPE=$1
 
+
 if [ -d $DIST ]; then
     echo "$DIST already exists" >&2
     exit 1
@@ -232,24 +256,32 @@ if [ "$ARCH" != "x86_64" ]; then
     #  we're looking at x86_64 version repositories for the config rpm
     INCLUDEHELPER=false
 fi
-
-URLS=""
-BASELIST="`curl -Ls $BASEURL/|grep "cvmfs-"|sed 's/.*href="//;s/".*//'`"
-CVMFSPKG="`echo "$BASELIST"|grep "^cvmfs-[0-9]"|tail -1`"
-if [ -z "$CVMFSPKG" ]; then
-    if [ "$DISTTYPE" = egi ] && [ "$EL" = 8 ]; then
-        echo "egi's UMD does not yet support rhel8" 2>&1
-    else
-        echo "No cvmfs package found from $BASEURL" >&2
-    fi
+if [ "$DISTTYPE" = egi ] && [ "$EL" = 8 ]; then
+    echo "egi's UMD does not yet support rhel8" 2>&1
     exit 1
 fi
 
+CVMFS_BASEURL_MIRROR1="https://cvmrepo.s3.cern.ch/cvmrepo"
+CVMFS_BASEURL_MIRROR2="https://cvmrepo.web.cern.ch/cvmrepo"
+CVMFS_BASEURL=$CVMFS_BASEURL_MIRROR1
+CVMFSVERSION="`curl -Ls $CVMFS_BASE_URL/.latest_cvmfs_version_prod`"
+if [ -z "CVMFSVERSION" ]; then
+  echo "WARN: Couldn't find .latest_cvmfs_version_prod in mirror $CVMFS_BASEURL, switching over.."
+  CVMFS_BASEURL="$CVMFS_BASE_URL_MIRROR2"
+fi
+CVMFSVERSION="`curl -Ls $CVMFS_BASEURL/.latest_cvmfs_version_prod`"
+if [ -z "CVMFSVERSION" ]; then
+  echo "ERROR: Couldn't find .latest_cvmfs_version_prod in mirror $CVMFS_BASEURL, and no mirrors left! "
+  exit 1
+fi
+
+URLS=""
 CVMFSURL=""
+CVMFSRPMURLS=""
 if [ "$ARCH" = "ppc64le" ]; then
     # Grab cvmfs package from copr, using yumdownloader to calculate
     #  the URL
-    CVMFSRPMURL="$(getcoprurl cvmfs)"
+    CVMFSRPMURLS="$(getcoprurl cvmfs)"
     if [ -z "$CVMFSRPMURL" ]; then
         echo "Failed to get $ARCH cvmfs rpm from copr" >&2
         exit 1
@@ -258,32 +290,30 @@ if [ "$ARCH" = "ppc64le" ]; then
     CVMFSRPMNAME="$(basename $CVMFSRPMURL)"
     CVMFSVERSION="`echo "$CVMFSRPMNAME" 's/.*cvmfs-\([^-]*\)-.*/\1/'`"
 else
-    # Now that we can figure out the latest cvmfs version, download the
-    #   corresponding cvmfs packages from CERN instead
-    CVMFSVERSION="`echo "$CVMFSPKG"|sed 's/.*cvmfs-\([^-]*\)-.*/\1/'`"
-    CVMFSURL="https://ecsft.cern.ch/dist/cvmfs/cvmfs-$CVMFSVERSION"
-    CVMFSRPMNAME="`curl -Ls $CVMFSURL/|grep "cvmfs-$CVMFSVERSION-.*$MACH.*\.$ARCH"|sed 's/.*href="//;s/".*//'|tail -1`"
-    if [ -z "$CVMFSRPMNAME" ]; then
-        echo "No matching cvmfs rpm found at $CVMFSURL" >&2
-        exit 1
-    fi
-    CVMFSRPMURL="$CVMFSURL/$CVMFSRPMNAME"
+    CVMFSURL="$CVMFS_BASEURL/yum/cvmfs/$DISTRO_YUM/$VERS/$ARCH"
+    CVMFSRPMNAME="cvmfs-$CVMFSVERSION-1.$MACH.$ARCH.rpm"
+    CVMFSRPMURLS="$CVMFSURL/$CVMFSRPMNAME"
 fi
-URLS="$CVMFSRPMURL"
 
 MINORVERSION="`echo "$CVMFSVERSION"|cut -d. -f2`"
 if [ "$MINORVERSION" -ge 10 ]; then
     # include cvmfs-libs
-    URLS="$URLS $CVMFSURL/`echo $CVMFSRPMNAME|sed 's/cvmfs-/cvmfs-libs-/'`"
+    CVMFSRPMURLS="$CVMFSRPMURLS $CVMFSURL/`echo $CVMFSRPMNAME|sed 's/cvmfs-/cvmfs-libs-/'`"
 fi
 
+URLS="$URLS $CVMFSRPMURLS"
+
 if [ "$DISTTYPE" != "none" ]; then
-    CONFIGREPO="`echo "$BASELIST"|grep "^cvmfs-config-$DISTTYPE-[0-9]"|tail -1`"
-    if [ -z "$CONFIGREPO" ]; then
-        echo "cvmfs-config not found at $BASEURL!" >&2
-        exit 1
+    if [ "$DISTTYPE" != "default" ]; then
+        CONFIGREPO="`echo "$BASELIST"|grep "^cvmfs-config-$DISTTYPE-[0-9]"|tail -1`"
+        if [ -z "$CONFIGREPO" ]; then
+            echo "cvmfs-config not found at $BASEURL!" >&2
+            exit 1
+        fi
+        URLS="$URLS $BASEURL/$CONFIGREPO"
+    else 
+        URLS="$URLS $CVMFS_BASEURL/yum/cvmfs-config-default-latest.noarch.rpm"
     fi
-    URLS="$URLS $BASEURL/$CONFIGREPO"
 fi
 
 # return the url of the latest version of a package given
@@ -325,6 +355,7 @@ fi
 
 FUSESUBDIR=false
 CVMFSFUSE3URL="$CVMFSURL/`echo $CVMFSRPMNAME|sed 's/cvmfs-/cvmfs-fuse3-/'`"
+CVMFSRPMURLS="$CVMFSRPMURLS $CVMFSFUSE3URL"
 if $SING; then
     URLS="$URLS $CVMFSFUSE3URL"
     if [ "$EL" -lt 8 ]; then