Support building a Homebrew package on macOS (#7)

Issue: yugabyte/yugabyte-db#2967

Also removing s3cmd from Linuxbrew dependencies because it pulls in a lot of dependencies that we don't actually need.

azure-pipelines.yml

@@ -1,14 +1,14 @@
 resources:
   containers:
-  - container: centos7
-    image: yugabytedb/yb_build_infra_centos7:v2019-09-01T14_58_09_mbautin
+    - container: centos7
+      image: yugabytedb/yb_build_infra_centos7:v2019-09-01T14_58_09_mbautin
 
 # https://docs.microsoft.com/en-us/azure/devops/pipelines/build/triggers?view=azure-devops&tabs=yaml
 trigger:
   batch: true
   branches:
     include:
-    - master
+      - master
 
 jobs:
   - job: RunInContainer
@@ -25,6 +25,7 @@ jobs:
 
     steps:
       - script: |
+          set -euxo pipefail
           sudo mkdir -p /opt/yb-build
           sudo chown yugabyte-ci /opt/yb-build
           checkout_dir=$PWD
@@ -33,3 +34,20 @@ jobs:
           sudo -u yugabyte-ci -E /bin/bash -c "cd '$checkout_dir' && ./test.sh && ./build-and-release.sh"
         env:
           GITHUB_TOKEN: $(yugabyte.githubToken)
+
+  - job: macOS
+    pool:
+      vmImage: 'macOS-10.14'
+
+    timeoutInMinutes: 0
+
+    steps:
+      - script: |
+          set -euxo pipefail
+          brew install hub
+          sudo mkdir -p /opt/yb-build
+          sudo chown $USER /opt/yb-build
+          ./test.sh
+          ./build-and-release.sh
+        env:
+          GITHUB_TOKEN: $(yugabyte.githubToken)

brew-build.sh

@@ -29,19 +29,23 @@ BREW_FROM_SRC_PACKAGES=(
   automake
   bzip2
   flex
-  gcc
   icu4c
   libtool
-  libuuid
   ninja
-  openssl
   readline
-  s3cmd
+  openssl
 )
 
-BREW_BIN_PACKAGES=(
-  gcc@8
-)
+BREW_BIN_PACKAGES=()
+
+echo "OSTYPE: $OSTYPE"
+
+if [[ $OSTYPE == linux* ]]; then
+  BREW_BIN_PACKAGES+=( gcc@8 )
+  BREW_FROM_SRC_PACKAGES+=( gcc libuuid )
+else
+  BREW_FROM_SRC_PACKAGES+=( gnu-tar )
+fi
 
 # -------------------------------------------------------------------------------------------------
 # Functions
@@ -50,12 +54,19 @@ BREW_BIN_PACKAGES=(
 brew_install_packages() {
   local package
   for package in "$@"; do
+    if [[ -n $install_args ]]; then
+      heading "Installing $package, arguments: $install_args"
+    else
+      heading "Installing $package"
+    fi
     if ( set -x; ./bin/brew install $install_args "$package" ); then
+      log "Successfully installed package: $package"
       successful_packages+=( "$package" )
     else
       log "Failed to install package: $package"
       failed_packages+=( "$package" )
     fi
+    separator_with_spacing
   done
 }
 
@@ -146,19 +157,31 @@ unset sse4_flags
 if [[ ${YB_BREW_BUILD_UNIT_TEST_MODE:-0} == "1" ]]; then
   BREW_FROM_SRC_PACKAGES=()
   BREW_BIN_PACKAGES=( patchelf )
+  if [[ $OSTYPE == darwin* ]]; then
+    BREW_BIN_PACKAGES+=( gnu-tar )
+  fi
 fi
 
 successful_packages=()
 failed_packages=()
 
-install_args=""
-if [[ ${#BREW_BIN_PACKAGES[@]} -gt 0 ]]; then
-  brew_install_packages "${BREW_BIN_PACKAGES[@]}"
-fi
-install_args="--build-from-source"
 if [[ ${#BREW_FROM_SRC_PACKAGES[@]} -gt 0 ]]; then
+  big_heading "Installing packages built from source."
+  install_args="--build-from-source"
   brew_install_packages "${BREW_FROM_SRC_PACKAGES[@]}"
+else
+  big_heading "No packages built from source to install."
+fi
+
+# Install binary packages.
+if [[ ${#BREW_BIN_PACKAGES[@]} -gt 0 ]]; then
+  big_heading "Installing packages from binary downloads (bottles)."
+  install_args=""
+  brew_install_packages "${BREW_BIN_PACKAGES[@]}"
+else
+  big_heading "No packages from binary downloads to install."
 fi
+
 unset install_args
 
 log "Successfully installed packages: ${successful_packages[*]}"

brew-common.sh

@@ -56,10 +56,34 @@ fatal() {
   exit 1
 }
 
-heading() {
+separator() {
   echo >&2 "--------------------------------------------------------------------------------------"
+}
+
+thick_separator() {
+  echo >&2 "======================================================================================"
+}
+
+separator_with_spacing() {
+  echo >&2
+  separator
+  echo >&2
+}
+
+heading() {
+  echo >&2
+  separator
   echo >&2 "$*"
-  echo >&2 "--------------------------------------------------------------------------------------"
+  separator
+  echo >&2
+}
+
+big_heading() {
+  echo >&2
+  thick_separator
+  echo >&2 "$*"
+  thick_separator
+  echo >&2
 }
 
 create_symlink() {
@@ -75,9 +99,15 @@ create_symlink() {
 
 run_tar() {
   if [[ $OSTYPE == linux* ]]; then
-    tar "$@"
+    ( set -x; tar "$@" )
   else
-    gtar "$@"
+    if which -s gtar; then
+      ( set -x; gtar "$@" )
+    elif [[ -n ${BREW_HOME:-} && -f $BREW_HOME/bin/gtar ]]; then
+      ( set -x; "$BREW_HOME/bin/gtar" "$@" )
+    else
+      fatal "Could not find gtar"
+    fi
   fi
 }
 
@@ -87,6 +117,12 @@ set_brew_timestamp() {
   fi
 }
 
+if [[ $OSTYPE == darwin* ]] && ! which -s realpath; then
+  realpath() {
+    python -c "import sys, os; sys.stdout.write(os.path.realpath(sys.argv[1]))" "$@"
+  }
+fi
+
 # Returns the prefix for a new Homebrew/Linuxbrew installation path, based on the current directory,
 # YB_BREW_TYPE ("homebrew" or "linuxbrew") and YB_BREW_TIMESTAMP (which would be set on demand).
 # The return value is placed in the brew_path_prefix variable in the parent scope.