wip: Refactor CI to use GitHub actions and generic bash scripts (#64)

.github/workflows/main.yaml

@@ -0,0 +1,71 @@
+name: CI
+on: [ push ]
+
+jobs:
+  run_tests:
+    name: Run tests
+    if: ${{ !contains(github.event.head_commit.message, 'skip tests') }}
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+      - name: Init runner
+        run: bash ./scripts/init_runner.sh ${{ github.job }}
+      - name: Setup workspace
+        env:
+          GITHUB_DEVELOPMENT_PAT: ${{ secrets.GH_DEVELOPMENT_PAT }}
+        run: bash ./scripts/setup_workspace.sh
+      - name: Run tests
+        env:
+          SOPS_AGE_KEY: ${{ secrets.SOPS_AGE_KEY }}
+        run: bash ./scripts/run_tests.sh -c -r ${{ github.job }}
+
+  additional_tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        node: [ 16, 18, 20 ]
+        mongo: [ 4, 5, 6 ]
+        exclude:
+          - node: 16
+            mongo: 4
+    name: Additional tests (node ${{ matrix.node }}, mongo ${{ matrix.mongo }})
+    if: ${{ contains(github.event.head_commit.message, 'additional tests') }}
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+      - name: Init runner
+        run: bash ./scripts/init_runner.sh ${{ github.job }}
+      - name: Setup workspace
+        env:
+          GITHUB_DEVELOPMENT_PAT: ${{ secrets.GH_DEVELOPMENT_PAT }}
+        run: bash ./scripts/setup_workspace.sh -n ${{ matrix.node }}
+      - name: Run tests
+        env:
+          SOPS_AGE_KEY: ${{ secrets.SOPS_AGE_KEY }}
+        run: bash ./scripts/run_tests.sh -n ${{ matrix.node }} -m ${{ matrix.mongo }}
+
+  build:
+    name: Build jobs
+    if: ${{ !contains(github.event.head_commit.message, 'skip app') }}
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+      - name: Init runner
+        run: bash ./scripts/init_runner.sh ${{ github.job }}
+      - name: Setup workspace
+        env:
+          GITHUB_DEVELOPMENT_PAT: ${{ secrets.GH_DEVELOPMENT_PAT }}
+        run: bash ./scripts/setup_workspace.sh
+      - name: Build jobs
+        env:
+          SOPS_AGE_KEY: ${{ secrets.SOPS_AGE_KEY }}
+        run: bash ./scripts/build.sh -p -r ${{ github.job }}

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "scripts/kash"]
+	path = scripts/kash
+	url = https://github.com/kalisio/kash.git

scripts/build.sh

@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+set -euo pipefail
+# set -x
+
+THIS_FILE=$(readlink -f "${BASH_SOURCE[0]}")
+THIS_DIR=$(dirname "$THIS_FILE")
+ROOT_DIR=$(dirname "$THIS_DIR")
+
+. "$THIS_DIR/kash/kash.sh"
+
+## Parse options
+##
+
+PUBLISH=false
+CI_STEP_NAME="Build"
+while getopts "prml:" option; do
+    case $option in
+        p) # publish
+            PUBLISH=true
+            ;;
+        r) # report outcome to slack
+            CI_STEP_NAME=$OPTARG
+            trap 'slack_ci_report "$ROOT_DIR" "$CI_STEP_NAME" "$?" "$SLACK_WEBHOOK_JOBS"' EXIT
+            ;;
+        m) # weacast model image to be build: gfs, arpege
+            MODEL=$OPTARG
+            ;;
+        l) # weacast loader image to be build: europe, isobaric-europe, ...
+            LOADER=$OPTARG
+            ;;
+        *)
+            ;;
+    esac
+done
+
+## Init workspace
+##
+
+WORKSPACE_DIR="$(dirname "$ROOT_DIR")"
+init_job_infos "$ROOT_DIR"
+
+JOB=$(get_job_name)
+KRAWLER_VERSION=$(get_job_krawler_version)
+GIT_TAG=$(get_job_tag)
+
+if [[ -z "$GIT_TAG" ]]; then
+    echo "About to build ${JOB}:${LOADER} development version based on krawler development version..."
+else
+    echo "About to build ${JOB}:${LOADER} v${VERSION} based on krawler ${KRAWLER_VERSION}..."
+fi
+
+load_env_files "$WORKSPACE_DIR/development/common/kalisio_dockerhub.enc.env" "$WORKSPACE_DIR/development/common/SLACK_WEBHOOK_JOBS.enc.env"
+load_value_files "$WORKSPACE_DIR/development/common/KALISIO_DOCKERHUB_PASSWORD.enc.value"
+
+## Build container
+##
+
+IMAGE_NAME="weacast/weacast-$MODEL"
+DOCKERFILE=dockerfile.$MODEL-$LOADER
+if [[ -z "$GIT_TAG" ]]; then
+    VERSION=latest
+else
+    VERSION=$(get_job_version)
+fi
+IMAGE_TAG=$LOADER-$VERSION
+
+begin_group "Building container ..."
+
+docker login --username "$KALISIO_DOCKERHUB_USERNAME" --password-stdin < "$KALISIO_DOCKERHUB_PASSWORD"
+# DOCKER_BUILDKIT is here to be able to use Dockerfile specific dockerginore (app.Dockerfile.dockerignore)
+DOCKER_BUILDKIT=1 docker build -f "$ROOT_DIR/$DOCKERFILE" \
+    --build-arg TAG=$VERSION -t "$IMAGE_NAME:$IMAGE_TAG" \
+    "$ROOT_DIR"
+
+if [ "$PUBLISH" = true ]; then
+    docker push "$IMAGE_NAME:$IMAGE_TAG"
+fi
+
+docker logout
+
+end_group "Building container ..."

scripts/init_runner.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+set -euo pipefail
+# set -x
+
+JOB_ID=$1
+
+THIS_FILE=$(readlink -f "${BASH_SOURCE[0]}")
+THIS_DIR=$(dirname "$THIS_FILE")
+# ROOT_DIR=$(dirname "$THIS_DIR")
+
+. "$THIS_DIR/kash/kash.sh"
+
+### Github Actions
+
+init_github_run_tests() {
+    install_reqs yq age sops nvm node16 node18 node20 mongo4 mongo5 mongo6 cc_test_reporter
+}
+
+init_github_build() {
+    install_reqs age sops nvm node16
+}
+
+init_github_additional_tests() {
+    install_reqs age sops nvm node16 node18 node20 mongo4 mongo5 mongo6
+}
+
+begin_group "Init $CI_ID for $JOB_ID"
+
+init_"${CI_ID}_${JOB_ID}"
+
+end_group "Init $CI_ID for $JOB_ID"

scripts/run_tests.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+set -euo pipefail
+# set -x
+
+THIS_FILE=$(readlink -f "${BASH_SOURCE[0]}")
+THIS_DIR=$(dirname "$THIS_FILE")
+ROOT_DIR=$(dirname "$THIS_DIR")
+WORKSPACE_DIR="$(dirname "$ROOT_DIR")"
+
+. "$THIS_DIR/kash/kash.sh"
+
+## Parse options
+##
+
+NODE_VER=16
+MONGO_VER=4
+CI_STEP_NAME="Run tests"
+CODE_COVERAGE=false
+while getopts "m:n:cr:" option; do
+    case $option in
+        m) # defines mongo version
+            MONGO_VER=$OPTARG
+            ;;
+        n) # defines node version
+            NODE_VER=$OPTARG
+             ;;
+        c) # publish code coverage
+            CODE_COVERAGE=true
+            ;;
+        r) # report outcome to slack
+            load_env_files "$WORKSPACE_DIR/development/common/SLACK_WEBHOOK_JOBS.enc.env"
+            CI_STEP_NAME=$OPTARG
+            trap 'slack_ci_report "$ROOT_DIR" "$CI_STEP_NAME" "$?" "$SLACK_WEBHOOK_JOBS"' EXIT
+            ;;
+        *)
+            ;;
+    esac
+done
+
+## Init workspace
+##
+
+. "$WORKSPACE_DIR/development/workspaces/jobs/jobs.sh" weacast-loaders
+
+# Required by tests
+git clone https://github.com/kalisio/krawler.git && cd krawler && yarn install && yarn link && cd ..
+yarn link @kalisio/krawler
+yarn global add @weacast/gtiff2json@${GTIFF2JSON_TAG}
+yarn global add @weacast/grib2json@${GRIB2JSON_TAG}
+
+## Run tests
+##
+
+run_lib_tests "$ROOT_DIR" "$CODE_COVERAGE" "$NODE_VER" "$MONGO_VER"

scripts/setup_workspace.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+set -euo pipefail
+# set -x
+
+THIS_FILE=$(readlink -f "${BASH_SOURCE[0]}")
+THIS_DIR=$(dirname "$THIS_FILE")
+ROOT_DIR=$(dirname "$THIS_DIR")
+
+. "$THIS_DIR/kash/kash.sh"
+
+## Parse options
+##
+
+WORKSPACE_BRANCH=
+WORKSPACE_TAG=
+
+begin_group "Setting up workspace ..."
+
+GTIFF2JSON_TAG=$(node -p -e "require('./package.json').peerDependencies['@weacast/gtiff2json']")
+GRIB2JSON_TAG=$(node -p -e "require('./package.json').peerDependencies['@weacast/grib2json']")
+
+if [ "$CI" = true ]; then
+    WORKSPACE_DIR="$(dirname "$ROOT_DIR")"
+    DEVELOPMENT_REPO_URL="https://$GITHUB_DEVELOPMENT_PAT@github.com/kalisio/development.git"
+else
+    while getopts "b:t" option; do
+        case $option in
+            b) # defines branch
+                WORKSPACE_BRANCH=$OPTARG;;
+            t) # defines tag
+                WORKSPACE_TAG=$OPTARG;;
+            *)
+            ;;
+        esac
+    done
+    shift $((OPTIND-1))
+    WORKSPACE_DIR="$1"
+
+    # NOTE: cloning weacast could be avoided if we could parse app_version from tag/branch name instead
+    # In this case, the kli would clone weacast
+    GIT_OPS="--recurse-submodules"
+    if [ -n "$WORKSPACE_TAG" ] || [ -n "$WORKSPACE_BRANCH" ]; then
+        GIT_OPS="$GIT_OPS --branch ${WORKSPACE_TAG:-$WORKSPACE_BRANCH}"
+    fi
+    git clone --depth 1 $GIT_OPS "$GITHUB_URL/weacast/weacast.git" "$WORKSPACE_DIR/weacast"
+
+    DEVELOPMENT_REPO_URL="$GITHUB_URL/kalisio/development.git"
+
+    # unset KALISIO_DEVELOPMENT_DIR because we want kli to clone everything in $WORKSPACE_DIR
+    unset KALISIO_DEVELOPMENT_DIR
+fi
+
+# clone development in $WORKSPACE_DIR
+DEVELOPMENT_DIR="$WORKSPACE_DIR/development"
+git clone --depth 1 "$DEVELOPMENT_REPO_URL" "$DEVELOPMENT_DIR"
+
+end_group "Setting up workspace ..."