ci-cd-pipeline.yaml

name: CI/CD Pipeline

on:
  pull_request:
    branches:
      - main
  push:
    branches:
      - main
  workflow_dispatch: ~ # Allows manual triggering of the workflow

permissions:
  contents: read

concurrency:
  group: vm-ci-cd
  cancel-in-progress: false

jobs:
  setup:
    name: Setup
    runs-on: ubuntu-latest

    outputs:
      vm_started: ${{ steps.vm-status.outputs.vm_started }}

    steps:
      - name: Checkout Code on Runner # Just to get required actions in .github/. The actual code checkout is done on the VM
        uses: actions/checkout@v4
        with:
          lfs: false

      - name: Access Azure VM
        id: vm-status
        uses: azure/CLI@v2
        with:
          azcliversion: 2.62.0
          inlineScript: |
            echo "Logging into Azure..."
            az login --service-principal -u ${{ secrets.AZURE_CLIENT_ID }} -p ${{ secrets.AZURE_CLIENT_SECRET }} --tenant ${{ secrets.AZURE_TENANT_ID }}
            if [ $? -eq 0 ]; then
              echo "Logged in successfully"
            else
              echo "Failed to login"
              exit 1
            fi
            echo "Setting Azure subscription..."
            az account set --subscription ${{ secrets.AZURE_SUBSCRIPTION_ID }}
            if [ $? -eq 0 ]; then
                echo "Subscription set successfully"
            else
                echo "Failed to set subscription"
                exit 1
            fi
            echo "Checking if VM is running..."
            VM_STATUS=$(az vm get-instance-view --resource-group ${{ secrets.AZURE_RESOURCE_GROUP }} --name ${{ secrets.AZURE_VM_NAME }} --query instanceView.statuses[1].displayStatus --output tsv)
            if [ "$VM_STATUS" != "VM running" ]; then
              echo "Starting VM..."
              az vm start --resource-group ${{ secrets.AZURE_RESOURCE_GROUP }} --name ${{ secrets.AZURE_VM_NAME }}
              echo "vm_started=true" >> $GITHUB_OUTPUT
            else
              echo "VM is already running"
              echo "vm_started=false" >> $GITHUB_OUTPUT
            fi

      - name: Common SSH and Azure CLI Setup
        uses: ./.github/actions/common-steps
        with:
          VM_IP: ${{ secrets.VM_IP }}
          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}

      - name: Checkout Code on VM
        run: |
          ssh -T -o ConnectTimeout=10 ${{ secrets.VM_USERNAME }}@${{ secrets.VM_IP }} << 'EOF'
            set -e # Exit on error

            cd ~/aperi-mech
            echo "Fetching git branches..."
            git fetch --all

            echo "Stashing any unstaged changes..."
            git stash --include-untracked

            echo "Checking out main branch and pulling latest changes..."
            git checkout main
            git pull origin main --rebase

            # Configure git to use the CICD_REPO_SECRET for fetching submodules
            if ! git config --global --get url."https://${{ secrets.CICD_REPO_SECRET }}@github.com/".insteadOf; then
              git config --global url."https://${{ secrets.CICD_REPO_SECRET }}@github.com/".insteadOf "https://github.com/"
            fi
            # For authenticated access to the repository the secret will be changed periodically and the git config will have to be updated
            # On the VM, do:
            #   git config --global --get-regexp url.*.insteadOf
            # to see the current git config. And then remove the config with:
            #   git config --global --unset-all url.https://OLD_SECRET@github.com/.insteadof
            # Where OLD_SECRET is the old secret from the first command. Then add the new secret with:
            #   git config --global url."https://NEW_SECRET@github.com/".insteadOf "
            # Where the NEW_SECRET is genertated in developer settings on GitHub and added to the repository secrets.
            # or, it will be automatically updated when this action runs again.

            echo "Initializing and updating submodules..."
            git submodule update --init --recursive --remote

            echo "Checking out appropriate branch..."
            if [ "${{ github.event_name }}" = "pull_request" ]; then
              git fetch origin ${{ github.sha }}
              git checkout ${{ github.sha }}
            elif [ "${{ github.event_name }}" = "push" ] && [ "${{ github.ref }}" = "refs/heads/main" ]; then
              git checkout main
            else
              git checkout ${{ github.ref }}
            fi
            git lfs pull # Pull LFS files
            rm -rf build # Remove build directory, this prevents some false positives in tests
          EOF

  build-release:
    name: Build Release
    runs-on: ubuntu-latest
    needs: setup
    concurrency:
      group: build-and-test-vm-ci-cd
      cancel-in-progress: false
    steps:
      - name: Checkout Code on Runner # Just to get required actions in .github/. The actual code checkout is done on the VM
        uses: actions/checkout@v4
        with:
          lfs: false

      - name: Build Release
        uses: ./.github/actions/build-action
        with:
          build-type: Release
          VM_IP: ${{ secrets.VM_IP }}
          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
          VM_USERNAME: ${{ secrets.VM_USERNAME }}
          gpu: false

  build-debug:
    name: Build Debug
    runs-on: ubuntu-latest
    needs: setup
    concurrency:
      group: build-and-test-vm-ci-cd
      cancel-in-progress: false
    steps:
      - name: Checkout code # Just to get required actions in .github/
        uses: actions/checkout@v4
        with:
          lfs: false

      - name: Build Debug
        uses: ./.github/actions/build-action
        with:
          build-type: Debug
          VM_IP: ${{ secrets.VM_IP }}
          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
          VM_USERNAME: ${{ secrets.VM_USERNAME }}
          gpu: false

  build-release-gpu:
    name: Build Release, GPU
    runs-on: ubuntu-latest
    needs: [setup, test-release]
    concurrency:
      group: build-and-test-vm-ci-cd
      cancel-in-progress: false
    steps:
      - name: Checkout code # Just to get required actions in .github/
        uses: actions/checkout@v4
        with:
          lfs: false

      - name: Build Release, GPU
        uses: ./.github/actions/build-action
        with:
          build-type: Release
          VM_IP: ${{ secrets.VM_IP }}
          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
          VM_USERNAME: ${{ secrets.VM_USERNAME }}
          gpu: true

  build-debug-gpu:
    name: Build Debug, GPU
    runs-on: ubuntu-latest
    needs: [setup, build-debug, test-debug]
    concurrency:
      group: build-and-test-vm-ci-cd
      cancel-in-progress: false
    steps:
      - name: Checkout code # Just to get required actions in .github/
        uses: actions/checkout@v4
        with:
          lfs: false

      - name: Build Debug, GPU
        uses: ./.github/actions/build-action
        with:
          build-type: Debug
          VM_IP: ${{ secrets.VM_IP }}
          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
          VM_USERNAME: ${{ secrets.VM_USERNAME }}
          gpu: true

  test-release:
    name: Test Release
    runs-on: ubuntu-latest
    needs: [setup, build-release]
    concurrency:
      group: build-and-test-vm-ci-cd
      cancel-in-progress: false
    env:
      BUILD_TYPE: Release
      GPU: false
    steps:
      - name: Checkout Code on Runner # Just to get required actions in .github/. The actual code checkout is done on the VM
        uses: actions/checkout@v4
        with:
          lfs: false

      - name: Common SSH and Azure CLI Setup
        uses: ./.github/actions/common-steps
        with:
          VM_IP: ${{ secrets.VM_IP }}
          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}

      - name: Run unit tests
        uses: ./.github/actions/run-unit-tests
        with:
          build-type: ${{ env.BUILD_TYPE }}
          VM_IP: ${{ secrets.VM_IP }}
          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
          VM_USERNAME: ${{ secrets.VM_USERNAME }}
          gpu: ${{ env.GPU }}
          num-processes: 1

      - name: Run unit tests, parallel
        uses: ./.github/actions/run-unit-tests
        with:
          build-type: ${{ env.BUILD_TYPE }}
          VM_IP: ${{ secrets.VM_IP }}
          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
          VM_USERNAME: ${{ secrets.VM_USERNAME }}
          gpu: ${{ env.GPU }}
          num-processes: 3

      - name: Run material tests
        uses: ./.github/actions/run-material-tests
        with:
          build-type: ${{ env.BUILD_TYPE }}
          VM_IP: ${{ secrets.VM_IP }}
          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
          VM_USERNAME: ${{ secrets.VM_USERNAME }}
          gpu: ${{ env.GPU }}

      - name: Run utils modules tests
        uses: ./.github/actions/run-utils-modules-tests
        with:
          build-type: ${{ env.BUILD_TYPE }}
          VM_IP: ${{ secrets.VM_IP }}
          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
          VM_USERNAME: ${{ secrets.VM_USERNAME }}
          gpu: ${{ env.GPU }}

  regression-test-release:
    name: Regression Test Release
    runs-on: ubuntu-latest
    needs: [setup, build-release, test-release, test-release-gpu]
    concurrency:
      group: build-and-test-vm-ci-cd
      cancel-in-progress: false
    env:
      GPU: false
      BUILD: Release
    steps:
      - name: Checkout Code on Runner # Just to get required actions in .github/. The actual code checkout is done on the VM
        uses: actions/checkout@v4
        with:
          lfs: false

      - name: Common SSH and Azure CLI Setup
        uses: ./.github/actions/common-steps
        with:
          VM_IP: ${{ secrets.VM_IP }}
          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}

      - name: Run regression tests, serial
        uses: ./.github/actions/run-regression-tests
        with:
          VM_IP: ${{ secrets.VM_IP }}
          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
          VM_USERNAME: ${{ secrets.VM_USERNAME }}
          gpu: ${{ env.GPU }}
          parallel: false
          build: ${{ env.BUILD }}

      - name: Run regression tests, parallel
        uses: ./.github/actions/run-regression-tests
        with:
          VM_IP: ${{ secrets.VM_IP }}
          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
          VM_USERNAME: ${{ secrets.VM_USERNAME }}
          gpu: ${{ env.GPU }}
          parallel: true
          build: ${{ env.BUILD }}

  regression-test-debug:
    name: Regression Test Debug
    runs-on: ubuntu-latest
    needs: [setup, build-debug, test-debug, test-debug-gpu]
    concurrency:
      group: build-and-test-vm-ci-cd
      cancel-in-progress: false
    env:
      GPU: false
      BUILD: Debug
    steps:
      - name: Checkout Code on Runner # Just to get required actions in .github/. The actual code checkout is done on the VM
        uses: actions/checkout@v4
        with:
          lfs: false

      - name: Common SSH and Azure CLI Setup
        uses: ./.github/actions/common-steps
        with:
          VM_IP: ${{ secrets.VM_IP }}
          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}

      - name: Run regression tests, serial
        uses: ./.github/actions/run-regression-tests
        with:
          VM_IP: ${{ secrets.VM_IP }}
          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
          VM_USERNAME: ${{ secrets.VM_USERNAME }}
          gpu: ${{ env.GPU }}
          parallel: false
          build: ${{ env.BUILD }}

      - name: Run regression tests, parallel
        uses: ./.github/actions/run-regression-tests
        with:
          VM_IP: ${{ secrets.VM_IP }}
          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
          VM_USERNAME: ${{ secrets.VM_USERNAME }}
          gpu: ${{ env.GPU }}
          parallel: true
          build: ${{ env.BUILD }}

  test-debug:
    name: Test Debug
    runs-on: ubuntu-latest
    needs: [setup, build-debug]
    concurrency:
      group: build-and-test-vm-ci-cd
      cancel-in-progress: false
    env:
      BUILD_TYPE: Debug
      GPU: false
    steps:
      - name: Checkout Code on Runner # Just to get required actions in .github/. The actual code checkout is done on the VM
        uses: actions/checkout@v4
        with:
          lfs: false

      - name: Common SSH and Azure CLI Setup
        uses: ./.github/actions/common-steps
        with:
          VM_IP: ${{ secrets.VM_IP }}
          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}

      - name: Run unit tests
        uses: ./.github/actions/run-unit-tests
        with:
          build-type: ${{ env.BUILD_TYPE }}
          VM_IP: ${{ secrets.VM_IP }}
          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
          VM_USERNAME: ${{ secrets.VM_USERNAME }}
          gpu: ${{ env.GPU }}
          num-processes: 1

      - name: Run unit tests, parallel
        uses: ./.github/actions/run-unit-tests
        with:
          build-type: ${{ env.BUILD_TYPE }}
          VM_IP: ${{ secrets.VM_IP }}
          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
          VM_USERNAME: ${{ secrets.VM_USERNAME }}
          gpu: ${{ env.GPU }}
          num-processes: 3

      - name: Run material tests
        uses: ./.github/actions/run-material-tests
        with:
          build-type: ${{ env.BUILD_TYPE }}
          VM_IP: ${{ secrets.VM_IP }}
          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
          VM_USERNAME: ${{ secrets.VM_USERNAME }}
          gpu: ${{ env.GPU }}

      - name: Run utils modules tests
        uses: ./.github/actions/run-utils-modules-tests
        with:
          build-type: ${{ env.BUILD_TYPE }}
          VM_IP: ${{ secrets.VM_IP }}
          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
          VM_USERNAME: ${{ secrets.VM_USERNAME }}
          gpu: ${{ env.GPU }}

  test-release-gpu:
    name: Test Release, GPU
    runs-on: ubuntu-latest
    needs: [setup, build-release-gpu]
    concurrency:
      group: build-and-test-vm-ci-cd
      cancel-in-progress: false
    env:
      BUILD_TYPE: Release
      GPU: true
    steps:
      - name: Checkout Code on Runner # Just to get required actions in .github/. The actual code checkout is done on the VM
        uses: actions/checkout@v4
        with:
          lfs: false

      - name: Common SSH and Azure CLI Setup
        uses: ./.github/actions/common-steps
        with:
          VM_IP: ${{ secrets.VM_IP }}
          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}

      - name: Run unit tests
        uses: ./.github/actions/run-unit-tests
        with:
          build-type: ${{ env.BUILD_TYPE }}
          VM_IP: ${{ secrets.VM_IP }}
          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
          VM_USERNAME: ${{ secrets.VM_USERNAME }}
          gpu: ${{ env.GPU }}
          num-processes: 1

      - name: Run material tests
        uses: ./.github/actions/run-material-tests
        with:
          build-type: ${{ env.BUILD_TYPE }}
          VM_IP: ${{ secrets.VM_IP }}
          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
          VM_USERNAME: ${{ secrets.VM_USERNAME }}
          gpu: ${{ env.GPU }}

      - name: Run utils modules tests
        uses: ./.github/actions/run-utils-modules-tests
        with:
          build-type: ${{ env.BUILD_TYPE }}
          VM_IP: ${{ secrets.VM_IP }}
          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
          VM_USERNAME: ${{ secrets.VM_USERNAME }}
          gpu: ${{ env.GPU }}

  regression-test-release-gpu:
    name: Regression Test Release, GPU
    runs-on: ubuntu-latest
    needs: [setup, build-release-gpu, test-release-gpu, regression-test-release]
    concurrency:
      group: build-and-test-vm-ci-cd
      cancel-in-progress: false
    env:
      GPU: true
      BUILD: Release
    steps:
      - name: Checkout Code on Runner # Just to get required actions in .github/. The actual code checkout is done on the VM
        uses: actions/checkout@v4
        with:
          lfs: false

      - name: Common SSH and Azure CLI Setup
        uses: ./.github/actions/common-steps
        with:
          VM_IP: ${{ secrets.VM_IP }}
          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}

      - name: Run regression tests, serial
        uses: ./.github/actions/run-regression-tests
        with:
          VM_IP: ${{ secrets.VM_IP }}
          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
          VM_USERNAME: ${{ secrets.VM_USERNAME }}
          gpu: ${{ env.GPU }}
          parallel: false
          build: ${{ env.BUILD }}

  regression-test-debug-gpu:
    name: Regression Test Debug, GPU
    runs-on: ubuntu-latest
    needs: [setup, build-debug-gpu, test-debug-gpu, regression-test-debug]
    concurrency:
      group: build-and-test-vm-ci-cd
      cancel-in-progress: false
    env:
      GPU: true
      BUILD: Debug
    steps:
      - name: Checkout Code on Runner # Just to get required actions in .github/. The actual code checkout is done on the VM
        uses: actions/checkout@v4
        with:
          lfs: false

      - name: Common SSH and Azure CLI Setup
        uses: ./.github/actions/common-steps
        with:
          VM_IP: ${{ secrets.VM_IP }}
          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}

      - name: Run regression tests, serial
        uses: ./.github/actions/run-regression-tests
        with:
          VM_IP: ${{ secrets.VM_IP }}
          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
          VM_USERNAME: ${{ secrets.VM_USERNAME }}
          gpu: ${{ env.GPU }}
          parallel: false
          build: ${{ env.BUILD }}

  test-debug-gpu:
    name: Test Debug, GPU
    runs-on: ubuntu-latest
    needs: [setup, build-debug-gpu]
    concurrency:
      group: build-and-test-vm-ci-cd
      cancel-in-progress: false
    env:
      BUILD_TYPE: Debug
      GPU: true
    steps:
      - name: Checkout Code on Runner # Just to get required actions in .github/. The actual code checkout is done on the VM
        uses: actions/checkout@v4
        with:
          lfs: false

      - name: Common SSH and Azure CLI Setup
        uses: ./.github/actions/common-steps
        with:
          VM_IP: ${{ secrets.VM_IP }}
          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}

      - name: Run unit tests
        uses: ./.github/actions/run-unit-tests
        with:
          build-type: ${{ env.BUILD_TYPE }}
          VM_IP: ${{ secrets.VM_IP }}
          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
          VM_USERNAME: ${{ secrets.VM_USERNAME }}
          gpu: ${{ env.GPU }}
          num-processes: 1

      - name: Run material tests
        uses: ./.github/actions/run-material-tests
        with:
          build-type: ${{ env.BUILD_TYPE }}
          VM_IP: ${{ secrets.VM_IP }}
          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
          VM_USERNAME: ${{ secrets.VM_USERNAME }}
          gpu: ${{ env.GPU }}

      - name: Run utils modules tests
        uses: ./.github/actions/run-utils-modules-tests
        with:
          build-type: ${{ env.BUILD_TYPE }}
          VM_IP: ${{ secrets.VM_IP }}
          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
          VM_USERNAME: ${{ secrets.VM_USERNAME }}
          gpu: ${{ env.GPU }}

  code-coverage:
    name: Build Debug, Code Coverage
    runs-on: ubuntu-latest
    needs: [setup, regression-test-debug, regression-test-debug-gpu]
    concurrency:
      group: build-and-test-vm-ci-cd
      cancel-in-progress: false
    steps:
      - name: Checkout Code on Runner # Just to get required actions in .github/. The actual code checkout is done on the VM
        uses: actions/checkout@v4
        with:
          lfs: false

      - name: Build Debug, Code Coverage
        uses: ./.github/actions/build-action
        with:
          build-type: Debug
          VM_IP: ${{ secrets.VM_IP }}
          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
          VM_USERNAME: ${{ secrets.VM_USERNAME }}
          gpu: false
          code-coverage: true

      - name: Upload coverage reports to Codecov
        run: |
          ssh -T -o ConnectTimeout=10 ${{ secrets.VM_USERNAME }}@${{ secrets.VM_IP }} << 'EOF'
            # Install Codecov on VM (not inside Docker, but could be done inside Docker)
            # curl -Os https://cli.codecov.io/latest/linux/codecov
            # sudo chmod +x codecov
            cd ~/aperi-mech
            ../codecov --verbose upload-process --fail-on-error -t ${{ secrets.CODECOV_TOKEN }} -n unittest-${{ github.run_id }} --commit-sha ${{ github.sha }} -f build/Debug_cov/coverage.info --disable-search
          EOF

  teardown:
    runs-on: ubuntu-latest
    name: Teardown
    needs:
      [
        setup,
        build-release,
        test-release,
        build-debug,
        test-debug,
        build-release-gpu,
        test-release-gpu,
        build-debug-gpu,
        test-debug-gpu,
        regression-test-release,
        regression-test-release-gpu,
        regression-test-debug,
        regression-test-debug-gpu,
        code-coverage,
      ]
    steps:
      # For debugging purposes
      - name: Print VM status
        if: always()
        run: |
          echo "VM started: ${{ needs.setup.outputs.vm_started }}"

      - name: Stop Azure VM
        if: always() && needs.setup.outputs.vm_started == 'true'
        uses: azure/CLI@v2
        with:
          azcliversion: 2.62.0
          inlineScript: |
            echo "Logging into Azure..."
            az login --service-principal -u ${{ secrets.AZURE_CLIENT_ID }} -p ${{ secrets.AZURE_CLIENT_SECRET }} --tenant ${{ secrets.AZURE_TENANT_ID }}
            if [ $? -eq 0 ]; then
              echo "Logged in successfully"
            else
              echo "Failed to login"
              exit 1
            fi
            echo "Setting Azure subscription..."
            az account set --subscription ${{ secrets.AZURE_SUBSCRIPTION_ID }}
            if [ $? -eq 0 ]; then
                echo "Subscription set successfully"
            else
                echo "Failed to set subscription"
                exit 1
            fi
            echo "Deallocating VM..."
            az vm deallocate --resource-group ${{ secrets.AZURE_RESOURCE_GROUP }} --name ${{ secrets.AZURE_VM_NAME }}
            if [ $? -eq 0 ]; then
                echo "VM deallocated successfully"
            else
                echo "Failed to deallocate VM"
                exit 1
            fi