automatic-instance-shelving.yml

name: Automatic Instance Shelving
on:
  schedule:
    # Run every 5 mins
    - cron: "* * * * *"
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ vars.MORPHOCLOUD_OS_CLOUD }}
  cancel-in-progress: true

permissions:
  issues: write
  contents: read

jobs:
  auto-shelve:
    runs-on: self-hosted
    steps:
      - uses: actions/checkout@v4

      - uses: actions/create-github-app-token@31c86eb3b33c9b601a1f60f98dcbfd1d70f379b4 # v1.10.3
        id: app-token
        with:
          app-id: ${{ vars.MORPHOCLOUD_WORKFLOW_APP_ID }}
          private-key: ${{ secrets.MORPHOCLOUD_WORKFLOW_APP_PRIVATE_KEY }}

      - name: Auto Shelve
        run: |
          source ~/venv/bin/activate

          instance_prefix=${PREFIX:+${PREFIX}_}
          instance_basename="${instance_prefix}instance"

          openstack server list --name "^${instance_basename}-\d+" -f json | \
          jq -r '.[] | [.Name, .Status, ."OS-EXT-STS:task_state"] | @tsv' | \
            while IFS=$'\t' read -r instance_name status task_state; do
              echo "instance_name [$instance_name] status [$status] task_state [$task_state]"

              # Skip the instance if it is not active
              if [[ "$status" != "ACTIVE" ]]; then
                continue
              fi

              # Skip the instance if it is currently undergoing a status update
              if [[ "$task_state" != "" ]]; then
                continue
              fi

              # Extract the issue number from the instance name
              issue_number=${instance_name##*-}
              echo "issue_number [$issue_number]"

              # Retrieve the IP address of the instance
              instance_ip=$(
                openstack server show $instance_name -c addresses -f json | \
                jq -r '.addresses.auto_allocated_network[1]'
              )
              echo "instance_ip [$instance_ip]"

              # Skip the instance if the IP address could not be retrieved
              if [[ "$instance_ip" == "null" ]]; then
                echo "::warning ::Failed to retrieve $instance_name IP"
                continue
              fi

              # Notes on SSH usage:
              # * Redirecting SSH standard input to /dev/null ('< /dev/null') is required to work around
              #   an issue where SSH breaks out of the while loop in Bash.
              #   Reference: https://stackoverflow.com/questions/9393038/ssh-breaks-out-of-while-loop-in-bash

              has_check_instance_shelve_script=$(ssh \
                -o StrictHostKeyChecking=no \
                -o UserKnownHostsFile=/dev/null \
                -o LogLevel=ERROR \
                exouser@$instance_ip \
                  'bash -c "[[ -f /opt/instance-config-support/dist/check-instance-shelve.sh ]] && echo yes || echo no"' < /dev/null)
              if [[ $? -ne 0 ]]; then
                echo "::warning ::Failed to check if 'check-instance-shelve.sh' was found on $instance_name using IP $instance_ip"
                continue
              fi

              if [[ "$has_check_instance_shelve_script" == "yes" ]]; then
                uptime_hours=$(ssh \
                -o StrictHostKeyChecking=no \
                -o UserKnownHostsFile=/dev/null \
                -o LogLevel=ERROR \
                exouser@$instance_ip \
                  '/opt/instance-config-support/dist/check-instance-shelve.sh -d' < /dev/null)
                if [[ $? -ne 0 ]]; then
                  echo "::warning ::Failed to retrieve uptime for $instance_name using IP $instance_ip"
                  continue
                fi
              else
                # Retrieve the instance uptime using SSH
                uptime_seconds=$(ssh \
                  -o StrictHostKeyChecking=no \
                  -o UserKnownHostsFile=/dev/null \
                  -o LogLevel=ERROR \
                  exouser@$instance_ip \
                    'cat /proc/uptime | awk  "{print \$1}"' < /dev/null)
                if [[ $? -ne 0 ]]; then
                  echo "::warning ::Failed to retrieve uptime for $instance_name using IP $instance_ip"
                  continue
                fi
                # Convert uptime from seconds to hours
                uptime_hours=$(echo "scale=2; $uptime_seconds / 3600" | bc)
              fi
              # Check uptime and define action
              if $(python3 -c "valid=($uptime_hours > 3.5 and $uptime_hours <= 4.0); EXIT_SUCCESS=0; EXIT_FAILURE=1; exit(EXIT_SUCCESS if valid else EXIT_FAILURE)"); then
                action="notify"
              elif $(python3 -c "valid=($uptime_hours > 4.0); EXIT_SUCCESS=0; EXIT_FAILURE=1; exit(EXIT_SUCCESS if valid else EXIT_FAILURE)"); then
                action="shelve"
              else
                action=""
              fi
              echo "instance_name [$instance_name], has_check_instance_shelve_script [$has_check_instance_shelve_script], uptime_hours [$uptime_hours] -> action[$action]"
              # Perform action
              if [[ "$action" == "shelve" ]]; then
                gh workflow run control-instance-from-workflow.yml \
                  -f issue_number=$issue_number \
                  -f command_name=shelve
              fi
            done
        env:
          OS_CLOUD: ${{ vars.MORPHOCLOUD_OS_CLOUD }}
          GH_TOKEN: ${{ steps.app-token.outputs.token }}
          GH_REPO: ${{ github.repository }}
          PREFIX: ${{ vars.INSTANCE_NAME_PREFIX }}