Collect Instance Uptime #17638
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Collect Instance Uptime | |
| on: | |
| schedule: | |
| # Run every 5 mins | |
| - cron: "* * * * *" | |
| workflow_dispatch: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ vars.MORPHOCLOUD_OS_CLOUD }} | |
| cancel-in-progress: true | |
| jobs: | |
| collect-instance-uptime: | |
| runs-on: self-hosted | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Define metadata | |
| id: define-metadata | |
| run: | | |
| echo "branch=instance-uptime-data" >> $GITHUB_OUTPUT | |
| echo "directory=data" >> $GITHUB_OUTPUT | |
| - name: Create & push branch | |
| run: | | |
| exist_in_remote=$(git ls-remote --heads origin $BRANCH) | |
| if [[ -z ${exist_in_remote} ]]; then | |
| echo "Creating $BRANCH branch" | |
| mkdir $DIRECTORY | |
| cp -r .git $DIRECTORY/.git | |
| cd $DIRECTORY | |
| user_name='github-actions[bot]' | |
| git config user.name "$user_name" && echo "user.name: ${user_name}" | |
| user_email='41898282+github-actions[bot]@users.noreply.github.com' | |
| git config user.email "$user_email" && echo "user.email: ${user_email}" | |
| git checkout --orphan $BRANCH | |
| git reset --hard | |
| git commit --allow-empty -m "Initializing $BRANCH branch" | |
| git push origin $BRANCH | |
| fi | |
| env: | |
| BRANCH: ${{ steps.define-metadata.outputs.branch }} | |
| DIRECTORY: ${{ steps.define-metadata.outputs.directory }} | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: instance-uptime-data | |
| path: ${{ steps.define-metadata.outputs.directory }} | |
| - name: Collect Instance Uptime | |
| id: collect-instance-uptime | |
| run: | | |
| source ~/venv/bin/activate | |
| instance_prefix=${PREFIX:+${PREFIX}_} | |
| instance_basename="${instance_prefix}instance" | |
| # Define the JSON file paths for storing/updating uptimes | |
| JSON_FILE=$JSON_DIR/uptime.json | |
| JSON_FILE_TMP=$(mktemp $RUNNER_TEMP/uptime.XXXXXX.json) | |
| # Initialize the JSON file if it doesn't exist, with a basic structure | |
| if [[ ! -f $JSON_FILE ]]; then | |
| jq --null-input \ | |
| --arg allocation_id "$OS_CLOUD" \ | |
| '{"allocations": [{"id": $allocation_id, "instances": []}]}' > $JSON_FILE | |
| fi | |
| # List all instances matching the naming pattern and process each one | |
| openstack server list --name "^${instance_basename}-\d+" -f json | \ | |
| jq -r '.[] | [.Name, .Status, ."OS-EXT-STS:task_state"] | @tsv' | \ | |
| while IFS=$'\t' read -r instance_name status task_state; do | |
| echo "instance_name [$instance_name] status [$status] task_state [$task_state]" | |
| # Skip the instance if it is not active | |
| if [[ "$status" != "ACTIVE" ]]; then | |
| continue | |
| fi | |
| if [[ "$task_state" != "" ]]; then | |
| continue | |
| fi | |
| # Extract issue number | |
| issue_number=${instance_name##*-} | |
| echo "issue_number [$issue_number]" | |
| # Retrieve the IP address of the instance | |
| instance_ip=$( | |
| openstack server show $instance_name -c addresses -f json | \ | |
| jq -r '.addresses.auto_allocated_network[1]' | |
| ) | |
| echo "instance_ip [$instance_ip]" | |
| # Skip the instance if the IP address could not be retrieved | |
| if [[ "$instance_ip" == "null" ]]; then | |
| echo "::warning ::Failed to retrieve $instance_name IP" | |
| continue | |
| fi | |
| # Notes on SSH usage: | |
| # * Redirecting SSH standard input to /dev/null ('< /dev/null') is required to work around | |
| # an issue where SSH breaks out of the while loop in Bash. | |
| # Reference: https://stackoverflow.com/questions/9393038/ssh-breaks-out-of-while-loop-in-bash | |
| # Retrieve the instance uptime using SSH | |
| uptime=$(ssh \ | |
| -o StrictHostKeyChecking=no \ | |
| -o UserKnownHostsFile=/dev/null \ | |
| -o LogLevel=ERROR \ | |
| exouser@$instance_ip \ | |
| 'cat /proc/uptime | awk "{print \$1}"' < /dev/null) | |
| if [[ $? -ne 0 ]]; then | |
| echo "::warning ::Failed to retrieve uptime for $instance_name using IP $instance_ip" | |
| continue | |
| fi | |
| # Retrieve the startup time of the instance in yyyy-mm-dd HH:MM:SS format | |
| startup_time=$(ssh \ | |
| -o StrictHostKeyChecking=no \ | |
| -o UserKnownHostsFile=/dev/null \ | |
| -o LogLevel=ERROR \ | |
| exouser@$instance_ip \ | |
| 'uptime -s' < /dev/null) | |
| if [[ $? -ne 0 ]]; then | |
| echo "::warning ::Failed to retrieve startup time for $instance_name using IP $instance_ip" | |
| continue | |
| fi | |
| # Check if the instance is already recorded in the JSON file | |
| has_instance=$(cat $JSON_FILE | jq \ | |
| --arg allocation_id "$OS_CLOUD" \ | |
| --arg instance_name "$instance_name" \ | |
| --arg startup_time "$startup_time" \ | |
| 'any(.allocations[] | select(.id == $allocation_id) | .instances[]; .name == $instance_name)') | |
| # If the instance is not found, add it to the JSON file | |
| if [[ "$has_instance" == "false" ]]; then | |
| cat $JSON_FILE | jq \ | |
| --arg allocation_id "$OS_CLOUD" \ | |
| --arg instance_name "$instance_name" \ | |
| '(.allocations[] | select(.id == $allocation_id) | .instances) | |
| += [{"name": $instance_name, "uptimes": {}}]' > $JSON_FILE_TMP \ | |
| && mv $JSON_FILE_TMP $JSON_FILE | |
| fi | |
| # Check if there is an uptime entry for the startup time | |
| has_startup_time=$(cat $JSON_FILE | jq \ | |
| --arg allocation_id "$OS_CLOUD" \ | |
| --arg instance_name "$instance_name" \ | |
| --arg startup_time "$startup_time" \ | |
| '(.allocations[] | select(.id == $allocation_id) | .instances[] | select(.name == $instance_name) | .uptimes | has($startup_time))') | |
| if [[ "$has_startup_time" == "false" ]]; then | |
| # If the startup time is not recorded, add the uptime entry | |
| cat $JSON_FILE | jq \ | |
| --arg allocation_id "$OS_CLOUD" \ | |
| --arg instance_name "$instance_name" \ | |
| --arg startup_time "$startup_time" \ | |
| --arg uptime "$uptime" \ | |
| '(.allocations[] | select(.id == $allocation_id) | .instances[] | select(.name == $instance_name) | .uptimes) | |
| += {($startup_time): $uptime|tonumber}' > $JSON_FILE_TMP \ | |
| && mv $JSON_FILE_TMP $JSON_FILE | |
| else | |
| # If the startup time is already recorded, update the uptime entry | |
| cat $JSON_FILE | jq \ | |
| --arg allocation_id "$OS_CLOUD" \ | |
| --arg instance_name "$instance_name" \ | |
| --arg startup_time "$startup_time" \ | |
| --arg uptime "$uptime" \ | |
| '(.allocations[] | select(.id == $allocation_id) | .instances[] | select(.name == $instance_name) | .uptimes[$startup_time]) | |
| = ($uptime|tonumber)' > $JSON_FILE_TMP \ | |
| && mv $JSON_FILE_TMP $JSON_FILE | |
| fi | |
| done | |
| # Display the final JSON content for verification | |
| echo "--------" | |
| cat $JSON_FILE | |
| echo "--------" | |
| env: | |
| OS_CLOUD: ${{ vars.MORPHOCLOUD_OS_CLOUD }} | |
| PREFIX: ${{ vars.INSTANCE_NAME_PREFIX }} | |
| JSON_DIR: ${{ steps.define-metadata.outputs.directory }} | |
| - name: Publish | |
| uses: s0/git-publish-subdir-action@61eb9e6420447e7cbf010f7cce37103665c46bfa # develop | |
| env: | |
| REPO: self | |
| BRANCH: ${{ steps.define-metadata.outputs.branch }} | |
| FOLDER: ${{ steps.define-metadata.outputs.directory }} | |
| SQUASH_HISTORY: true | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |