Add Hardware Support (#9)

* Initial alpha of hardware * Don't gather facts * Set better failure * Fix default * Clearer error * Add hardware profiles * Set smarter inventory * Set inventory * Make executable * Set JSON ouput * Getting closer * Fixed host lookup * Working, but not perfect * Ready for testing * Ready for first test * Simplified references * Added local workflow * Prevent local builds from being committed * Added dev script * Added debug * Added validation * Renamed to api_token * Fix package * Working hetzner deployment * Fixed defaults * Reorganized order * Fix default name * Working vultr deployment * Added ip validation * Added digital ocean support * Added inventory refresh * Swapped location * Fix inventory * Fixed address * Reorganized default file * Added digital ocean * Added comments * Fix requirements * Explicit script call * Sanitize group names * Moved vars * Add inventory validation * Removed server_ groups * Fixed variable names * Fix SSH options * Restructured updates * Remove server validation * Fixed loop * Moved to pre-task * Moved into own play * Move back into play * Added firewall * Set API token * Fix typo * Append hosts * Fixed validations and workers * Fixed remote user issues * Set fact * Improved validation messages * Adjusted defaults * Adjusted hsots * Big refactor and clean up * Default backups to be on * Set connection to local * Convert to import tasks * Remove inventory sources * Added maintain * Added firewall to Vultr * Removed comments * Added digital ocean firewall * Added locktimeout * Fixed Hetzner key id * Normalized key * Simplfied normalization * Set keys * Dont gather facts * Fixed tag * Set tags * Renable facts gathering * Adjusted tags * Moved docker user to its own role * Adjusted dependencies * Remove meta * Dont gather facts * Added sudo_keys * Raw output * Set keys * Update keys * Added first manager * Changed to spin_environment * Have script dynamically build inventory based on environment * Dynamic script * Empty arrays * Merged into prepare_ci_environment * Set full path * Fixed authorized keys * PRevent hashing of keys * Set tags * Improved SSH keyscan * Fixed requirements syntax * Set changed when false * Add get variable * Set to run once * Fix variable name * Write to file * Revert to inmemory only * Removed tag * Added inventory refresh * Show gorup * Adjust refresh * Save address * Set inventory based on address * Set to IP * Adjust inventory refresh * Added Cron entry. Fixes #6 * Adjust validation * Changed swarm group names to match v2 * Validate inventory * Fix notice * Added v2 support * Added ansible vault support * Adjusted execution permissions * Support validating the password * Added tag * Enhance environment validation logic to allow for '_workers' suffix. Updated validation checks to ensure environments can be recognized with or without the '_workers' suffix, improving flexibility in server configuration.
serversideup · Dec 19, 2024 · 1d5b5bb · 1d5b5bb
1 parent d4715c8
commit 1d5b5bb
Show file tree

Hide file tree

Showing 33 changed files with 1,199 additions and 96 deletions.
diff --git a/.ansible-lint b/.ansible-lint
@@ -2,4 +2,5 @@ skip_list:
  - 'no-changed-when'
  - 'package-latest'
  - 'var-naming[no-role-prefix]'
- - 'no-handler'
+ - 'no-handler'
+ - 'galaxy[no-changelog]'
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,4 @@
+*.tar.gz
+test/results
+.spin.yml
+.vault-password
diff --git a/.spin-inventory.example.ini b/.spin-inventory.example.ini
diff --git a/.spin.example.yml b/.spin.example.yml
@@ -1,46 +1,97 @@
----
-###########################################
-# 👇 Basic Server Configuration - Change your email
-###########################################
-server_timezone: "Etc/UTC"
-server_contact: [email protected]
-
 ##############################################################
 # 👇 Users - You must set at least one user
 ##############################################################
 
-# users:
-#   - username: alice
-#     name: Alice Smith
-#     groups: ['sudo']
-#     authorized_keys:
-#       - public_key: "ssh-ed25519 AAAAC3NzaC1lmyfakeublickeyMVIzwQXBzxxD9b8Erd1FKVvu alice"
+users:
+  # - username: alice
+  #   name: Alice Smith
+  #   groups: ['sudo']
+  #   authorized_keys:
+  #     - public_key: "ssh-ed25519 AAAAC3NzaC1lmyfakeublickeyMVIzwQXBzxxD9b8Erd1FKVvu alice"
 
-#   - username: bob
-#     name: Bob Smith
-#     state: present
-#     password: "$6$mysecretsalt$qJbapG68nyRab3gxvKWPUcs2g3t0oMHSHMnSKecYNpSi3CuZm.GbBqXO8BE6EI6P1JUefhA0qvD7b5LSh./PU1"
-#     groups: ['sudo']
-#     shell: "/bin/bash"
-#     authorized_keys:
-#       - public_key: "ssh-ed25519 AAAAC3NzaC1anotherfakekeyIMVIzwQXBzxxD9b8Erd1FKVvu bob"
+  # - username: bob
+  #   name: Bob Smith
+  #   state: present
+  #   password: "$6$mysecretsalt$qJbapG68nyRab3gxvKWPUcs2g3t0oMHSHMnSKecYNpSi3CuZm.GbBqXO8BE6EI6P1JUefhA0qvD7b5LSh./PU1"
+  #   groups: ['sudo']
+  #   shell: "/bin/bash"
+  #   authorized_keys:
+  #     - public_key: "ssh-ed25519 AAAAC3NzaC1anotherfakekeyIMVIzwQXBzxxD9b8Erd1FKVvu bob"
 
 ##############################################################
-# 👇 Deploy User - If you're using GitHub Actions, create a dedicated deploy key
+# 👇 Providers - You must set at least one provider
 ##############################################################
 
-### Learn how to create a secure DEPLOY key and enter the public value below
-## https://serversideup.net/open-source/spin/docs/advanced/generating-a-secure-ssh-key#generating-a-deployment-key
+providers:
+#   - name: digitalocean
+#     api_token: Set token here OR delete this line and set environment variable DO_API_TOKEN
 
-# docker_user:
-#   username: deploy
-#   authorized_ssh_keys: 
-#     - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKNJGtd7a4DBHsQi7HGrC5xz0eAEFHZ3Ogh3FEFI2345 fake@key"
-#     - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFRfXxUZ8q9vHRcQZ6tLb0KwGHu8xjQHfYopZKLmnopQ anotherfake@key"
+#   - name: hetzner
+#     api_token: Set token here OR delete this line and set environment variable HCLOUD_TOKEN
+
+#   - name: vultr
+#     api_token: Set token here OR delete this line and set environment variable VULTR_API_KEY
+
+##############################################################
+# 👇 Servers - You must set at least one server
+##############################################################
+
+servers:
+  # - server_name: ubuntu-2gb-ash-1
+  #   environment: production
+  #   hardware_profile: hetzner_2c_2gb_ubuntu2404
+
+  # - server_name: ubuntu-1gb-ord-2
+  #   environment: staging
+  #   hardware_profile: vultr_1c_1gb_ubuntu2404
+
+##############################################################
+# 🤖 Hardware Profiles
+##############################################################
 
-###########################################
+hardware_profiles:
+  # Hetzner
+  - name: hetzner_2c_2gb_ubuntu2404
+    provider: hetzner
+    profile_config:
+      location: ash
+      server_type: cpx11
+      image: ubuntu-24.04
+      backups: true
+
+  # Vultr
+  - name: vultr_1c_1gb_ubuntu2404
+    provider: vultr
+    profile_config:
+      region: ord
+      plan: vc2-1c-1gb
+      os: "Ubuntu 24.04 LTS x64"
+      backups: true
+
+  # DigitalOcean
+  - name: digitalocean_1c_1gb_ubuntu2404
+    provider: digitalocean
+    profile_config:
+      region: nyc3
+      size: s-1vcpu-1gb
+      image: ubuntu-24-04-x64
+      backups: true
+
+##############################################################
+# 🌎 Environments
+##############################################################
+environments:
+  - name: production
+  - name: staging
+  - name: development
+
+##############################################################
 # 🤓 Advanced Server Configuration
-###########################################
+##############################################################
+
+# Timezone and contact settings
+server_timezone: "Etc/UTC"
+server_contact: [email protected]
 
 # If you the SSH port below, you may need to run `spin provision -p <your-default-ssh-port>`
 # to get a connection on your first provision. Otherwise, SSH will try connecting 
@@ -61,3 +112,10 @@ postfix_hostname: "{{ inventory_hostname }}"
 # postfix_relayhost_port: "587"
 # postfix_relayhost_username: "myusername"
 # postfix_relayhost_password: "mysupersecretpassword"
+
+## Deploy user customization - You can customize the deploy user below if you'd like
+# docker_user:
+#   username: deploy
+#   authorized_ssh_keys: 
+#     - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKNJGtd7a4DBHsQi7HGrC5xz0eAEFHZ3Ogh3FEFI2345 fake@key"
+#     - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFRfXxUZ8q9vHRcQZ6tLb0KwGHu8xjQHfYopZKLmnopQ anotherfake@key"
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -47,4 +47,11 @@ molecule create # Builds the container
 molecule converge # Runs the playbook
 molecule verify # Runs the tests
 molecule destroy # Destroys the container
+```
+
+## Testing the collection
+Instead of committing to a brach and testing on another machine, it might be easier to just build the collection and install it locally. This will build and install the collection locally on your machine. Look at the file `dev.sh` to see how this is done.
+
+```bash
+bash dev.sh
 ```
diff --git a/dev.sh b/dev.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+set -x
+
+# Parse command line arguments
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --debug)
+      export ANSIBLE_STDOUT_CALLBACK=debug
+      shift
+      ;;
+    *)
+      extra_arguments+=("$1")
+      shift
+      ;;
+  esac
+done
+
+# Set environment variables
+export ANSIBLE_WORK_DIR="${ANSIBLE_WORK_DIR:-$(pwd)}"
+export ANSIBLE_VARIABLE_FILE_NAME="${ANSIBLE_VARIABLE_FILE_NAME:-".spin.yml"}"
+export ANSIBLE_VARIABLE_FILEPATH="${ANSIBLE_VARIABLE_FILEPATH:-"${ANSIBLE_WORK_DIR}/${ANSIBLE_VARIABLE_FILE_NAME}"}"
+
+# Use ANSIBLE_VARIABLE_FILEPATH instead of variable_file_path
+variable_file_path="${ANSIBLE_VARIABLE_FILEPATH}"
+
+version=$(awk '/version:/ {print $2; exit}' galaxy.yml)
+ansible-galaxy collection build --force
+ansible-galaxy collection install "serversideup-spin-${version}.tar.gz" --force
+ansible-playbook -i plugins/inventory/spin-dynamic-inventory.sh playbooks/provision.yml --extra-vars "@${variable_file_path}" "${extra_arguments[@]}"
diff --git a/galaxy.yml b/galaxy.yml
@@ -17,6 +17,9 @@ dependencies:
   ansible.posix: '*'
   community.general: '*'
   community.docker: '*'
+  hetzner.hcloud: '*'
+  vultr.cloud: '*'
+  community.digitalocean: '*'
 repository: https://github.com/serversideup/ansible-collection-spin
 documentation: https://serversideup.net/open-source/spin/docs/
 homepage: https://serversideup.net/open-source/spin/

diff --git a/meta/runtime.yml b/meta/runtime.yml
@@ -1,2 +1,2 @@
 ---
-requires_ansible: '>=2.13.0'
+requires_ansible: '>=2.15.0'
diff --git a/molecule/default/converge.yml b/molecule/default/converge.yml
@@ -7,16 +7,18 @@
 
   pre_tasks:
     - name: Update apt cache.
-      apt: update_cache=yes cache_valid_time=600
+      ansible.builtin.apt:
+        update_cache: yes
+        cache_valid_time: 600
       when: ansible_os_family == 'Debian'
 
     - name: Install PIP dependencies.
-      pip:
+      ansible.builtin.pip:
         name: docker
         state: present
 
     - name: Wait for systemd to complete initialization.  # noqa 303
-      command: systemctl is-system-running
+      ansible.builtin.command: systemctl is-system-running
       register: systemctl_status
       until: >
         'running' in systemctl_status.stdout or

diff --git a/playbooks/get_variable.yml b/playbooks/get_variable.yml
@@ -0,0 +1,16 @@
+---
+- name: Get variable value
+  hosts: localhost
+  gather_facts: false
+  tasks:
+    - name: Display failure if variable_name is not set
+      ansible.builtin.fail:
+        msg: "variable_name is not set. Use -e variable_name=<name_of_variable>"
+      when: variable_name is not defined
+      run_once: true
+
+    - name: Output variable value
+      ansible.builtin.debug:
+        msg: "{{ lookup('vars', variable_name) }}"
+      when: variable_name in vars
+      run_once: true
diff --git a/playbooks/maintain.yml b/playbooks/maintain.yml
@@ -0,0 +1,11 @@
+---
+- name: Perform server maintenance tasks.
+  hosts: "{{ target | default('all') }}"
+  remote_user: "{{ spin_remote_user | default('root') }}"
+  become: true
+  vars:
+    ansible_port: "{{ ssh_port }}"
+    ansible_ssh_common_args: "-o IgnoreUnknown=UseKeychain -o StrictHostKeyChecking=accept-new"
+    ansible_python_interpreter: auto_silent
+  roles:
+    - serversideup.spin.update_server 
diff --git a/playbooks/prepare_ci_environment.yml b/playbooks/prepare_ci_environment.yml
@@ -0,0 +1,93 @@
+---
+- name: Prepare CI environment.
+  hosts: localhost
+  gather_facts: false
+  tasks:
+    - name: Display failure if required variables are not set.
+      ansible.builtin.fail:
+        msg: "{{ item }} variable is not set. Use -e {{ item }}=<env_value>"
+      when: item is not defined
+      loop:
+        - spin_environment
+        - spin_ci_folder
+      tags:
+        - always
+        - get-host
+
+    # Support for Spin v2 inventory names, add them to the new managers group
+    - name: Add hosts from manager_servers group to managers group if it exists
+      ansible.builtin.add_host:
+        name: "{{ item }}"
+        groups: "{{ spin_environment }}_managers"
+      when: groups[spin_environment + '_manager_servers'] is defined
+      loop: "{{ groups[spin_environment + '_manager_servers'] | default([]) }}"
+      tags:
+        - always
+        - get-host
+
+    - name: Validate inventory groups exist
+      ansible.builtin.fail:
+        msg: "Required inventory group '{{ spin_environment }}_managers' is empty or not defined. Verify your .spin.yml file or custom inventory file."
+      when: >-
+        groups[spin_environment + '_managers'] is not defined or
+        groups[spin_environment + '_managers'] | length == 0
+      tags:
+        - always
+        - get-host
+
+    - name: Set fact of full path to CI folder
+      ansible.builtin.set_fact:
+        spin_ci_folder_full_path: "{{ spin_ci_folder_full_path | default('/ansible/' + spin_ci_folder) }}"
+      tags:
+        - always
+        - get-host
+
+    - name: Set {{ spin_environment | upper }}_SSH_REMOTE_HOSTNAME with first manager host
+      ansible.builtin.copy:
+        content: "{{ groups[spin_environment + '_managers'][0] }}"
+        dest: "{{ spin_ci_folder_full_path }}/{{ spin_environment | upper }}_SSH_REMOTE_HOSTNAME"
+      when: groups[spin_environment + '_managers'] is defined and groups[spin_environment + '_managers'] | length > 0
+      tags:
+        - always
+        - get-host
+
+    - name: Set AUTHORIZED_KEYS file with sudo users' SSH keys
+      ansible.builtin.copy:
+        content: "{{ users | selectattr('groups', 'contains', 'sudo') | map(attribute='authorized_keys') | flatten | map(attribute='public_key') | join('\n') }}\n"
+        dest: "{{ spin_ci_folder_full_path }}/AUTHORIZED_KEYS"
+        mode: '0600'
+      tags:
+        - always
+        - get-authorized-keys
+
+    - name: Run ssh-keyscan on manager hosts.
+      ansible.builtin.shell: >-
+        ssh-keyscan -p {{ ssh_port }} {{ groups[spin_environment + '_managers'] | join(' ') }} | sort
+      register: keyscan_result
+      when: groups[spin_environment + '_managers'] is defined and groups[spin_environment + '_managers'] | length > 0
+      changed_when: false
+
+    - name: Write SSH_REMOTE_KNOWN_HOSTS file
+      ansible.builtin.copy:
+        content: "{{ keyscan_result.stdout }}\n"
+        dest: "{{ spin_ci_folder_full_path }}/SSH_REMOTE_KNOWN_HOSTS"
+        mode: '0600'
+      when: keyscan_result.stdout is defined
+
+
+- name: Update deploy user authorized keys.
+  hosts: "{{ target | default('all') }}"
+  remote_user: "{{ spin_remote_user | default('root') }}"
+  gather_facts: false
+  become: true
+  vars:
+    ansible_port: "{{ ssh_port }}"
+    ansible_ssh_common_args: "-o IgnoreUnknown=UseKeychain -o StrictHostKeyChecking=accept-new"
+    ansible_python_interpreter: auto_silent
+  pre_tasks:
+    - name: Display failure if deploy_public_key is not set.
+      ansible.builtin.fail:
+        msg: "deploy_public_key variable is not set. Use -e deploy_public_key=<key_value>"
+      when: deploy_public_key is not defined
+  roles:
+    - role: serversideup.spin.docker_user