From 76f7e8bc5a27cff181589699b3ff0f4e1c3558fc Mon Sep 17 00:00:00 2001 From: Georg Schulz <112310008+GeorgSchulz@users.noreply.github.com> Date: Mon, 21 Oct 2024 17:15:59 +0200 Subject: [PATCH] #53: Up to date and maintenance of all Ansible Collections - Non-HA and HA (#54) --- CHANGELOG.md | 4 + README.md | 46 +-- .../extensions/molecule/ha_setup/converge.yml | 10 + .../extensions/molecule/ha_setup/create.yml | 103 +++++++ .../extensions/molecule/ha_setup/destroy.yml | 22 ++ .../extensions/molecule/ha_setup/molecule.yml | 95 ++++++ .../molecule/ha_setup/requirements.yml | 3 + .../molecule/ha_setup/tasks/create-fail.yml | 14 + .../server/tasks/configure-primary-kdc.yml | 4 +- .../server/tasks/configure-secondary-kdc.yml | 4 + .../roles/server/tasks/install-ha.yml | 12 +- .../server/tasks/setup-kerberos-server-ha.yml | 4 +- .../extensions/molecule/default/molecule.yml | 12 + .../extensions/molecule/ha_setup/converge.yml | 8 + .../extensions/molecule/ha_setup/create.yml | 93 ++++++ .../extensions/molecule/ha_setup/destroy.yml | 22 ++ .../extensions/molecule/ha_setup/molecule.yml | 91 ++++++ .../molecule/ha_setup/requirements.yml | 3 + .../molecule/ha_setup/tasks/create-fail.yml | 14 + .../ldap/roles/check/tasks/main.yml | 21 +- .../tasks/configure-ldap-provider.yml | 4 +- .../ldap/roles/replication/tasks/main.yml | 4 +- .../templates/consumer_simple_sync.ldif | 3 +- .../ldap/roles/server/templates/ssl.ldif | 7 - .../extensions/molecule/default/molecule.yml | 12 + .../extensions/molecule/ha_setup/converge.yml | 8 + .../extensions/molecule/ha_setup/create.yml | 88 ++++++ .../extensions/molecule/ha_setup/destroy.yml | 22 ++ .../extensions/molecule/ha_setup/molecule.yml | 83 ++++++ .../molecule/ha_setup/requirements.yml | 3 + .../molecule/ha_setup/tasks/create-fail.yml | 14 + .../zookeeper/roles/install/templates/zoo.cfg | 2 + .../client/roles/setup/defaults/main.yml | 6 +- .../common/roles/setup/defaults/main.yml | 4 +- .../hadoop/common/roles/setup/tasks/main.yml | 13 - .../roles/setup/templates/core-site-ha.xml | 54 ++-- .../roles/setup/templates/core-site.xml | 3 +- .../roles/setup/templates/ssl-server.xml | 7 - .../hadoop/common/roles/setup/vars/main.yml | 11 +- .../extensions/molecule/default/create.yml | 3 - .../extensions/molecule/ha_setup/converge.yml | 57 ++++ .../extensions/molecule/ha_setup/create.yml | 137 +++++++++ .../extensions/molecule/ha_setup/destroy.yml | 22 ++ .../extensions/molecule/ha_setup/molecule.yml | 158 ++++++++++ .../molecule/ha_setup/requirements.yml | 3 + .../molecule/ha_setup/tasks/create-fail.yml | 14 + .../hadoop/hdfs/roles/check/tasks/main.yml | 4 +- .../hdfs/roles/common/defaults/main.yml | 2 +- .../roles/common/templates/hdfs-site-ha.xml | 56 ++-- .../hdfs/roles/journalnode/tasks/main.yml | 4 +- .../templates/hdfs-journalnode.service | 5 +- .../hadoop/hdfs/roles/namenode/tasks/main.yml | 34 +-- .../namenode/templates/hdfs-namenode.service | 2 +- .../extensions/molecule/ha_setup/converge.yml | 84 ++++++ .../extensions/molecule/ha_setup/create.yml | 172 +++++++++++ .../extensions/molecule/ha_setup/destroy.yml | 22 ++ .../extensions/molecule/ha_setup/molecule.yml | 165 +++++++++++ .../molecule/ha_setup/requirements.yml | 3 + .../molecule/ha_setup/tasks/create-fail.yml | 14 + .../hadoop/yarn/roles/check/tasks/main.yml | 3 +- .../yarn/roles/common/defaults/main.yml | 2 +- .../roles/common/templates/yarn-site-ha.xml | 14 +- .../yarn/roles/resourcemanager/tasks/main.yml | 4 +- .../extensions/molecule/ha_setup/converge.yml | 115 ++++++++ .../extensions/molecule/ha_setup/create.yml | 217 ++++++++++++++ .../extensions/molecule/ha_setup/destroy.yml | 22 ++ .../extensions/molecule/ha_setup/molecule.yml | 266 +++++++++++++++++ .../molecule/ha_setup/requirements.yml | 3 + .../molecule/ha_setup/tasks/create-fail.yml | 14 + .../install/files/jupyterhubenvironment.txt | 38 +-- .../setup/files/jupyterlabenvironment.txt | 174 ++++++----- .../keepalived/roles/setup/tasks/main.yml | 6 +- .../roles/setup/templates/failover.sh | 2 +- .../roles/setup/templates/keepalived.conf | 2 +- .../extensions/molecule/ha_setup/molecule.yml | 6 +- .../roles/install/tasks/install-postgres.yml | 3 + .../postgres/roles/schemes/tasks/main.yml | 16 + .../certs/roles/distribute/defaults/main.yml | 12 +- .../tls/certs/roles/distribute/tasks/main.yml | 15 + .../issue/tasks/issue-selfsigned-certs.yml | 274 +++++++++++++----- .../tls/certs/roles/setup/meta/main.yml | 1 + .../java/roles/truststore/defaults/main.yml | 2 +- requirements.sh | 4 +- setup.yml | 3 +- terraform/inventory_ha_ide.tpl | 16 +- terraform/inventory_non_ha_ide.tpl | 1 - 86 files changed, 2761 insertions(+), 368 deletions(-) create mode 100644 collections/ansible_collections/authentication/kerberos/extensions/molecule/ha_setup/converge.yml create mode 100644 collections/ansible_collections/authentication/kerberos/extensions/molecule/ha_setup/create.yml create mode 100644 collections/ansible_collections/authentication/kerberos/extensions/molecule/ha_setup/destroy.yml create mode 100644 collections/ansible_collections/authentication/kerberos/extensions/molecule/ha_setup/molecule.yml create mode 100644 collections/ansible_collections/authentication/kerberos/extensions/molecule/ha_setup/requirements.yml create mode 100644 collections/ansible_collections/authentication/kerberos/extensions/molecule/ha_setup/tasks/create-fail.yml create mode 100644 collections/ansible_collections/authorization/ldap/extensions/molecule/ha_setup/converge.yml create mode 100644 collections/ansible_collections/authorization/ldap/extensions/molecule/ha_setup/create.yml create mode 100644 collections/ansible_collections/authorization/ldap/extensions/molecule/ha_setup/destroy.yml create mode 100644 collections/ansible_collections/authorization/ldap/extensions/molecule/ha_setup/molecule.yml create mode 100644 collections/ansible_collections/authorization/ldap/extensions/molecule/ha_setup/requirements.yml create mode 100644 collections/ansible_collections/authorization/ldap/extensions/molecule/ha_setup/tasks/create-fail.yml create mode 100644 collections/ansible_collections/bigdata/zookeeper/extensions/molecule/ha_setup/converge.yml create mode 100644 collections/ansible_collections/bigdata/zookeeper/extensions/molecule/ha_setup/create.yml create mode 100644 collections/ansible_collections/bigdata/zookeeper/extensions/molecule/ha_setup/destroy.yml create mode 100644 collections/ansible_collections/bigdata/zookeeper/extensions/molecule/ha_setup/molecule.yml create mode 100644 collections/ansible_collections/bigdata/zookeeper/extensions/molecule/ha_setup/requirements.yml create mode 100644 collections/ansible_collections/bigdata/zookeeper/extensions/molecule/ha_setup/tasks/create-fail.yml create mode 100644 collections/ansible_collections/hadoop/hdfs/extensions/molecule/ha_setup/converge.yml create mode 100644 collections/ansible_collections/hadoop/hdfs/extensions/molecule/ha_setup/create.yml create mode 100644 collections/ansible_collections/hadoop/hdfs/extensions/molecule/ha_setup/destroy.yml create mode 100644 collections/ansible_collections/hadoop/hdfs/extensions/molecule/ha_setup/molecule.yml create mode 100644 collections/ansible_collections/hadoop/hdfs/extensions/molecule/ha_setup/requirements.yml create mode 100644 collections/ansible_collections/hadoop/hdfs/extensions/molecule/ha_setup/tasks/create-fail.yml create mode 100644 collections/ansible_collections/hadoop/yarn/extensions/molecule/ha_setup/converge.yml create mode 100644 collections/ansible_collections/hadoop/yarn/extensions/molecule/ha_setup/create.yml create mode 100644 collections/ansible_collections/hadoop/yarn/extensions/molecule/ha_setup/destroy.yml create mode 100644 collections/ansible_collections/hadoop/yarn/extensions/molecule/ha_setup/molecule.yml create mode 100644 collections/ansible_collections/hadoop/yarn/extensions/molecule/ha_setup/requirements.yml create mode 100644 collections/ansible_collections/hadoop/yarn/extensions/molecule/ha_setup/tasks/create-fail.yml create mode 100644 collections/ansible_collections/jupyter/hub/extensions/molecule/ha_setup/converge.yml create mode 100644 collections/ansible_collections/jupyter/hub/extensions/molecule/ha_setup/create.yml create mode 100644 collections/ansible_collections/jupyter/hub/extensions/molecule/ha_setup/destroy.yml create mode 100644 collections/ansible_collections/jupyter/hub/extensions/molecule/ha_setup/molecule.yml create mode 100644 collections/ansible_collections/jupyter/hub/extensions/molecule/ha_setup/requirements.yml create mode 100644 collections/ansible_collections/jupyter/hub/extensions/molecule/ha_setup/tasks/create-fail.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 413b4f8..cdf1ce0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,9 +8,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] - communitylab#26: Error handling in Conda Package Manager +- communitylab#52: Consider usage of Jupyter Enterprise Gateway instead of not maintained Jupyter on Hadoop + +## [v1.5.0] - 2024-10-21 ### Added +- communitylab#53: Up to date and maintenance of all Ansible Collections - Non-HA and HA - communitylab#50: Simplifying architecture for IDE Non-HA setup - communitylab#48: Downgrading OpenJDK and updating documentation images diff --git a/README.md b/README.md index 5db7774..9dae463 100644 --- a/README.md +++ b/README.md @@ -59,11 +59,11 @@ If you are german speaking you may be interested in my related academic work: [T ## 1. Prerequisites ### required - Ubuntu (was tested on Ubuntu 24.04 LTS) -- Ansible (was tested on Ansible version 2.17.3) -- Python (was tested on Python version 3.12.5) +- Ansible (was tested on Ansible version 2.17.4) +- Python (was tested on Python version 3.12.6) ### optional -- Molecule (was tested on Molecule version 24.7.0) +- Molecule (was tested on Molecule version 24.9.0) - Docker (was tested on Docker version 27.1.2, required for Ansible Molecule) - Terraform (was tested on Terraform version v1.9.3) - Go (was tested on Go version go1.18.1) @@ -80,24 +80,30 @@ georg@notebook:~/git/CommunityLab$ bash requirements.sh ### 2.2 Ansible Molecule is available for following Ansible Collections ```console -georg@notebook:~/git/CommunityLab$ find . -name extensions -./collections/ansible_collections/jupyter/hub/extensions -./collections/ansible_collections/authentication/kerberos/extensions -./collections/ansible_collections/hadoop/hdfs/extensions -./collections/ansible_collections/hadoop/yarn/extensions -./collections/ansible_collections/bigdata/spark/extensions -./collections/ansible_collections/bigdata/zookeeper/extensions -./collections/ansible_collections/rdbms/postgres/extensions -./collections/ansible_collections/authorization/ldap/extensions +georg@notebook:~/git/CommunityLab$ find . -name molecule +./collections/ansible_collections/jupyter/hub/extensions/molecule +./collections/ansible_collections/authentication/kerberos/extensions/molecule +./collections/ansible_collections/hadoop/hdfs/extensions/molecule +./collections/ansible_collections/hadoop/yarn/extensions/molecule +./collections/ansible_collections/bigdata/spark/extensions/molecule +./collections/ansible_collections/bigdata/zookeeper/extensions/molecule +./collections/ansible_collections/rdbms/postgres/extensions/molecule +./collections/ansible_collections/authorization/ldap/extensions/molecule ``` -### 2.3 The IDE can be installed using Ansible Molecule like this (Docker is required) +### 2.3 Each Ansible Collection has two Ansible Molecule scenarios - default and ha_setup +##### (The Ansible Collection bigdata.spark is an exception since the only purpose is the installation of common Apache Spark libraries) + +- default (Simple installation process of the Ansible Collection without High Availability) +- ha_setup (More complex installation process of the Ansible Collection with High Availability) + +### 2.4 The IDE can be installed using Ansible Molecule like this (Docker is required) - Non-HA IDE ```console georg@notebook:~/git/CommunityLab$ cd collections/ansible_collections/jupyter/hub/extensions/ -georg@notebook:~/git/CommunityLab/collections/ansible_collections/jupyter/hub/extensions$ molecule converge +georg@notebook:~/git/CommunityLab/collections/ansible_collections/jupyter/hub/extensions$ molecule converge -s default ``` -### 2.4 Check container created by Ansible Molecule +### 2.5 Check container created by Ansible Molecule ```console georg@notebook:~/git/CommunityLab/collections/ansible_collections/jupyter/hub/extensions$ docker container ls CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES @@ -109,25 +115,25 @@ a33331864a0e geerlingguy/docker-ubuntu2404-ansible "/usr/lib/systemd/sy…" 16591b433003 geerlingguy/docker-ubuntu2404-ansible "/usr/lib/systemd/sy…" About an hour ago Up About an hour instance-1 ``` -### 2.5 Get IP address of Docker container running JupyterHub +### 2.6 Get IP address of Docker container running JupyterHub ```console georg@notebook:~/git/CommunityLab/collections/ansible_collections/jupyter/hub/extensions$ docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' instance-1 172.23.27.3 ``` -### 2.6 Start Firefox browser +### 2.7 Start Firefox browser ```console georg@notebook:~/git/CommunityLab/collections/ansible_collections/jupyter/hub/extensions$ firefox ``` -### 2.7 Login to JupyterHub here using credentials of variable [ldap_users](./collections/ansible_collections/jupyter/hub/extensions/molecule/default/molecule.yml): +### 2.8 Login to JupyterHub here using credentials of variable [ldap_users](./collections/ansible_collections/jupyter/hub/extensions/molecule/default/molecule.yml): https://172.23.27.3:8443 ![IDE Docker](https://github.com/GeorgSchulz/CommunityLab/blob/master/images/ide_in_docker.bmp?raw=True) -### 2.6 Delete all Docker container using Ansible Molecule +### 2.9 Delete all Docker container using Ansible Molecule ```console -georg@notebook:~/git/CommunityLab/collections/ansible_collections/jupyter/hub/extensions$ molecule destroy +georg@notebook:~/git/CommunityLab/collections/ansible_collections/jupyter/hub/extensions$ molecule destroy -s default ``` ## 3. Use Hetzner Cloud diff --git a/collections/ansible_collections/authentication/kerberos/extensions/molecule/ha_setup/converge.yml b/collections/ansible_collections/authentication/kerberos/extensions/molecule/ha_setup/converge.yml new file mode 100644 index 0000000..e05314f --- /dev/null +++ b/collections/ansible_collections/authentication/kerberos/extensions/molecule/ha_setup/converge.yml @@ -0,0 +1,10 @@ +--- +- name: Setup Kerberos server + hosts: kerberos + gather_facts: true + roles: + - role: ide.environment.user_and_groups + - role: tls.certs.issue + - role: tls.certs.distribute + - role: authorization.ldap.setup + - role: authentication.kerberos.setup diff --git a/collections/ansible_collections/authentication/kerberos/extensions/molecule/ha_setup/create.yml b/collections/ansible_collections/authentication/kerberos/extensions/molecule/ha_setup/create.yml new file mode 100644 index 0000000..b38d1ec --- /dev/null +++ b/collections/ansible_collections/authentication/kerberos/extensions/molecule/ha_setup/create.yml @@ -0,0 +1,103 @@ +--- +- name: Create + hosts: localhost + gather_facts: false + vars: + molecule_inventory: + all: + hosts: {} + tasks: + - name: Create docker network + community.docker.docker_network: + name: example.com + ipam_config: + - subnet: 172.23.27.0/24 + gateway: 172.23.27.2 + iprange: 172.23.27.0/26 + + - name: Create container + become: true + community.docker.docker_container: + name: "{{ item.name }}" + image: "{{ item.image }}" + state: started + command: "{{ item.command }}" + log_driver: json-file + publish_all_ports: true + cgroupns_mode: "{{ item.cgroupns_mode }}" + privileged: "{{ item.privileged }}" + volumes: "{{ item.volumes }}" + networks: + - name: example.com + ipv4_address: "{{ item.ipv4_address }}" + hostname: "{{ item.name }}.example.com" + tls_hostname: "{{ item.name }}.example.com" + capabilities: + - NET_ADMIN + register: result + loop: "{{ molecule_yml.platforms }}" + + - name: Fail if container is not running + when: > + item.container.State.ExitCode != 0 or + not item.container.State.Running + ansible.builtin.include_tasks: + file: tasks/create-fail.yml + loop: "{{ result.results }}" + loop_control: + label: "{{ item.container.Name }}" + + - name: Add container to molecule_inventory + vars: + inventory_partial_yaml: | + all: + vars: + ansible_connection: community.docker.docker + children: + kerberos1: + hosts: + instance-1: + kerberos2: + hosts: + instance-2: + kerberos: + children: + kerberos1: + kerberos2: + ldap1: + hosts: + instance-1: + ldap2: + hosts: + instance-2: + ldap: + children: + ldap1: + ldap2: + ansible.builtin.set_fact: + molecule_inventory: > + {{ molecule_inventory | combine(inventory_partial_yaml | from_yaml, recursive=true) }} + + - name: Dump molecule_inventory + ansible.builtin.copy: + content: | + {{ molecule_inventory | to_yaml }} + dest: "{{ molecule_ephemeral_directory }}/inventory/molecule_inventory.yml" + mode: "0600" + + - name: Force inventory refresh + ansible.builtin.meta: refresh_inventory + +# we want to avoid errors like "Failed to create temporary directory" +- name: Validate that inventory was refreshed + hosts: all + gather_facts: false + tasks: + - name: Check uname + ansible.builtin.raw: uname -a + register: result + changed_when: false + + - name: Display uname info + ansible.builtin.debug: + msg: "{{ result.stdout }}" diff --git a/collections/ansible_collections/authentication/kerberos/extensions/molecule/ha_setup/destroy.yml b/collections/ansible_collections/authentication/kerberos/extensions/molecule/ha_setup/destroy.yml new file mode 100644 index 0000000..e6af2ce --- /dev/null +++ b/collections/ansible_collections/authentication/kerberos/extensions/molecule/ha_setup/destroy.yml @@ -0,0 +1,22 @@ +--- +- name: Destroy molecule containers + hosts: all + gather_facts: false + tasks: + - name: Stop and remove container + delegate_to: localhost + become: true + community.docker.docker_container: + name: "{{ inventory_hostname }}" + state: absent + auto_remove: true + +- name: Remove dynamic molecule inventory + hosts: localhost + gather_facts: false + tasks: + - name: Remove dynamic inventory file + become: true + ansible.builtin.file: + path: "{{ molecule_ephemeral_directory }}/inventory/molecule_inventory.yml" + state: absent diff --git a/collections/ansible_collections/authentication/kerberos/extensions/molecule/ha_setup/molecule.yml b/collections/ansible_collections/authentication/kerberos/extensions/molecule/ha_setup/molecule.yml new file mode 100644 index 0000000..9eaa598 --- /dev/null +++ b/collections/ansible_collections/authentication/kerberos/extensions/molecule/ha_setup/molecule.yml @@ -0,0 +1,95 @@ +--- +dependency: + name: galaxy + options: + requirements-file: requirements.yml +platforms: + - name: instance-1 + image: docker.io/geerlingguy/docker-ubuntu2404-ansible + command: /usr/lib/systemd/systemd + pre_build_image: true + cgroupns_mode: host + ipv4_address: 172.23.27.3 + privileged: true + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + - /var/lib/containerd + - name: instance-2 + image: docker.io/geerlingguy/docker-ubuntu2404-ansible + command: /usr/lib/systemd/systemd + pre_build_image: true + cgroupns_mode: host + ipv4_address: 172.23.27.4 + privileged: true + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + - /var/lib/containerd +provisioner: + name: ansible + log: false + options: + vvv: false + playbooks: + converge: converge.yml + inventory: + group_vars: + all: + molecule_deployment: true + ldap_external: false + ldap_user: openldap + ldap_uid: "5001" + ldap_group: openldap + ldap_gid: "4001" + service_user: "{{ ldap_user }}" + service_name: Open LDAP + service_uid: "{{ ldap_uid }}" + service_group: "{{ ldap_group }}" + service_gid: "{{ ldap_gid }}" + tls_user: "{{ ldap_user }}" + tls_group: "{{ ldap_group }}" + self_signed_certificates: true + certs_source: + - /tmp/selfsigned_certs/{{ ansible_fqdn }}/cert.pem + - /tmp/selfsigned_certs/{{ ansible_fqdn }}/chain.pem + - /tmp/selfsigned_certs/{{ ansible_fqdn }}/key.pem + certs_dest: + - cert.pem + - chain.pem + - key.pem + certs_mode: + - "0660" + - "0660" + - "0400" + domain: example.com + keystore_file: /etc/ssl/private/{{ ansible_fqdn }}.jks + keystore_password: changeit + truststore_file: /etc/ssl/certs/truststore.jks + truststore_password: changeit + ldap_server_address: "{% for host in groups.ldap | shuffle %}ldaps://{{ host }}.example.com:636{% if not loop.last %},{% endif %}{% endfor %}" + ldap_organization: dc=example,dc=com + ldap_user_search_base: ou=people,{{ ldap_organization }} + ldap_group_search_base: ou=groups,{{ ldap_organization }} + ldap_bind_user: cn=admin,{{ ldap_organization }} + ldap_password: changeit + ldap_bind_dn_template: uid={username},{{ ldap_user_search_base }} + ide_users_group: ide_users + ide_users_gid: "5001" + ldap_users: + - name: teppler + uid_number: 6000 + gid_number: "{{ ide_users_gid }}" + password: albstadt + - name: anolle + uid_number: 6001 + gid_number: "{{ ide_users_gid }}" + password: sigmaringen + - name: gschulz + uid_number: 6002 + gid_number: "{{ ide_users_gid }}" + password: datascience + ldap_replication_user: cn=replicator,{{ ldap_organization }} + ldap_replication_password: changeit + ldap_kdc_service_password: changeit + ldap_kadmin_service_password: changeit + realm: COMMUNITY.LAB + realm_password: changeit diff --git a/collections/ansible_collections/authentication/kerberos/extensions/molecule/ha_setup/requirements.yml b/collections/ansible_collections/authentication/kerberos/extensions/molecule/ha_setup/requirements.yml new file mode 100644 index 0000000..660f775 --- /dev/null +++ b/collections/ansible_collections/authentication/kerberos/extensions/molecule/ha_setup/requirements.yml @@ -0,0 +1,3 @@ +--- +collections: + - name: community.docker diff --git a/collections/ansible_collections/authentication/kerberos/extensions/molecule/ha_setup/tasks/create-fail.yml b/collections/ansible_collections/authentication/kerberos/extensions/molecule/ha_setup/tasks/create-fail.yml new file mode 100644 index 0000000..4227c46 --- /dev/null +++ b/collections/ansible_collections/authentication/kerberos/extensions/molecule/ha_setup/tasks/create-fail.yml @@ -0,0 +1,14 @@ +--- +- name: Retrieve container log + ansible.builtin.command: + cmd: >- + {% raw %} + docker logs + {% endraw %} + {{ item.stdout_lines[0] }} + changed_when: false + register: logfile_cmd + +- name: Display container log + ansible.builtin.fail: + msg: "{{ logfile_cmd.stderr }}" diff --git a/collections/ansible_collections/authentication/kerberos/roles/server/tasks/configure-primary-kdc.yml b/collections/ansible_collections/authentication/kerberos/roles/server/tasks/configure-primary-kdc.yml index 1f97849..b3b581d 100644 --- a/collections/ansible_collections/authentication/kerberos/roles/server/tasks/configure-primary-kdc.yml +++ b/collections/ansible_collections/authentication/kerberos/roles/server/tasks/configure-primary-kdc.yml @@ -104,10 +104,10 @@ - krb5-admin-server - name: Fetch stash and service.keyfile to ansible server - ansible.posix.synchronize: + ansible.builtin.fetch: src: /etc/krb5kdc/{{ item }} dest: /tmp/ - mode: pull + flat: true loop: - .k5.{{ realm }} - service.keyfile diff --git a/collections/ansible_collections/authentication/kerberos/roles/server/tasks/configure-secondary-kdc.yml b/collections/ansible_collections/authentication/kerberos/roles/server/tasks/configure-secondary-kdc.yml index d351f39..747ea56 100644 --- a/collections/ansible_collections/authentication/kerberos/roles/server/tasks/configure-secondary-kdc.yml +++ b/collections/ansible_collections/authentication/kerberos/roles/server/tasks/configure-secondary-kdc.yml @@ -61,6 +61,10 @@ - krb5-kdc - krb5-admin-server +- name: Pause 1 minute for krb5-kdc service to start + ansible.builtin.pause: + minutes: 1 + - name: Start Kerberos services ansible.builtin.systemd: name: "{{ item }}" diff --git a/collections/ansible_collections/authentication/kerberos/roles/server/tasks/install-ha.yml b/collections/ansible_collections/authentication/kerberos/roles/server/tasks/install-ha.yml index 5ea46ff..135f639 100644 --- a/collections/ansible_collections/authentication/kerberos/roles/server/tasks/install-ha.yml +++ b/collections/ansible_collections/authentication/kerberos/roles/server/tasks/install-ha.yml @@ -1,11 +1,19 @@ --- +- name: Ensure kerberos.schema.gz is present after installation + when: + - molecule_deployment is defined + - molecule_deployment + ansible.builtin.file: + path: /etc/dpkg/dpkg.cfg.d/excludes + state: absent + - name: Install necessary packages on primary KDC ansible.builtin.apt: name: "{{ item }}" state: present update_cache: true loop: "{{ server_primary_kdc_packages }}" - when: '"security1" in group_names' + when: '"kerberos1" in group_names' - name: Install necessary packages on secondary KDC ansible.builtin.apt: @@ -13,4 +21,4 @@ state: present update_cache: true loop: "{{ server_secondary_kdc_packages }}" - when: '"security2" in group_names' + when: '"kerberos2" in group_names' diff --git a/collections/ansible_collections/authentication/kerberos/roles/server/tasks/setup-kerberos-server-ha.yml b/collections/ansible_collections/authentication/kerberos/roles/server/tasks/setup-kerberos-server-ha.yml index beb1d72..c8af088 100644 --- a/collections/ansible_collections/authentication/kerberos/roles/server/tasks/setup-kerberos-server-ha.yml +++ b/collections/ansible_collections/authentication/kerberos/roles/server/tasks/setup-kerberos-server-ha.yml @@ -4,8 +4,8 @@ - name: Configure Kerberos primary KDC ansible.builtin.include_tasks: configure-primary-kdc.yml - when: "'security1' in group_names" + when: "'kerberos1' in group_names" - name: Configure Kerberos secondary KDC ansible.builtin.include_tasks: configure-secondary-kdc.yml - when: "'security2' in group_names" + when: "'kerberos2' in group_names" diff --git a/collections/ansible_collections/authorization/ldap/extensions/molecule/default/molecule.yml b/collections/ansible_collections/authorization/ldap/extensions/molecule/default/molecule.yml index bcef858..4143f23 100644 --- a/collections/ansible_collections/authorization/ldap/extensions/molecule/default/molecule.yml +++ b/collections/ansible_collections/authorization/ldap/extensions/molecule/default/molecule.yml @@ -38,6 +38,18 @@ provisioner: tls_user: "{{ ldap_user }}" tls_group: "{{ ldap_group }}" self_signed_certificates: true + certs_source: + - /tmp/selfsigned_certs/{{ ansible_fqdn }}/cert.pem + - /tmp/selfsigned_certs/{{ ansible_fqdn }}/chain.pem + - /tmp/selfsigned_certs/{{ ansible_fqdn }}/key.pem + certs_dest: + - cert.pem + - chain.pem + - key.pem + certs_mode: + - "0660" + - "0660" + - "0400" domain: example.com keystore_file: /etc/ssl/private/{{ ansible_fqdn }}.jks keystore_password: changeit diff --git a/collections/ansible_collections/authorization/ldap/extensions/molecule/ha_setup/converge.yml b/collections/ansible_collections/authorization/ldap/extensions/molecule/ha_setup/converge.yml new file mode 100644 index 0000000..f0d24b1 --- /dev/null +++ b/collections/ansible_collections/authorization/ldap/extensions/molecule/ha_setup/converge.yml @@ -0,0 +1,8 @@ +--- +- name: Setup LDAP server + hosts: ldap + gather_facts: true + roles: + - role: ide.environment.user_and_groups + - role: tls.certs.issue + - role: authorization.ldap.setup diff --git a/collections/ansible_collections/authorization/ldap/extensions/molecule/ha_setup/create.yml b/collections/ansible_collections/authorization/ldap/extensions/molecule/ha_setup/create.yml new file mode 100644 index 0000000..a3a92b6 --- /dev/null +++ b/collections/ansible_collections/authorization/ldap/extensions/molecule/ha_setup/create.yml @@ -0,0 +1,93 @@ +--- +- name: Create + hosts: localhost + gather_facts: false + vars: + molecule_inventory: + all: + hosts: {} + tasks: + - name: Create docker network + community.docker.docker_network: + name: example.com + ipam_config: + - subnet: 172.23.27.0/24 + gateway: 172.23.27.2 + iprange: 172.23.27.0/26 + + - name: Create container + become: true + community.docker.docker_container: + name: "{{ item.name }}" + image: "{{ item.image }}" + state: started + command: "{{ item.command }}" + log_driver: json-file + publish_all_ports: true + cgroupns_mode: "{{ item.cgroupns_mode }}" + privileged: "{{ item.privileged }}" + volumes: "{{ item.volumes }}" + networks: + - name: example.com + ipv4_address: "{{ item.ipv4_address }}" + hostname: "{{ item.name }}.example.com" + tls_hostname: "{{ item.name }}.example.com" + capabilities: + - NET_ADMIN + register: result + loop: "{{ molecule_yml.platforms }}" + + - name: Fail if container is not running + when: > + item.container.State.ExitCode != 0 or + not item.container.State.Running + ansible.builtin.include_tasks: + file: tasks/create-fail.yml + loop: "{{ result.results }}" + loop_control: + label: "{{ item.container.Name }}" + + - name: Add container to molecule_inventory + vars: + inventory_partial_yaml: | + all: + vars: + ansible_connection: community.docker.docker + children: + ldap1: + hosts: + instance-1: + ldap2: + hosts: + instance-2: + ldap: + children: + ldap1: + ldap2: + ansible.builtin.set_fact: + molecule_inventory: > + {{ molecule_inventory | combine(inventory_partial_yaml | from_yaml, recursive=true) }} + + - name: Dump molecule_inventory + ansible.builtin.copy: + content: | + {{ molecule_inventory | to_yaml }} + dest: "{{ molecule_ephemeral_directory }}/inventory/molecule_inventory.yml" + mode: "0600" + + - name: Force inventory refresh + ansible.builtin.meta: refresh_inventory + +# we want to avoid errors like "Failed to create temporary directory" +- name: Validate that inventory was refreshed + hosts: all + gather_facts: false + tasks: + - name: Check uname + ansible.builtin.raw: uname -a + register: result + changed_when: false + + - name: Display uname info + ansible.builtin.debug: + msg: "{{ result.stdout }}" diff --git a/collections/ansible_collections/authorization/ldap/extensions/molecule/ha_setup/destroy.yml b/collections/ansible_collections/authorization/ldap/extensions/molecule/ha_setup/destroy.yml new file mode 100644 index 0000000..e6af2ce --- /dev/null +++ b/collections/ansible_collections/authorization/ldap/extensions/molecule/ha_setup/destroy.yml @@ -0,0 +1,22 @@ +--- +- name: Destroy molecule containers + hosts: all + gather_facts: false + tasks: + - name: Stop and remove container + delegate_to: localhost + become: true + community.docker.docker_container: + name: "{{ inventory_hostname }}" + state: absent + auto_remove: true + +- name: Remove dynamic molecule inventory + hosts: localhost + gather_facts: false + tasks: + - name: Remove dynamic inventory file + become: true + ansible.builtin.file: + path: "{{ molecule_ephemeral_directory }}/inventory/molecule_inventory.yml" + state: absent diff --git a/collections/ansible_collections/authorization/ldap/extensions/molecule/ha_setup/molecule.yml b/collections/ansible_collections/authorization/ldap/extensions/molecule/ha_setup/molecule.yml new file mode 100644 index 0000000..ec0bd4d --- /dev/null +++ b/collections/ansible_collections/authorization/ldap/extensions/molecule/ha_setup/molecule.yml @@ -0,0 +1,91 @@ +--- +dependency: + name: galaxy + options: + requirements-file: requirements.yml +platforms: + - name: instance-1 + image: docker.io/geerlingguy/docker-ubuntu2404-ansible + command: /usr/lib/systemd/systemd + pre_build_image: true + cgroupns_mode: host + ipv4_address: 172.23.27.3 + privileged: true + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + - /var/lib/containerd + - name: instance-2 + image: docker.io/geerlingguy/docker-ubuntu2404-ansible + command: /usr/lib/systemd/systemd + pre_build_image: true + cgroupns_mode: host + ipv4_address: 172.23.27.4 + privileged: true + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + - /var/lib/containerd +provisioner: + name: ansible + log: false + options: + vvv: false + playbooks: + converge: converge.yml + inventory: + group_vars: + all: + molecule_deployment: true + ldap_external: false + ldap_user: openldap + ldap_uid: "5001" + ldap_group: openldap + ldap_gid: "4001" + service_user: "{{ ldap_user }}" + service_name: Open LDAP + service_uid: "{{ ldap_uid }}" + service_group: "{{ ldap_group }}" + service_gid: "{{ ldap_gid }}" + tls_user: "{{ ldap_user }}" + tls_group: "{{ ldap_group }}" + self_signed_certificates: true + certs_source: + - /tmp/selfsigned_certs/{{ ansible_fqdn }}/cert.pem + - /tmp/selfsigned_certs/{{ ansible_fqdn }}/chain.pem + - /tmp/selfsigned_certs/{{ ansible_fqdn }}/key.pem + certs_dest: + - cert.pem + - chain.pem + - key.pem + certs_mode: + - "0660" + - "0660" + - "0400" + domain: example.com + keystore_file: /etc/ssl/private/{{ ansible_fqdn }}.jks + keystore_password: changeit + truststore_file: /etc/ssl/certs/truststore.jks + truststore_password: changeit + ldap_server_address: "{% for host in groups.ldap | shuffle %}ldaps://{{ host }}.example.com:636{% if not loop.last %},{% endif %}{% endfor %}" + ldap_organization: dc=example,dc=com + ldap_user_search_base: ou=people,{{ ldap_organization }} + ldap_group_search_base: ou=groups,{{ ldap_organization }} + ldap_bind_user: cn=admin,{{ ldap_organization }} + ldap_password: changeit + ldap_bind_dn_template: uid={username},{{ ldap_user_search_base }} + ide_users_group: ide_users + ide_users_gid: "5001" + ldap_users: + - name: teppler + uid_number: 6000 + gid_number: "{{ ide_users_gid }}" + password: albstadt + - name: anolle + uid_number: 6001 + gid_number: "{{ ide_users_gid }}" + password: sigmaringen + - name: gschulz + uid_number: 6002 + gid_number: "{{ ide_users_gid }}" + password: datascience + ldap_replication_user: cn=replicator,{{ ldap_organization }} + ldap_replication_password: changeit diff --git a/collections/ansible_collections/authorization/ldap/extensions/molecule/ha_setup/requirements.yml b/collections/ansible_collections/authorization/ldap/extensions/molecule/ha_setup/requirements.yml new file mode 100644 index 0000000..660f775 --- /dev/null +++ b/collections/ansible_collections/authorization/ldap/extensions/molecule/ha_setup/requirements.yml @@ -0,0 +1,3 @@ +--- +collections: + - name: community.docker diff --git a/collections/ansible_collections/authorization/ldap/extensions/molecule/ha_setup/tasks/create-fail.yml b/collections/ansible_collections/authorization/ldap/extensions/molecule/ha_setup/tasks/create-fail.yml new file mode 100644 index 0000000..4227c46 --- /dev/null +++ b/collections/ansible_collections/authorization/ldap/extensions/molecule/ha_setup/tasks/create-fail.yml @@ -0,0 +1,14 @@ +--- +- name: Retrieve container log + ansible.builtin.command: + cmd: >- + {% raw %} + docker logs + {% endraw %} + {{ item.stdout_lines[0] }} + changed_when: false + register: logfile_cmd + +- name: Display container log + ansible.builtin.fail: + msg: "{{ logfile_cmd.stderr }}" diff --git a/collections/ansible_collections/authorization/ldap/roles/check/tasks/main.yml b/collections/ansible_collections/authorization/ldap/roles/check/tasks/main.yml index ac9a2f4..fea10a7 100644 --- a/collections/ansible_collections/authorization/ldap/roles/check/tasks/main.yml +++ b/collections/ansible_collections/authorization/ldap/roles/check/tasks/main.yml @@ -20,29 +20,34 @@ executable: /bin/bash register: ldap_search - - name: Print message depending on ldap check for one ldap server + - name: Print message depending on ldap check for one ldap server or Kerberos (LDAP) not present + when: groups.ldap | length == 1 or ldap_search.stdout_lines | length == ldap_users | length ansible.builtin.assert: that: ldap_search.stdout_lines | length == ldap_users | length fail_msg: Not all IDE users are added to 'cn={{ ide_users_group }},ou=groups,{{ ldap_organization }}', check Logs for more information success_msg: All IDE users are added to 'cn={{ ide_users_group }},ou=groups,{{ ldap_organization }}' - when: groups.ldap | length == 1 - - name: Print message depending on ldap check for two ldap servers + - name: Print message depending on ldap check for two ldap server and Kerberos (LDAP) present + when: + - groups.ldap | length == 2 + - ldap_search.stdout_lines | length == ldap_users | length + 2 ansible.builtin.assert: - that: ldap_search.stdout_lines | length == ldap_users | length + 3 + that: ldap_search.stdout_lines | length == ldap_users | length + 2 fail_msg: Not all IDE users are added to 'cn={{ ide_users_group }},ou=groups,{{ ldap_organization }}', check Logs for more information success_msg: All IDE users are added to 'cn={{ ide_users_group }},ou=groups,{{ ldap_organization }}' - when: groups.ldap | length == 2 - name: Set fact when all LDAP users are present ansible.builtin.set_fact: ldap_users_present: true - name: Only execute if two ldap server are present - when: groups.ldap | length == 2 + when: + - groups.ldap | length == 2 + - molecule_deployment is defined + - molecule_deployment block: - name: Check if ldap standard replication is active - ansible.builtin.command: ldapsearch -z1 -LLL -x -s base -b {{ ldap_organization }} contextCSN + ansible.builtin.command: ldapsearch -z1 -LLL -H ldapi:/// -s base -b {{ ldap_organization }} contextCSN register: ldap_search_context - name: Print message depending on ldap replication check @@ -50,7 +55,7 @@ that: - ldap_search_context.stdout_lines | length == 2 - ldap_search_context.rc == 0 - fail_msg: LDAP standard replicaiton is not active, check Logs for more information + fail_msg: LDAP standard replication is not active, check Logs for more information success_msg: "LDAP standard replication is configured and active: {{ ldap_search_context.stdout_lines }}" - name: Set fact when LDAP standard replication is configured and active diff --git a/collections/ansible_collections/authorization/ldap/roles/replication/tasks/configure-ldap-provider.yml b/collections/ansible_collections/authorization/ldap/roles/replication/tasks/configure-ldap-provider.yml index 3503a6c..37aea07 100644 --- a/collections/ansible_collections/authorization/ldap/roles/replication/tasks/configure-ldap-provider.yml +++ b/collections/ansible_collections/authorization/ldap/roles/replication/tasks/configure-ldap-provider.yml @@ -7,11 +7,11 @@ - name: Add replication user '{{ ldap_replication_user }}' failed_when: ldapadd.rc not in [0,68] - ansible.builtin.command: ldapadd -x -D cn=admin,{{ ldap_organization }} -w {{ ldap_password }} -f /tmp/replicator.ldif + ansible.builtin.command: ldapadd -H ldapi:/// -D cn=admin,{{ ldap_organization }} -w {{ ldap_password }} -f /tmp/replicator.ldif register: ldapadd - name: Set password for IDE user '{{ ldap_replication_user }}' - ansible.builtin.command: ldappasswd -x -D cn=admin,{{ ldap_organization }} -w {{ ldap_password }} \ + ansible.builtin.command: ldappasswd -H ldapi:/// -D cn=admin,{{ ldap_organization }} -w {{ ldap_password }} \ -s {{ ldap_replication_password }} {{ ldap_replication_user }} - name: Copy replicator-acl-limits.ldif to ldap provider diff --git a/collections/ansible_collections/authorization/ldap/roles/replication/tasks/main.yml b/collections/ansible_collections/authorization/ldap/roles/replication/tasks/main.yml index 1ee6d32..47ad32d 100644 --- a/collections/ansible_collections/authorization/ldap/roles/replication/tasks/main.yml +++ b/collections/ansible_collections/authorization/ldap/roles/replication/tasks/main.yml @@ -1,8 +1,8 @@ --- - name: Configure ldap provider ansible.builtin.include_tasks: configure-ldap-provider.yml - when: "'security1' in group_names" + when: "'ldap1' in group_names" - name: Configure ldap consumer ansible.builtin.include_tasks: configure-ldap-consumer.yml - when: "'security2' in group_names" + when: "'ldap2' in group_names" diff --git a/collections/ansible_collections/authorization/ldap/roles/replication/templates/consumer_simple_sync.ldif b/collections/ansible_collections/authorization/ldap/roles/replication/templates/consumer_simple_sync.ldif index ce6c4dd..b365a0c 100644 --- a/collections/ansible_collections/authorization/ldap/roles/replication/templates/consumer_simple_sync.ldif +++ b/collections/ansible_collections/authorization/ldap/roles/replication/templates/consumer_simple_sync.ldif @@ -10,10 +10,9 @@ olcDbIndex: entryUUID eq - add: olcSyncrepl olcSyncrepl: rid=0 - provider=ldap://security1.{{ domain }} + provider=ldap://{{ groups.ldap1[0] + '.' + domain if molecule_deployment is defined and molecule_deployment else groups.ldap1[0] }} bindmethod=simple binddn="{{ ldap_replication_user }}" credentials={{ ldap_replication_password }} searchbase="{{ ldap_organization }}" schemachecking=on type=refreshAndPersist retry="60 +" - starttls=critical tls_reqcert=demand diff --git a/collections/ansible_collections/authorization/ldap/roles/server/templates/ssl.ldif b/collections/ansible_collections/authorization/ldap/roles/server/templates/ssl.ldif index ed1290d..1299fc8 100644 --- a/collections/ansible_collections/authorization/ldap/roles/server/templates/ssl.ldif +++ b/collections/ansible_collections/authorization/ldap/roles/server/templates/ssl.ldif @@ -1,15 +1,8 @@ dn: cn=config changetype: modify -{% if self_signed_certificates is defined and self_signed_certificates == "false" %} add: olcTLSCACertificateFile olcTLSCACertificateFile: /etc/ssl/private/chain.pem - -{% endif %} -{% if self_signed_certificates is defined and self_signed_certificates == "true" %} -add: olcTLSCACertificateFile -olcTLSCACertificateFile: /etc/ssl/private/cert.pem -- -{% endif %} add: olcTLSCertificateFile olcTLSCertificateFile: /etc/ssl/private/cert.pem - diff --git a/collections/ansible_collections/bigdata/zookeeper/extensions/molecule/default/molecule.yml b/collections/ansible_collections/bigdata/zookeeper/extensions/molecule/default/molecule.yml index 28660bd..331e0c5 100644 --- a/collections/ansible_collections/bigdata/zookeeper/extensions/molecule/default/molecule.yml +++ b/collections/ansible_collections/bigdata/zookeeper/extensions/molecule/default/molecule.yml @@ -37,6 +37,18 @@ provisioner: tls_user: "{{ zookeeper_user }}" tls_group: "{{ zookeeper_group }}" self_signed_certificates: true + certs_source: + - /tmp/selfsigned_certs/{{ ansible_fqdn }}/cert.pem + - /tmp/selfsigned_certs/{{ ansible_fqdn }}/chain.pem + - /tmp/selfsigned_certs/{{ ansible_fqdn }}/key.pem + certs_dest: + - cert.pem + - chain.pem + - key.pem + certs_mode: + - "0660" + - "0660" + - "0400" domain: example.com keystore_file: /etc/ssl/private/{{ ansible_fqdn }}.jks keystore_password: changeit diff --git a/collections/ansible_collections/bigdata/zookeeper/extensions/molecule/ha_setup/converge.yml b/collections/ansible_collections/bigdata/zookeeper/extensions/molecule/ha_setup/converge.yml new file mode 100644 index 0000000..9a40084 --- /dev/null +++ b/collections/ansible_collections/bigdata/zookeeper/extensions/molecule/ha_setup/converge.yml @@ -0,0 +1,8 @@ +--- +- name: Setup Zookeeper + hosts: zookeeper + gather_facts: true + roles: + - role: ide.environment.user_and_groups + - role: tls.certs.setup + - role: bigdata.zookeeper.setup diff --git a/collections/ansible_collections/bigdata/zookeeper/extensions/molecule/ha_setup/create.yml b/collections/ansible_collections/bigdata/zookeeper/extensions/molecule/ha_setup/create.yml new file mode 100644 index 0000000..5a7e71f --- /dev/null +++ b/collections/ansible_collections/bigdata/zookeeper/extensions/molecule/ha_setup/create.yml @@ -0,0 +1,88 @@ +--- +- name: Create + hosts: localhost + gather_facts: false + vars: + molecule_inventory: + all: + hosts: {} + tasks: + - name: Create docker network + community.docker.docker_network: + name: example.com + ipam_config: + - subnet: 172.23.27.0/24 + gateway: 172.23.27.2 + iprange: 172.23.27.0/26 + + - name: Create container + become: true + community.docker.docker_container: + name: "{{ item.name }}" + image: "{{ item.image }}" + state: started + command: "{{ item.command }}" + log_driver: json-file + publish_all_ports: true + cgroupns_mode: "{{ item.cgroupns_mode }}" + privileged: "{{ item.privileged }}" + volumes: "{{ item.volumes }}" + networks: + - name: example.com + ipv4_address: "{{ item.ipv4_address }}" + hostname: "{{ item.name }}.example.com" + tls_hostname: "{{ item.name }}.example.com" + capabilities: + - NET_ADMIN + register: result + loop: "{{ molecule_yml.platforms }}" + + - name: Fail if container is not running + when: > + item.container.State.ExitCode != 0 or + not item.container.State.Running + ansible.builtin.include_tasks: + file: tasks/create-fail.yml + loop: "{{ result.results }}" + loop_control: + label: "{{ item.container.Name }}" + + - name: Add container to molecule_inventory + vars: + inventory_partial_yaml: | + all: + children: + zookeeper: + hosts: + "{{ item.name }}": + ansible_connection: community.docker.docker + ansible.builtin.set_fact: + molecule_inventory: > + {{ molecule_inventory | combine(inventory_partial_yaml | from_yaml, recursive=true) }} + loop: "{{ molecule_yml.platforms }}" + loop_control: + label: "{{ item.name }}" + + - name: Dump molecule_inventory + ansible.builtin.copy: + content: | + {{ molecule_inventory | to_yaml }} + dest: "{{ molecule_ephemeral_directory }}/inventory/molecule_inventory.yml" + mode: "0600" + + - name: Force inventory refresh + ansible.builtin.meta: refresh_inventory + +# we want to avoid errors like "Failed to create temporary directory" +- name: Validate that inventory was refreshed + hosts: all + gather_facts: false + tasks: + - name: Check uname + ansible.builtin.raw: uname -a + register: result + changed_when: false + + - name: Display uname info + ansible.builtin.debug: + msg: "{{ result.stdout }}" diff --git a/collections/ansible_collections/bigdata/zookeeper/extensions/molecule/ha_setup/destroy.yml b/collections/ansible_collections/bigdata/zookeeper/extensions/molecule/ha_setup/destroy.yml new file mode 100644 index 0000000..e6af2ce --- /dev/null +++ b/collections/ansible_collections/bigdata/zookeeper/extensions/molecule/ha_setup/destroy.yml @@ -0,0 +1,22 @@ +--- +- name: Destroy molecule containers + hosts: all + gather_facts: false + tasks: + - name: Stop and remove container + delegate_to: localhost + become: true + community.docker.docker_container: + name: "{{ inventory_hostname }}" + state: absent + auto_remove: true + +- name: Remove dynamic molecule inventory + hosts: localhost + gather_facts: false + tasks: + - name: Remove dynamic inventory file + become: true + ansible.builtin.file: + path: "{{ molecule_ephemeral_directory }}/inventory/molecule_inventory.yml" + state: absent diff --git a/collections/ansible_collections/bigdata/zookeeper/extensions/molecule/ha_setup/molecule.yml b/collections/ansible_collections/bigdata/zookeeper/extensions/molecule/ha_setup/molecule.yml new file mode 100644 index 0000000..31c4ad2 --- /dev/null +++ b/collections/ansible_collections/bigdata/zookeeper/extensions/molecule/ha_setup/molecule.yml @@ -0,0 +1,83 @@ +--- +dependency: + name: galaxy + options: + requirements-file: requirements.yml +platforms: + - name: instance-1 + image: docker.io/geerlingguy/docker-ubuntu2404-ansible + command: /usr/lib/systemd/systemd + pre_build_image: true + cgroupns_mode: host + ipv4_address: 172.23.27.3 + privileged: true + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + - /var/lib/containerd + - name: instance-2 + image: docker.io/geerlingguy/docker-ubuntu2404-ansible + command: /usr/lib/systemd/systemd + pre_build_image: true + cgroupns_mode: host + ipv4_address: 172.23.27.4 + privileged: true + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + - /var/lib/containerd + - name: instance-3 + image: docker.io/geerlingguy/docker-ubuntu2404-ansible + command: /usr/lib/systemd/systemd + pre_build_image: true + cgroupns_mode: host + ipv4_address: 172.23.27.5 + privileged: true + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + - /var/lib/containerd +provisioner: + name: ansible + log: false + options: + vvv: false + playbooks: + converge: converge.yml + inventory: + group_vars: + all: + molecule_deployment: true + zookeeper_user: zookeeper + zookeeper_uid: "5001" + zookeeper_group: zookeeper + zookeeper_gid: "4001" + service_user: "{{ zookeeper_user }}" + service_name: Apache Zookeeper + service_uid: "{{ zookeeper_uid }}" + service_group: "{{ zookeeper_group }}" + service_gid: "{{ zookeeper_gid }}" + tls_user: "{{ zookeeper_user }}" + tls_group: "{{ zookeeper_group }}" + self_signed_certificates: true + certs_source: + - /tmp/selfsigned_certs/{{ ansible_fqdn }}/cert.pem + - /tmp/selfsigned_certs/{{ ansible_fqdn }}/chain.pem + - /tmp/selfsigned_certs/{{ ansible_fqdn }}/key.pem + certs_dest: + - cert.pem + - chain.pem + - key.pem + certs_mode: + - "0660" + - "0660" + - "0400" + domain: example.com + keystore_file: /etc/ssl/private/{{ ansible_fqdn }}.jks + keystore_password: changeit + truststore_file: /etc/ssl/certs/truststore.jks + truststore_password: changeit + host_vars: + instance-1: + zookeeper_id: 0 + instance-2: + zookeeper_id: 1 + instance-3: + zookeeper_id: 2 diff --git a/collections/ansible_collections/bigdata/zookeeper/extensions/molecule/ha_setup/requirements.yml b/collections/ansible_collections/bigdata/zookeeper/extensions/molecule/ha_setup/requirements.yml new file mode 100644 index 0000000..660f775 --- /dev/null +++ b/collections/ansible_collections/bigdata/zookeeper/extensions/molecule/ha_setup/requirements.yml @@ -0,0 +1,3 @@ +--- +collections: + - name: community.docker diff --git a/collections/ansible_collections/bigdata/zookeeper/extensions/molecule/ha_setup/tasks/create-fail.yml b/collections/ansible_collections/bigdata/zookeeper/extensions/molecule/ha_setup/tasks/create-fail.yml new file mode 100644 index 0000000..4227c46 --- /dev/null +++ b/collections/ansible_collections/bigdata/zookeeper/extensions/molecule/ha_setup/tasks/create-fail.yml @@ -0,0 +1,14 @@ +--- +- name: Retrieve container log + ansible.builtin.command: + cmd: >- + {% raw %} + docker logs + {% endraw %} + {{ item.stdout_lines[0] }} + changed_when: false + register: logfile_cmd + +- name: Display container log + ansible.builtin.fail: + msg: "{{ logfile_cmd.stderr }}" diff --git a/collections/ansible_collections/bigdata/zookeeper/roles/install/templates/zoo.cfg b/collections/ansible_collections/bigdata/zookeeper/roles/install/templates/zoo.cfg index 5bcd7c7..bdc6bee 100644 --- a/collections/ansible_collections/bigdata/zookeeper/roles/install/templates/zoo.cfg +++ b/collections/ansible_collections/bigdata/zookeeper/roles/install/templates/zoo.cfg @@ -17,3 +17,5 @@ ssl.quorum.hostnameVerification=true ssl.quorum.clientAuth=need sslQuorumReloadCertFiles=true ssl.clientAuth=need +ssl.protocol=TLSv1.3 +ssl.enabledProtocols=TLSv1.3,TLSv1.2 diff --git a/collections/ansible_collections/hadoop/client/roles/setup/defaults/main.yml b/collections/ansible_collections/hadoop/client/roles/setup/defaults/main.yml index 6b0a053..5748adb 100644 --- a/collections/ansible_collections/hadoop/client/roles/setup/defaults/main.yml +++ b/collections/ansible_collections/hadoop/client/roles/setup/defaults/main.yml @@ -5,9 +5,9 @@ hadoop_client_group: root hdfs_data_dir: /var/hadoop/hdfs setup_hadoop_conf_dir: /opt/apache-hadoop setup_hadoop_log_dir: /var/log/hadoop -setup_core_site_name: "{{ 'core-site-ha.xml' if (ide_ha_setup is defined and ide_ha_setup) else 'core-site.xml' }}" -setup_hdfs_site_name: "{{ 'hdfs-site-ha.xml' if (ide_ha_setup is defined and ide_ha_setup) else 'hdfs-site.xml' }}" -setup_yarn_site_name: "{{ 'yarn-site-ha.xml' if (ide_ha_setup is defined and ide_ha_setup) else 'yarn-site.xml' }}" +setup_core_site_name: "{{ 'core-site-ha.xml' if (groups.namenodes | length == 3) else 'core-site.xml' }}" +setup_hdfs_site_name: "{{ 'hdfs-site-ha.xml' if (groups.namenodes | length == 3) else 'hdfs-site.xml' }}" +setup_yarn_site_name: "{{ 'yarn-site-ha.xml' if (groups.namenodes | length == 3) else 'yarn-site.xml' }}" keystore_file: /etc/ssl/private/{{ inventory_hostname }}.jks keystore_password: changeit diff --git a/collections/ansible_collections/hadoop/common/roles/setup/defaults/main.yml b/collections/ansible_collections/hadoop/common/roles/setup/defaults/main.yml index 23b3ad6..86e510d 100644 --- a/collections/ansible_collections/hadoop/common/roles/setup/defaults/main.yml +++ b/collections/ansible_collections/hadoop/common/roles/setup/defaults/main.yml @@ -1,5 +1,5 @@ --- -setup_hadoop_version: 3.3.4 +setup_hadoop_version: 3.4.0 setup_hdfs_user: "{{ hdfs_user | default('hdfs') }}" setup_hdfs_uid: "{{ hdfs_uid | default('5003') }}" setup_hdfs_group: "{{ hdfs_group | default('hadoop') }}" @@ -11,7 +11,7 @@ keytab_user_hdfs: hdfs keytab_user_yarn: yarn keytab_user_jupyter: jupyterhub keytab_user_http: HTTP -setup_core_site_name: "{{ 'core-site-ha.xml' if (ide_ha_setup is defined and ide_ha_setup) else 'core-site.xml' }}" +setup_core_site_name: "{{ 'core-site-ha.xml' if (groups.namenodes | length == 3) else 'core-site.xml' }}" keystore_file: /etc/ssl/private/{{ ansible_fqdn if molecule_deployment is defined and molecule_deployment else inventory_hostname }}.jks keystore_password: changeit diff --git a/collections/ansible_collections/hadoop/common/roles/setup/tasks/main.yml b/collections/ansible_collections/hadoop/common/roles/setup/tasks/main.yml index b88a693..8d6c728 100644 --- a/collections/ansible_collections/hadoop/common/roles/setup/tasks/main.yml +++ b/collections/ansible_collections/hadoop/common/roles/setup/tasks/main.yml @@ -16,19 +16,6 @@ state: present shell: /bin/bash -- name: Create journalnode user - become: true - when: - - ide_ha_setup is defined - - ide_ha_setup - ansible.builtin.user: - name: "{{ journalnode_user }}" - comment: User for Apache Hadoop - uid: "{{ journalnode_uid }}" - group: "{{ setup_hdfs_gid }}" - state: present - shell: /bin/bash - - name: Create HDFS keytab when: - kerberos_external is defined diff --git a/collections/ansible_collections/hadoop/common/roles/setup/templates/core-site-ha.xml b/collections/ansible_collections/hadoop/common/roles/setup/templates/core-site-ha.xml index 78e8cd9..dbfff36 100644 --- a/collections/ansible_collections/hadoop/common/roles/setup/templates/core-site-ha.xml +++ b/collections/ansible_collections/hadoop/common/roles/setup/templates/core-site-ha.xml @@ -29,37 +29,40 @@ hadoop.rpc.protection privacy - - hadoop.security.token.service.use_ip - false - hadoop.security.authorization true ha.zookeeper.quorum - {% for host in groups.zookeeper %}{{ host }}:2281{% if not loop.last %},{% endif %}{% endfor %} + {% for host in groups.zookeeper %}{{ host + '.' + domain if molecule_deployment is defined and molecule_deployment else host }}:2281{% if not loop.last %},{% endif %}{% endfor %} ha.zookeeper.acl world:anyone:cdrwa - - hadoop.http.authentication.signature.secret.file - {{ keytab_folder }}/hadoop_secret - hadoop.http.authentication.type - simple kerberos + + hadoop.http.authentication.signature.secret.file + {{ keytab_folder }}/hadoop_secret + hadoop.http.authentication.kerberos.keytab {{ keytab_folder }}/{{ keytab_user_http }}.keytab hadoop.http.authentication.kerberos.principal + {{ keytab_user_http }}/{{ ansible_fqdn }}@{{ realm }} + + + hadoop.web.authentication.kerberos.keytab + {{ keytab_folder }}/{{ keytab_user_http }}.keytab + + + hadoop.web.authentication.kerberos.principal {{ keytab_user_http }}/_HOST@{{ realm }} @@ -75,41 +78,40 @@ kerberos - dfs.web.authentication.kerberos.keytab - {{ keytab_folder }}/{{ keytab_user_http }}.keytab - - - dfs.web.authentication.kerberos.principal - {{ keytab_user_http }}/_HOST@{{ realm }} + hadoop.security.resolver.impl + org.apache.hadoop.net.DNSDomainNameResolver hadoop.http.authentication.cookie.domain {{ domain }} + + hadoop.http.authentication.cookie.persistent + true + hadoop.security.auth_to_local RULE:[1:$1@$0](.*@{{ realm }})s/@.*///L RULE:[2:$1@$0]({{ keytab_user_hdfs }}@{{ realm }})s/.*/{{ keytab_user_hdfs }}/ - RULE:[2:$1@$0]({{ keytab_user_journalnode }}@{{ realm }})s/.*/{{ keytab_user_journalnode }}/ RULE:[2:$1@$0]({{ keytab_user_yarn }}@{{ realm }})s/.*/{{ keytab_user_yarn }}/ - RULE:[2:$1@$0]({{ keytab_user_jupyter }}@{{ realm }})s/.*/{{ keytab_user_jupyter }}/ RULE:[2:$1@$0]({{ keytab_user_http }}@{{ realm }})s/.*/{{ keytab_user_http }}/ + RULE:[2:$1@$0]({{ keytab_user_jupyter }}@{{ realm }})s/.*/{{ keytab_user_jupyter }}/ DEFAULT - hadoop.proxyuser.HTTP.hosts + hadoop.proxyuser.yarn.hosts * - hadoop.proxyuser.HTTP.groups + hadoop.proxyuser.yarn.groups * - hadoop.proxyuser.yarn.hosts + hadoop.proxyuser.HTTP.hosts * - hadoop.proxyuser.yarn.groups + hadoop.proxyuser.HTTP.groups * @@ -200,6 +202,14 @@ hadoop.security.group.mapping.ldap.ssl.keystore.password {{ keystore_password }} + + hadoop.security.group.mapping.ldap.ssl.truststore + {{ truststore_file }} + + + hadoop.security.group.mapping.ldap.ssl.truststore.password + {{ truststore_password }} + hadoop.security.group.mapping.provider.ad4users.ldap.bind.user {{ ldap_bind_user }} diff --git a/collections/ansible_collections/hadoop/common/roles/setup/templates/core-site.xml b/collections/ansible_collections/hadoop/common/roles/setup/templates/core-site.xml index 01968c9..596162d 100644 --- a/collections/ansible_collections/hadoop/common/roles/setup/templates/core-site.xml +++ b/collections/ansible_collections/hadoop/common/roles/setup/templates/core-site.xml @@ -37,10 +37,12 @@ hadoop.security.authorization true +{% if groups.zookeeper is defined %} ha.zookeeper.quorum {% for host in groups.zookeeper %}{{ host }}.{{ domain }}:2281{% if not loop.last %},{% endif %}{% endfor %} +{% endif %} ha.zookeeper.acl world:anyone:cdrwa @@ -51,7 +53,6 @@ hadoop.http.authentication.type - simple kerberos diff --git a/collections/ansible_collections/hadoop/common/roles/setup/templates/ssl-server.xml b/collections/ansible_collections/hadoop/common/roles/setup/templates/ssl-server.xml index 246a118..920aaef 100644 --- a/collections/ansible_collections/hadoop/common/roles/setup/templates/ssl-server.xml +++ b/collections/ansible_collections/hadoop/common/roles/setup/templates/ssl-server.xml @@ -45,11 +45,4 @@ ssl.server.keystore.type jks - - ssl.server.exclude.cipher.list - TLS_ECDHE_RSA_WITH_RC4_128_SHA,SSL_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA, - SSL_RSA_WITH_DES_CBC_SHA,SSL_DHE_RSA_WITH_DES_CBC_SHA, - SSL_RSA_EXPORT_WITH_RC4_40_MD5,SSL_RSA_EXPORT_WITH_DES40_CBC_SHA, - SSL_RSA_WITH_RC4_128_MD5 - diff --git a/collections/ansible_collections/hadoop/common/roles/setup/vars/main.yml b/collections/ansible_collections/hadoop/common/roles/setup/vars/main.yml index dde60cc..92d9e17 100644 --- a/collections/ansible_collections/hadoop/common/roles/setup/vars/main.yml +++ b/collections/ansible_collections/hadoop/common/roles/setup/vars/main.yml @@ -1,14 +1,5 @@ --- -kerberos_keytabs_non_ha: +kerberos_keytabs: - principal: "{{ hdfs_user }}" - principal: HTTP keytab_user: "{{ hdfs_user }}" - -kerberos_keytabs_ha: - - principal: "{{ hdfs_user }}" - - principal: "{{ journalnode_user }}" - keytab_user: "{{ journalnode_user }}" - - principal: HTTP - keytab_user: "{{ hdfs_user }}" - -kerberos_keytabs: "{% if ide_ha_setup is defined and ide_ha_setup %}{{ kerberos_keytabs_ha }}{% else %}{{ kerberos_keytabs_non_ha }}{% endif %}" diff --git a/collections/ansible_collections/hadoop/hdfs/extensions/molecule/default/create.yml b/collections/ansible_collections/hadoop/hdfs/extensions/molecule/default/create.yml index 057ebd4..b159625 100644 --- a/collections/ansible_collections/hadoop/hdfs/extensions/molecule/default/create.yml +++ b/collections/ansible_collections/hadoop/hdfs/extensions/molecule/default/create.yml @@ -81,9 +81,6 @@ namenodes: children: namenode1: - zookeeper: - children: - namenode1: datanodes: children: datanode1: diff --git a/collections/ansible_collections/hadoop/hdfs/extensions/molecule/ha_setup/converge.yml b/collections/ansible_collections/hadoop/hdfs/extensions/molecule/ha_setup/converge.yml new file mode 100644 index 0000000..4ca9c12 --- /dev/null +++ b/collections/ansible_collections/hadoop/hdfs/extensions/molecule/ha_setup/converge.yml @@ -0,0 +1,57 @@ +--- +- name: Setup environment + hosts: all + gather_facts: true + roles: + - role: ide.environment.setup + +- name: Setup TLS preparation + hosts: all + gather_facts: true + roles: + - role: tls.certs.setup + +- name: Setup LDAP server + hosts: ldap + gather_facts: true + roles: + - role: authorization.ldap.setup + +- name: Setup Kerberos server + hosts: kerberos + gather_facts: true + roles: + - role: authentication.kerberos.setup + +- name: Setup Zookeeper + hosts: zookeeper + become: true + roles: + - role: bigdata.zookeeper.setup + +- name: Setup HDFS installation + hosts: namenodes:datanodes + gather_facts: true + roles: + - role: hadoop.common.setup + - role: hadoop.hdfs.common + +- name: Setup HDFS namenode + hosts: namenodes + gather_facts: true + roles: + - role: hadoop.hdfs.journalnode + - role: hadoop.hdfs.namenode + +- name: Setup HDFS datanodes + hosts: datanodes + gather_facts: true + roles: + - role: hadoop.hdfs.datanode + +- name: Check HDFS is running and healthy + hosts: namenode1 + become: true + become_user: hdfs + roles: + - role: hadoop.hdfs.check diff --git a/collections/ansible_collections/hadoop/hdfs/extensions/molecule/ha_setup/create.yml b/collections/ansible_collections/hadoop/hdfs/extensions/molecule/ha_setup/create.yml new file mode 100644 index 0000000..8884e33 --- /dev/null +++ b/collections/ansible_collections/hadoop/hdfs/extensions/molecule/ha_setup/create.yml @@ -0,0 +1,137 @@ +--- +- name: Create + hosts: localhost + gather_facts: false + vars: + molecule_inventory: + all: + hosts: {} + tasks: + - name: Create docker network + community.docker.docker_network: + name: example.com + ipam_config: + - subnet: 172.23.27.0/24 + gateway: 172.23.27.2 + iprange: 172.23.27.0/26 + + - name: Create container + become: true + community.docker.docker_container: + name: "{{ item.name }}" + image: "{{ item.image }}" + state: started + command: "{{ item.command }}" + log_driver: json-file + publish_all_ports: true + cgroupns_mode: "{{ item.cgroupns_mode }}" + privileged: "{{ item.privileged }}" + volumes: "{{ item.volumes }}" + networks: + - name: example.com + ipv4_address: "{{ item.ipv4_address }}" + hostname: "{{ item.name }}.example.com" + tls_hostname: "{{ item.name }}.example.com" + capabilities: + - NET_ADMIN + register: result + loop: "{{ molecule_yml.platforms }}" + + - name: Fail if container is not running + when: > + item.container.State.ExitCode != 0 or + not item.container.State.Running + ansible.builtin.include_tasks: + file: tasks/create-fail.yml + loop: "{{ result.results }}" + loop_control: + label: "{{ item.container.Name }}" + + - name: Add container to molecule_inventory + vars: + inventory_partial_yaml: | + all: + vars: + ansible_connection: community.docker.docker + children: + kerberos1: + hosts: + instance-1: + ldap1: + hosts: + instance-1: + namenode1: + hosts: + instance-2: + namenode2: + hosts: + instance-3: + namenode3: + hosts: + instance-4: + zookeeper1: + hosts: + instance-2: + zookeeper2: + hosts: + instance-3: + zookeeper3: + hosts: + instance-4: + datanode1: + hosts: + instance-5: + datanode2: + hosts: + instance-6: + datanode3: + hosts: + instance-7: + ldap: + children: + ldap1: + kerberos: + children: + kerberos1: + namenodes: + children: + namenode1: + namenode2: + namenode3: + zookeeper: + children: + zookeeper1: + zookeeper2: + zookeeper3: + datanodes: + children: + datanode1: + datanode2: + datanode3: + ansible.builtin.set_fact: + molecule_inventory: > + {{ molecule_inventory | combine(inventory_partial_yaml | from_yaml, recursive=true) }} + + - name: Dump molecule_inventory + ansible.builtin.copy: + content: | + {{ molecule_inventory | to_yaml }} + dest: "{{ molecule_ephemeral_directory }}/inventory/molecule_inventory.yml" + mode: "0600" + + - name: Force inventory refresh + ansible.builtin.meta: refresh_inventory + +# we want to avoid errors like "Failed to create temporary directory" +- name: Validate that inventory was refreshed + hosts: all + gather_facts: false + tasks: + - name: Check uname + ansible.builtin.raw: uname -a + register: result + changed_when: false + + - name: Display uname info + ansible.builtin.debug: + msg: "{{ result.stdout }}" diff --git a/collections/ansible_collections/hadoop/hdfs/extensions/molecule/ha_setup/destroy.yml b/collections/ansible_collections/hadoop/hdfs/extensions/molecule/ha_setup/destroy.yml new file mode 100644 index 0000000..e6af2ce --- /dev/null +++ b/collections/ansible_collections/hadoop/hdfs/extensions/molecule/ha_setup/destroy.yml @@ -0,0 +1,22 @@ +--- +- name: Destroy molecule containers + hosts: all + gather_facts: false + tasks: + - name: Stop and remove container + delegate_to: localhost + become: true + community.docker.docker_container: + name: "{{ inventory_hostname }}" + state: absent + auto_remove: true + +- name: Remove dynamic molecule inventory + hosts: localhost + gather_facts: false + tasks: + - name: Remove dynamic inventory file + become: true + ansible.builtin.file: + path: "{{ molecule_ephemeral_directory }}/inventory/molecule_inventory.yml" + state: absent diff --git a/collections/ansible_collections/hadoop/hdfs/extensions/molecule/ha_setup/molecule.yml b/collections/ansible_collections/hadoop/hdfs/extensions/molecule/ha_setup/molecule.yml new file mode 100644 index 0000000..a2c5071 --- /dev/null +++ b/collections/ansible_collections/hadoop/hdfs/extensions/molecule/ha_setup/molecule.yml @@ -0,0 +1,158 @@ +--- +dependency: + name: galaxy + options: + requirements-file: requirements.yml +platforms: + - name: instance-1 + image: docker.io/geerlingguy/docker-ubuntu2404-ansible + command: /usr/lib/systemd/systemd + pre_build_image: true + cgroupns_mode: host + ipv4_address: 172.23.27.3 + privileged: true + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + - /var/lib/containerd + - name: instance-2 + image: docker.io/geerlingguy/docker-ubuntu2404-ansible + command: /usr/lib/systemd/systemd + pre_build_image: true + cgroupns_mode: host + ipv4_address: 172.23.27.4 + privileged: true + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + - /var/lib/containerd + - name: instance-3 + image: docker.io/geerlingguy/docker-ubuntu2404-ansible + command: /usr/lib/systemd/systemd + pre_build_image: true + cgroupns_mode: host + ipv4_address: 172.23.27.5 + privileged: true + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + - /var/lib/containerd + - name: instance-4 + image: docker.io/geerlingguy/docker-ubuntu2404-ansible + command: /usr/lib/systemd/systemd + pre_build_image: true + cgroupns_mode: host + ipv4_address: 172.23.27.6 + privileged: true + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + - /var/lib/containerd + - name: instance-5 + image: docker.io/geerlingguy/docker-ubuntu2404-ansible + command: /usr/lib/systemd/systemd + pre_build_image: true + cgroupns_mode: host + ipv4_address: 172.23.27.7 + privileged: true + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + - /var/lib/containerd + - name: instance-6 + image: docker.io/geerlingguy/docker-ubuntu2404-ansible + command: /usr/lib/systemd/systemd + pre_build_image: true + cgroupns_mode: host + ipv4_address: 172.23.27.8 + privileged: true + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + - /var/lib/containerd + - name: instance-7 + image: docker.io/geerlingguy/docker-ubuntu2404-ansible + command: /usr/lib/systemd/systemd + pre_build_image: true + cgroupns_mode: host + ipv4_address: 172.23.27.9 + privileged: true + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + - /var/lib/containerd +provisioner: + name: ansible + log: false + options: + vvv: false + playbooks: + converge: converge.yml + inventory: + group_vars: + all: + molecule_deployment: true + tls_external: false + kerberos_external: false + hdfs_user: hdfs + hdfs_group: hadoop + hadoop_nameservice: communitylab + keytab_user_journalnode: journalnode + ide_services_group: hadoop + self_signed_certificates: true + domain: example.com + keytab_folder: /etc/keytabs + realm: COMMUNITY.LAB + ldap_organization: dc=example,dc=com + ldap_server_address: "{% for host in groups.ldap | shuffle %}ldaps://{{ host }}.example.com:636{% if not loop.last %},{% endif %}{% endfor %}" + ldap_user_search_base: ou=people,{{ ldap_organization }} + ldap_group_search_base: ou=groups,{{ ldap_organization }} + ldap_bind_user: cn=admin,{{ ldap_organization }} + ldap_password: changeit + ldap_bind_dn_template: uid={username},{{ ldap_user_search_base }} + ide_users_group: ide_users + ide_users_gid: "5001" + ldap_users: + - name: teppler + uid_number: 6000 + gid_number: "{{ ide_users_gid }}" + password: albstadt + - name: anolle + uid_number: 6001 + gid_number: "{{ ide_users_gid }}" + password: sigmaringen + - name: gschulz + uid_number: 6002 + gid_number: "{{ ide_users_gid }}" + password: datascience + keystore_file: /etc/ssl/private/{{ ansible_fqdn }}.jks + keystore_password: changeit + truststore_file: /etc/ssl/certs/truststore.jks + truststore_password: changeit + ldap: + ldap_user: openldap + ldap_uid: "5001" + ldap_group: openldap + ldap_gid: "4001" + service_user: "{{ ldap_user }}" + service_name: OpenLdap + service_uid: "{{ ldap_uid }}" + service_group: "{{ ldap_group }}" + service_gid: "{{ ldap_gid }}" + tls_user: "{{ ldap_user }}" + tls_group: "{{ ldap_group }}" + namenodes: + service_user: hdfs + service_name: Apache Hadoop + service_uid: "5003" + service_group: hadoop + service_gid: "4001" + tls_user: hdfs + tls_group: hadoop + datanodes: + service_user: hdfs + service_name: Apache Hadoop + service_uid: "5003" + service_group: hadoop + service_gid: "4001" + tls_user: hdfs + tls_group: hadoop + zookeeper1: + zookeeper_id: 1 + zookeeper2: + zookeeper_id: 2 + zookeeper3: + zookeeper_id: 3 diff --git a/collections/ansible_collections/hadoop/hdfs/extensions/molecule/ha_setup/requirements.yml b/collections/ansible_collections/hadoop/hdfs/extensions/molecule/ha_setup/requirements.yml new file mode 100644 index 0000000..660f775 --- /dev/null +++ b/collections/ansible_collections/hadoop/hdfs/extensions/molecule/ha_setup/requirements.yml @@ -0,0 +1,3 @@ +--- +collections: + - name: community.docker diff --git a/collections/ansible_collections/hadoop/hdfs/extensions/molecule/ha_setup/tasks/create-fail.yml b/collections/ansible_collections/hadoop/hdfs/extensions/molecule/ha_setup/tasks/create-fail.yml new file mode 100644 index 0000000..4227c46 --- /dev/null +++ b/collections/ansible_collections/hadoop/hdfs/extensions/molecule/ha_setup/tasks/create-fail.yml @@ -0,0 +1,14 @@ +--- +- name: Retrieve container log + ansible.builtin.command: + cmd: >- + {% raw %} + docker logs + {% endraw %} + {{ item.stdout_lines[0] }} + changed_when: false + register: logfile_cmd + +- name: Display container log + ansible.builtin.fail: + msg: "{{ logfile_cmd.stderr }}" diff --git a/collections/ansible_collections/hadoop/hdfs/roles/check/tasks/main.yml b/collections/ansible_collections/hadoop/hdfs/roles/check/tasks/main.yml index 728d2a0..afaf87b 100644 --- a/collections/ansible_collections/hadoop/hdfs/roles/check/tasks/main.yml +++ b/collections/ansible_collections/hadoop/hdfs/roles/check/tasks/main.yml @@ -7,9 +7,7 @@ ansible.builtin.command: kinit -k hdfs/{{ keytab_principal_hostname }}@{{ realm }} -t {{ keytab_folder }}/hdfs.keytab - name: Block for IDE HA setup - when: - - ide_ha_setup is defined - - ide_ha_setup + when: groups.namenodes | length == 3 block: - name: Check if HDFS namenodes are running ansible.builtin.command: /opt/apache-hadoop/hadoop/bin/hdfs haadmin -getAllServiceState diff --git a/collections/ansible_collections/hadoop/hdfs/roles/common/defaults/main.yml b/collections/ansible_collections/hadoop/hdfs/roles/common/defaults/main.yml index 632ee74..bdc17fe 100644 --- a/collections/ansible_collections/hadoop/hdfs/roles/common/defaults/main.yml +++ b/collections/ansible_collections/hadoop/hdfs/roles/common/defaults/main.yml @@ -3,7 +3,7 @@ common_hdfs_user: "{{ hdfs_user | default('hdfs') }}" common_hdfs_group: "{{ hdfs_group | default('hadoop') }}" keytab_folder: /etc/keytabs hdfs_data_dir: /var/hadoop/hdfs -common_hdfs_site_name: "{{ 'hdfs-site-ha.xml' if (ide_ha_setup is defined and ide_ha_setup) else 'hdfs-site.xml' }}" +common_hdfs_site_name: "{{ 'hdfs-site-ha.xml' if (groups.namenodes | length == 3) else 'hdfs-site.xml' }}" keytab_user_hdfs: hdfs keytab_user_http: HTTP diff --git a/collections/ansible_collections/hadoop/hdfs/roles/common/templates/hdfs-site-ha.xml b/collections/ansible_collections/hadoop/hdfs/roles/common/templates/hdfs-site-ha.xml index 3ee4dc7..ab00f94 100644 --- a/collections/ansible_collections/hadoop/hdfs/roles/common/templates/hdfs-site-ha.xml +++ b/collections/ansible_collections/hadoop/hdfs/roles/common/templates/hdfs-site-ha.xml @@ -27,23 +27,23 @@ {% for host in groups.namenodes %} dfs.namenode.rpc-address.{{ hadoop_nameservice }}.{{ hostvars[host].inventory_hostname_short }} - {{ host }}:8020 + {{ host + '.' + domain if molecule_deployment is defined and molecule_deployment else host }}:8020 dfs.namenode.http-address.{{ hadoop_nameservice }}.{{ hostvars[host].inventory_hostname_short }} - {{ host }}:9870 + {{ host + '.' + domain if molecule_deployment is defined and molecule_deployment else host }}:9870 dfs.namenode.https-address.{{ hadoop_nameservice }}.{{ hostvars[host].inventory_hostname_short }} - {{ host }}:9871 + {{ host + '.' + domain if molecule_deployment is defined and molecule_deployment else host }}:9871 dfs.namenode.servicerpc-address.{{ hadoop_nameservice }}.{{ hostvars[host].inventory_hostname_short }} - {{ host }}:8022 + {{ host + '.' + domain if molecule_deployment is defined and molecule_deployment else host }}:8022 {% endfor %} dfs.namenode.name.dir - file://{{ hdfs_data_dir }}/namenode + file://{{ hdfs_data_dir }}/namenode/storage1,file://{{ hdfs_data_dir }}/namenode/storage2,file://{{ hdfs_data_dir }}/namenode/storage3 dfs.client.failover.proxy.provider.{{ hadoop_nameservice }} @@ -55,15 +55,31 @@ dfs.namenode.shared.edits.dir - qjournal://{% for host in groups.namenodes %}{{ host }}:8485{% if not loop.last %};{% endif %}{% endfor %}/{{ hadoop_nameservice }} + qjournal://{% for host in groups.namenodes %}{{ host + '.' + domain if molecule_deployment is defined and molecule_deployment else host }}:8485{% if not loop.last %};{% endif %}{% endfor %}/{{ hadoop_nameservice }} dfs.journalnode.edits.dir {{ hdfs_data_dir }}/journalnode + + dfs.journalnode.enable.sync + true + + + dfs.journalnode.rpc-address + {{ ansible_fqdn }}:8485 + + + dfs.journalnode.http-address + {{ ansible_fqdn }}:8480 + + + dfs.journalnode.https-address + {{ ansible_fqdn }}:8481 + dfs.replication - 3 + 3 dfs.permissions.enabled @@ -109,17 +125,29 @@ dfs.namenode.kerberos.principal {{ keytab_user_hdfs }}/_HOST@{{ realm }} + + dfs.secondary.namenode.keytab.file + {{ keytab_folder }}/{{ keytab_user_hdfs }}.keytab + + + dfs.secondary.namenode.kerberos.principal + {{ keytab_user_hdfs }}/_HOST@{{ realm }} + + + dfs.namenode.kerberos.internal.spnego.keytab.file + {{ keytab_folder }}/{{ keytab_user_http }}.keytab + dfs.namenode.kerberos.internal.spnego.principal {{ keytab_user_http }}/_HOST@{{ realm }} dfs.journalnode.keytab.file - {{ keytab_folder }}/{{ keytab_user_journalnode }}.keytab + {{ keytab_folder }}/{{ keytab_user_hdfs }}.keytab dfs.journalnode.kerberos.principal - {{ keytab_user_journalnode }}/_HOST@{{ realm }} + {{ keytab_user_hdfs }}/_HOST@{{ realm }} dfs.journalnode.kerberos.internal.spnego.principal @@ -155,7 +183,7 @@ dfs.datanode.kerberos.principal - hdfs/_HOST@{{ realm }} + hdfs/_HOST@{{ realm }} dfs.namenode.datanode.registration.ip-hostname-check @@ -169,12 +197,4 @@ dfs.client.https.need-auth false - - dfs.web.authentication.kerberos.keytab - {{ keytab_folder }}/{{ keytab_user_http }}.keytab - - - dfs.web.authentication.kerberos.principal - {{ keytab_user_http }}/_HOST@{{ realm }} - diff --git a/collections/ansible_collections/hadoop/hdfs/roles/journalnode/tasks/main.yml b/collections/ansible_collections/hadoop/hdfs/roles/journalnode/tasks/main.yml index b63c7ef..052782e 100644 --- a/collections/ansible_collections/hadoop/hdfs/roles/journalnode/tasks/main.yml +++ b/collections/ansible_collections/hadoop/hdfs/roles/journalnode/tasks/main.yml @@ -3,8 +3,8 @@ ansible.builtin.file: path: "{{ journalnode_hdfs_data_dir }}" state: directory - owner: "{{ journalnode_user }}" - group: "{{ journalnode_group }}" + owner: "{{ hdfs_user }}" + group: "{{ hdfs_group }}" mode: "0755" - name: Create system service diff --git a/collections/ansible_collections/hadoop/hdfs/roles/journalnode/templates/hdfs-journalnode.service b/collections/ansible_collections/hadoop/hdfs/roles/journalnode/templates/hdfs-journalnode.service index 3fd2504..f3f5bda 100644 --- a/collections/ansible_collections/hadoop/hdfs/roles/journalnode/templates/hdfs-journalnode.service +++ b/collections/ansible_collections/hadoop/hdfs/roles/journalnode/templates/hdfs-journalnode.service @@ -7,12 +7,11 @@ Wants=zookeeper.service Type=forking ExecStart=/opt/apache-hadoop/hadoop/bin/hdfs --daemon start journalnode ExecStop=/opt/apache-hadoop/hadoop/bin/hdfs --daemon stop journalnode -User={{ journalnode_user }} -Group={{ journalnode_group }} +User={{ hdfs_user }} +Group={{ hdfs_group }} TimeoutSec=30 RestartSec=30 Restart=always -LimitNOFILE=65536 [Install] WantedBy=multi-user.target diff --git a/collections/ansible_collections/hadoop/hdfs/roles/namenode/tasks/main.yml b/collections/ansible_collections/hadoop/hdfs/roles/namenode/tasks/main.yml index b797230..223a4f2 100644 --- a/collections/ansible_collections/hadoop/hdfs/roles/namenode/tasks/main.yml +++ b/collections/ansible_collections/hadoop/hdfs/roles/namenode/tasks/main.yml @@ -2,15 +2,19 @@ - name: Configure HDFS ZKFC ansible.builtin.include_tasks: configure-zkfc.yml when: - - ide_ha_setup is defined - - ide_ha_setup + - groups.namenodes | length == 3 - name: Configure HDFS namenode ansible.builtin.include_tasks: configure-namenode.yml +- name: Set fact for initializing process + ansible.builtin.set_fact: + namenode_hdfs_data_dir_current: "{{ namenode_hdfs_data_dir + '/namenode/storage1/current' if groups.namenodes | length == 3 else namenode_hdfs_data_dir + '/namenode/current' + }}" + - name: Check HDFS is already initialized ansible.builtin.stat: - path: "{{ namenode_hdfs_data_dir }}/namenode/current" + path: "{{ namenode_hdfs_data_dir_current }}" register: namenode_initialized_dir - name: Enable HDFS ZKFC @@ -18,8 +22,7 @@ name: hdfs-zkfc.service enabled: true when: - - ide_ha_setup is defined - - ide_ha_setup + - groups.namenodes | length == 3 - name: Enable HDFS namenode ansible.builtin.systemd_service: @@ -36,8 +39,7 @@ name: hdfs-zkfc.service state: stopped when: - - ide_ha_setup is defined - - ide_ha_setup + - groups.namenodes | length == 3 - name: Format HDFS ZKFC become: true @@ -46,8 +48,7 @@ when: - "'namenode1' in group_names" - not namenode_initialized_dir.stat.exists - - ide_ha_setup is defined - - ide_ha_setup + - groups.namenodes | length == 3 - name: Format HDFS namenode become: true @@ -62,8 +63,7 @@ name: hdfs-zkfc.service state: started when: - - ide_ha_setup is defined - - ide_ha_setup + - groups.namenodes | length == 3 - name: Start HDFS namenode ansible.builtin.systemd_service: @@ -71,21 +71,19 @@ state: started when: "'namenode1' in group_names" -- name: Bootstrap HDFS namenode +- name: Bootstrap HDFS secondary and tertiary namenode become: true become_user: "{{ hdfs_user }}" ansible.builtin.command: /opt/apache-hadoop/hadoop/bin/hdfs namenode -bootstrapStandby -nonInteractive -force when: - - "'master1' not in group_names" + - "'namenode1' not in group_names" - not namenode_initialized_dir.stat.exists - - ide_ha_setup is defined - - ide_ha_setup + - groups.namenodes | length == 3 -- name: Start HDFS namenode +- name: Start HDFS secondary and tertiary namenode ansible.builtin.systemd_service: name: hdfs-namenode.service state: started when: - "'namenode1' not in group_names" - - ide_ha_setup is defined - - ide_ha_setup + - groups.namenodes | length == 3 diff --git a/collections/ansible_collections/hadoop/hdfs/roles/namenode/templates/hdfs-namenode.service b/collections/ansible_collections/hadoop/hdfs/roles/namenode/templates/hdfs-namenode.service index 8c0e187..e27883f 100644 --- a/collections/ansible_collections/hadoop/hdfs/roles/namenode/templates/hdfs-namenode.service +++ b/collections/ansible_collections/hadoop/hdfs/roles/namenode/templates/hdfs-namenode.service @@ -1,6 +1,6 @@ [Unit] Description=Hadoop Namenode Service -{% if ide_ha_setup is defined and ide_ha_setup %} +{% if groups.namenodes | length == 3 %} After=hdfs-zkfc.service Wants=hdfs-zkfc.service After=hdfs-journalnode.service diff --git a/collections/ansible_collections/hadoop/yarn/extensions/molecule/ha_setup/converge.yml b/collections/ansible_collections/hadoop/yarn/extensions/molecule/ha_setup/converge.yml new file mode 100644 index 0000000..5fdf510 --- /dev/null +++ b/collections/ansible_collections/hadoop/yarn/extensions/molecule/ha_setup/converge.yml @@ -0,0 +1,84 @@ +--- +- name: Setup environment + hosts: all + gather_facts: true + roles: + - role: ide.environment.setup + +- name: Setup TLS preparation + hosts: all + gather_facts: true + roles: + - role: tls.certs.setup + +- name: Setup LDAP server + hosts: ldap + gather_facts: true + roles: + - role: authorization.ldap.setup + +- name: Setup LDAP client + hosts: all,!ldap + gather_facts: true + roles: + - role: authorization.ldap.client + +- name: Setup Kerberos server + hosts: kerberos + gather_facts: true + roles: + - role: authentication.kerberos.setup + +- name: Setup Zookeeper + hosts: zookeeper + become: true + roles: + - role: bigdata.zookeeper.setup + +- name: Setup HDFS installation + hosts: namenodes:datanodes + gather_facts: true + roles: + - role: hadoop.common.setup + - role: hadoop.hdfs.common + +- name: Setup HDFS namenode + hosts: namenodes + gather_facts: true + roles: + - role: hadoop.hdfs.journalnode + - role: hadoop.hdfs.namenode + +- name: Setup HDFS datanodes + hosts: datanodes + gather_facts: true + roles: + - role: hadoop.hdfs.datanode + +- name: Check HDFS is running and healthy + hosts: namenode1 + become: true + become_user: hdfs + roles: + - role: hadoop.hdfs.check + +- name: Setup YARN resourcemanager + hosts: resourcemanagers + gather_facts: true + roles: + - role: hadoop.yarn.common + - role: hadoop.yarn.resourcemanager + +- name: Setup YARN nodemanagers + hosts: nodemanagers + gather_facts: true + roles: + - role: hadoop.yarn.common + - role: hadoop.yarn.nodemanager + +- name: Check YARN is running and healthy + hosts: resourcemanager1 + become: true + become_user: yarn + roles: + - role: hadoop.yarn.check diff --git a/collections/ansible_collections/hadoop/yarn/extensions/molecule/ha_setup/create.yml b/collections/ansible_collections/hadoop/yarn/extensions/molecule/ha_setup/create.yml new file mode 100644 index 0000000..5db2d90 --- /dev/null +++ b/collections/ansible_collections/hadoop/yarn/extensions/molecule/ha_setup/create.yml @@ -0,0 +1,172 @@ +--- +- name: Create + hosts: localhost + gather_facts: false + vars: + molecule_inventory: + all: + hosts: {} + tasks: + - name: Create docker network + community.docker.docker_network: + name: example.com + ipam_config: + - subnet: 172.23.27.0/24 + gateway: 172.23.27.2 + iprange: 172.23.27.0/26 + + - name: Create container + become: true + community.docker.docker_container: + name: "{{ item.name }}" + image: "{{ item.image }}" + state: started + command: "{{ item.command }}" + log_driver: json-file + publish_all_ports: true + cgroupns_mode: "{{ item.cgroupns_mode }}" + privileged: "{{ item.privileged }}" + volumes: "{{ item.volumes }}" + networks: + - name: example.com + ipv4_address: "{{ item.ipv4_address }}" + hostname: "{{ item.name }}.example.com" + tls_hostname: "{{ item.name }}.example.com" + capabilities: + - NET_ADMIN + register: result + loop: "{{ molecule_yml.platforms }}" + + - name: Fail if container is not running + when: > + item.container.State.ExitCode != 0 or + not item.container.State.Running + ansible.builtin.include_tasks: + file: tasks/create-fail.yml + loop: "{{ result.results }}" + loop_control: + label: "{{ item.container.Name }}" + + - name: Add container to molecule_inventory + vars: + inventory_partial_yaml: | + all: + vars: + ansible_connection: community.docker.docker + children: + kerberos1: + hosts: + instance-1: + ldap1: + hosts: + instance-1: + namenode1: + hosts: + instance-2: + namenode2: + hosts: + instance-3: + namenode3: + hosts: + instance-4: + resourcemanager1: + hosts: + instance-2: + resourcemanager2: + hosts: + instance-3: + resourcemanager3: + hosts: + instance-4: + zookeeper1: + hosts: + instance-2: + zookeeper2: + hosts: + instance-3: + zookeeper3: + hosts: + instance-4: + datanode1: + hosts: + instance-5: + datanode2: + hosts: + instance-6: + datanode3: + hosts: + instance-7: + nodemanager1: + hosts: + instance-5: + nodemanager2: + hosts: + instance-6: + nodemanager3: + hosts: + instance-7: + ldap: + children: + ldap1: + kerberos: + children: + kerberos1: + namenodes: + children: + namenode1: + namenode2: + namenode3: + resourcemanagers: + children: + resourcemanager1: + resourcemanager2: + resourcemanager3: + zookeeper: + children: + zookeeper1: + zookeeper2: + zookeeper3: + datanodes: + children: + datanode1: + datanode2: + datanode3: + nodemanagers: + children: + nodemanager1: + nodemanager2: + nodemanager3: + ansible.builtin.set_fact: + molecule_inventory: > + {{ molecule_inventory | combine(inventory_partial_yaml | from_yaml, recursive=true) }} + + - name: Dump molecule_inventory + ansible.builtin.copy: + content: | + {{ molecule_inventory | to_yaml }} + dest: "{{ molecule_ephemeral_directory }}/inventory/molecule_inventory.yml" + mode: "0600" + + - name: Force inventory refresh + ansible.builtin.meta: refresh_inventory + + - name: Fail if hdfs group is missing + ansible.builtin.assert: + that: "'all' in groups" + fail_msg: | + all group was not found inside inventory groups: {{ groups }} + run_once: true # noqa: run-once[task] + +# we want to avoid errors like "Failed to create temporary directory" +- name: Validate that inventory was refreshed + hosts: all + gather_facts: false + tasks: + - name: Check uname + ansible.builtin.raw: uname -a + register: result + changed_when: false + + - name: Display uname info + ansible.builtin.debug: + msg: "{{ result.stdout }}" diff --git a/collections/ansible_collections/hadoop/yarn/extensions/molecule/ha_setup/destroy.yml b/collections/ansible_collections/hadoop/yarn/extensions/molecule/ha_setup/destroy.yml new file mode 100644 index 0000000..e6af2ce --- /dev/null +++ b/collections/ansible_collections/hadoop/yarn/extensions/molecule/ha_setup/destroy.yml @@ -0,0 +1,22 @@ +--- +- name: Destroy molecule containers + hosts: all + gather_facts: false + tasks: + - name: Stop and remove container + delegate_to: localhost + become: true + community.docker.docker_container: + name: "{{ inventory_hostname }}" + state: absent + auto_remove: true + +- name: Remove dynamic molecule inventory + hosts: localhost + gather_facts: false + tasks: + - name: Remove dynamic inventory file + become: true + ansible.builtin.file: + path: "{{ molecule_ephemeral_directory }}/inventory/molecule_inventory.yml" + state: absent diff --git a/collections/ansible_collections/hadoop/yarn/extensions/molecule/ha_setup/molecule.yml b/collections/ansible_collections/hadoop/yarn/extensions/molecule/ha_setup/molecule.yml new file mode 100644 index 0000000..8e15ab7 --- /dev/null +++ b/collections/ansible_collections/hadoop/yarn/extensions/molecule/ha_setup/molecule.yml @@ -0,0 +1,165 @@ +--- +dependency: + name: galaxy + options: + requirements-file: requirements.yml +platforms: + - name: instance-1 + image: docker.io/geerlingguy/docker-ubuntu2404-ansible + command: /usr/lib/systemd/systemd + pre_build_image: true + cgroupns_mode: host + ipv4_address: 172.23.27.3 + privileged: true + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + - /var/lib/containerd + - name: instance-2 + image: docker.io/geerlingguy/docker-ubuntu2404-ansible + command: /usr/lib/systemd/systemd + pre_build_image: true + cgroupns_mode: host + ipv4_address: 172.23.27.4 + privileged: true + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + - /var/lib/containerd + - name: instance-3 + image: docker.io/geerlingguy/docker-ubuntu2404-ansible + command: /usr/lib/systemd/systemd + pre_build_image: true + cgroupns_mode: host + ipv4_address: 172.23.27.5 + privileged: true + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + - /var/lib/containerd + - name: instance-4 + image: docker.io/geerlingguy/docker-ubuntu2404-ansible + command: /usr/lib/systemd/systemd + pre_build_image: true + cgroupns_mode: host + ipv4_address: 172.23.27.6 + privileged: true + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + - /var/lib/containerd + - name: instance-5 + image: docker.io/geerlingguy/docker-ubuntu2404-ansible + command: /usr/lib/systemd/systemd + pre_build_image: true + cgroupns_mode: host + ipv4_address: 172.23.27.7 + privileged: true + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + - /var/lib/containerd + - name: instance-6 + image: docker.io/geerlingguy/docker-ubuntu2404-ansible + command: /usr/lib/systemd/systemd + pre_build_image: true + cgroupns_mode: host + ipv4_address: 172.23.27.8 + privileged: true + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + - /var/lib/containerd + - name: instance-7 + image: docker.io/geerlingguy/docker-ubuntu2404-ansible + command: /usr/lib/systemd/systemd + pre_build_image: true + cgroupns_mode: host + ipv4_address: 172.23.27.9 + privileged: true + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + - /var/lib/containerd +provisioner: + name: ansible + log: false + options: + vvv: false + playbooks: + converge: converge.yml + inventory: + group_vars: + all: + molecule_deployment: true + tls_external: false + kerberos_external: false + hdfs_user: hdfs + hdfs_group: hadoop + hadoop_nameservice: communitylab + keytab_user_journalnode: journalnode + yarn_user: yarn + yarn_group: hadoop + self_signed_certificates: true + domain: example.com + keytab_folder: /etc/keytabs + realm: COMMUNITY.LAB + ide_services_group: hadoop + keytab_user_yarn: yarn + keytab_user_http: HTTP + ldap_organization: dc=example,dc=com + ldap_server_address: "{% for host in groups.ldap | shuffle %}ldaps://{{ host }}.example.com:636{% if not loop.last %},{% endif %}{% endfor %}" + ldap_user_search_base: ou=people,{{ ldap_organization }} + ldap_group_search_base: ou=groups,{{ ldap_organization }} + ldap_bind_user: cn=admin,{{ ldap_organization }} + ldap_password: changeit + ldap_bind_dn_template: uid={username},{{ ldap_user_search_base }} + ide_users_group: ide_users + ide_users_gid: "5001" + ldap_users: + - name: teppler + uid_number: 6000 + gid_number: "{{ ide_users_gid }}" + password: albstadt + - name: anolle + uid_number: 6001 + gid_number: "{{ ide_users_gid }}" + password: sigmaringen + - name: gschulz + uid_number: 6002 + gid_number: "{{ ide_users_gid }}" + password: datascience + keystore_file: /etc/ssl/private/{{ ansible_fqdn }}.jks + keystore_password: changeit + truststore_file: /etc/ssl/certs/truststore.jks + truststore_password: changeit + ldap: + ldap_user: openldap + ldap_uid: "5001" + ldap_group: openldap + ldap_gid: "4001" + service_user: "{{ ldap_user }}" + service_name: OpenLdap + service_uid: "{{ ldap_uid }}" + service_group: "{{ ldap_group }}" + service_gid: "{{ ldap_gid }}" + tls_user: "{{ ldap_user }}" + tls_group: "{{ ldap_group }}" + resourcemanagers: + service_user: yarn + service_name: Apache Hadoop + service_uid: "5004" + service_group: hadoop + service_gid: "4001" + tls_user: yarn + tls_group: hadoop + journalnode_user: journalnode + journalnode_group: hadoop + journalnode_uid: "5006" + nodemanagers: + service_user: yarn + service_name: Apache Hadoop + service_uid: "5004" + service_group: hadoop + service_gid: "4001" + tls_user: yarn + tls_group: hadoop + zookeeper1: + zookeeper_id: 1 + zookeeper2: + zookeeper_id: 2 + zookeeper3: + zookeeper_id: 3 diff --git a/collections/ansible_collections/hadoop/yarn/extensions/molecule/ha_setup/requirements.yml b/collections/ansible_collections/hadoop/yarn/extensions/molecule/ha_setup/requirements.yml new file mode 100644 index 0000000..660f775 --- /dev/null +++ b/collections/ansible_collections/hadoop/yarn/extensions/molecule/ha_setup/requirements.yml @@ -0,0 +1,3 @@ +--- +collections: + - name: community.docker diff --git a/collections/ansible_collections/hadoop/yarn/extensions/molecule/ha_setup/tasks/create-fail.yml b/collections/ansible_collections/hadoop/yarn/extensions/molecule/ha_setup/tasks/create-fail.yml new file mode 100644 index 0000000..4227c46 --- /dev/null +++ b/collections/ansible_collections/hadoop/yarn/extensions/molecule/ha_setup/tasks/create-fail.yml @@ -0,0 +1,14 @@ +--- +- name: Retrieve container log + ansible.builtin.command: + cmd: >- + {% raw %} + docker logs + {% endraw %} + {{ item.stdout_lines[0] }} + changed_when: false + register: logfile_cmd + +- name: Display container log + ansible.builtin.fail: + msg: "{{ logfile_cmd.stderr }}" diff --git a/collections/ansible_collections/hadoop/yarn/roles/check/tasks/main.yml b/collections/ansible_collections/hadoop/yarn/roles/check/tasks/main.yml index 5e68c29..ffca378 100644 --- a/collections/ansible_collections/hadoop/yarn/roles/check/tasks/main.yml +++ b/collections/ansible_collections/hadoop/yarn/roles/check/tasks/main.yml @@ -8,8 +8,7 @@ - name: Block for IDE HA setup when: - - ide_ha_setup is defined - - ide_ha_setup + - groups.resourcemanagers | length == 3 block: - name: Check if YARN resourcemanagers are running ansible.builtin.command: /opt/apache-hadoop/hadoop/bin/yarn rmadmin -getAllServiceState diff --git a/collections/ansible_collections/hadoop/yarn/roles/common/defaults/main.yml b/collections/ansible_collections/hadoop/yarn/roles/common/defaults/main.yml index 9c489b9..ef4baf3 100644 --- a/collections/ansible_collections/hadoop/yarn/roles/common/defaults/main.yml +++ b/collections/ansible_collections/hadoop/yarn/roles/common/defaults/main.yml @@ -3,4 +3,4 @@ common_yarn_user: "{{ yarn_user | default('yarn') }}" common_yarn_group: "{{ yarn_group | default('yarn') }}" hdfs_data_dir: /var/hadoop/hdfs yarn_data_dir: /var/hadoop/yarn -common_yarn_site_name: "{{ 'yarn-site-ha.xml' if (ide_ha_setup is defined and ide_ha_setup) else 'yarn-site.xml' }}" +common_yarn_site_name: "{{ 'yarn-site-ha.xml' if (groups.resourcemanagers | length == 3) else 'yarn-site.xml' }}" diff --git a/collections/ansible_collections/hadoop/yarn/roles/common/templates/yarn-site-ha.xml b/collections/ansible_collections/hadoop/yarn/roles/common/templates/yarn-site-ha.xml index 2c902be..c700612 100644 --- a/collections/ansible_collections/hadoop/yarn/roles/common/templates/yarn-site-ha.xml +++ b/collections/ansible_collections/hadoop/yarn/roles/common/templates/yarn-site-ha.xml @@ -25,31 +25,31 @@ yarn.resourcemanager.ha.rm-ids - {% for host in groups.resourcemanagers %}{{ host }}{% if not loop.last %},{% endif %}{% endfor %} + {% for host in groups.resourcemanagers %}{{ hostvars[host].inventory_hostname_short }}{% if not loop.last %},{% endif %}{% endfor %} {% for host in groups.resourcemanagers %} yarn.resourcemanager.hostname.{{ hostvars[host].inventory_hostname_short }} - {{ host }} + {{ host + '.' + domain if molecule_deployment is defined and molecule_deployment else host }} yarn.resourcemanager.webapp.address.{{ hostvars[host].inventory_hostname_short }} - {{ host }}:8088 + {{ host + '.' + domain if molecule_deployment is defined and molecule_deployment else host }}:8088 yarn.resourcemanager.webapp.https.address.{{ hostvars[host].inventory_hostname_short }} - {{ host }}:8090 + {{ host + '.' + domain if molecule_deployment is defined and molecule_deployment else host }}:8090 yarn.resourcemanager.resource-tracker.address.{{ hostvars[host].inventory_hostname_short }} - {{ host }}:8031 + {{ host + '.' + domain if molecule_deployment is defined and molecule_deployment else host }}:8031 yarn.resourcemanager.address.{{ hostvars[host].inventory_hostname_short }} - {{ host }}:8032 + {{ host + '.' + domain if molecule_deployment is defined and molecule_deployment else host }}:8032 yarn.resourcemanager.scheduler.address.{{ hostvars[host].inventory_hostname_short }} - {{ host }}:8030 + {{ host + '.' + domain if molecule_deployment is defined and molecule_deployment else host }}:8030 {% endfor %} diff --git a/collections/ansible_collections/hadoop/yarn/roles/resourcemanager/tasks/main.yml b/collections/ansible_collections/hadoop/yarn/roles/resourcemanager/tasks/main.yml index ea16d5a..b8d0752 100644 --- a/collections/ansible_collections/hadoop/yarn/roles/resourcemanager/tasks/main.yml +++ b/collections/ansible_collections/hadoop/yarn/roles/resourcemanager/tasks/main.yml @@ -20,10 +20,10 @@ ansible.builtin.systemd: name: yarn-resourcemanager.service state: started - when: "'master1' in group_names" + when: "'resourcemanager1' in group_names" - name: Start passive yarn-resourcemanagers ansible.builtin.systemd: name: yarn-resourcemanager.service state: started - when: "'master1' not in group_names" + when: "'resourcemanager1' not in group_names" diff --git a/collections/ansible_collections/jupyter/hub/extensions/molecule/ha_setup/converge.yml b/collections/ansible_collections/jupyter/hub/extensions/molecule/ha_setup/converge.yml new file mode 100644 index 0000000..4fcfe35 --- /dev/null +++ b/collections/ansible_collections/jupyter/hub/extensions/molecule/ha_setup/converge.yml @@ -0,0 +1,115 @@ +--- +- name: Setup environment + hosts: all + gather_facts: true + roles: + - role: ide.environment.setup + +- name: Setup TLS preparation + hosts: all + gather_facts: true + roles: + - role: tls.certs.setup + +- name: Setup LDAP server + hosts: ldap + gather_facts: true + roles: + - role: authorization.ldap.setup + +- name: Setup Kerberos server + hosts: kerberos + gather_facts: true + roles: + - role: authentication.kerberos.setup + +- name: Setup Zookeeper + hosts: zookeeper + become: true + roles: + - role: bigdata.zookeeper.setup + +- name: Setup HDFS installation + hosts: namenodes:datanodes + gather_facts: true + roles: + - role: hadoop.common.setup + - role: hadoop.hdfs.common + +- name: Setup HDFS namenode + hosts: namenodes + gather_facts: true + roles: + - role: hadoop.hdfs.journalnode + - role: hadoop.hdfs.namenode + +- name: Setup HDFS datanodes + hosts: datanodes + gather_facts: true + roles: + - role: hadoop.hdfs.datanode + +- name: Check HDFS is running and healthy + hosts: namenode1 + become: true + become_user: hdfs + roles: + - role: hadoop.hdfs.check + +- name: Setup YARN resourcemanager + hosts: resourcemanagers + gather_facts: true + roles: + - role: hadoop.yarn.common + - role: hadoop.yarn.resourcemanager + +- name: Setup YARN nodemanagers + hosts: nodemanagers + gather_facts: true + roles: + - role: hadoop.yarn.common + - role: hadoop.yarn.nodemanager + +- name: Check YARN is running and healthy + hosts: resourcemanager1 + become: true + become_user: yarn + roles: + - role: hadoop.yarn.check + +- name: Install Spark common libs + hosts: spark + become: true + gather_facts: true + roles: + - role: bigdata.spark.common + +- name: Install HAProxy and Keepalived for PostgreSQL access + hosts: loadbalancers + become: true + gather_facts: true + roles: + - role: loadbalancing.haproxy.setup + - role: loadbalancing.keepalived.setup + +- name: Install PostgreSQL as Backend for JupyterHub + hosts: postgres + become: true + gather_facts: true + roles: + - role: rdbms.postgres.setup + +- name: Install JupyterHub + hosts: hubs + become: true + gather_facts: true + roles: + - role: jupyter.hub.setup + +- name: Install JupyterLab + hosts: jupyterlab + become: true + gather_facts: true + serial: 1 + roles: + - role: jupyter.lab.setup diff --git a/collections/ansible_collections/jupyter/hub/extensions/molecule/ha_setup/create.yml b/collections/ansible_collections/jupyter/hub/extensions/molecule/ha_setup/create.yml new file mode 100644 index 0000000..b5dbb04 --- /dev/null +++ b/collections/ansible_collections/jupyter/hub/extensions/molecule/ha_setup/create.yml @@ -0,0 +1,217 @@ +--- +- name: Create + hosts: localhost + gather_facts: false + vars: + molecule_inventory: + all: + hosts: {} + tasks: + - name: Create docker network + community.docker.docker_network: + name: example.com + ipam_config: + - subnet: 172.23.27.0/24 + gateway: 172.23.27.2 + iprange: 172.23.27.0/26 + + - name: Create container + become: true + community.docker.docker_container: + name: "{{ item.name }}" + image: "{{ item.image }}" + state: started + command: "{{ item.command }}" + log_driver: json-file + publish_all_ports: true + cgroupns_mode: "{{ item.cgroupns_mode }}" + privileged: "{{ item.privileged }}" + volumes: "{{ item.volumes }}" + networks: + - name: example.com + ipv4_address: "{{ item.ipv4_address }}" + hostname: "{{ item.name }}.example.com" + tls_hostname: "{{ item.name }}.example.com" + capabilities: + - NET_ADMIN + register: result + loop: "{{ molecule_yml.platforms }}" + + - name: Fail if container is not running + when: > + item.container.State.ExitCode != 0 or + not item.container.State.Running + ansible.builtin.include_tasks: + file: tasks/create-fail.yml + loop: "{{ result.results }}" + loop_control: + label: "{{ item.container.Name }}" + + - name: Add container to molecule_inventory + vars: + inventory_partial_yaml: | + all: + vars: + ansible_connection: community.docker.docker + children: + hub1: + hosts: + instance-1: + hub2: + hosts: + instance-2: + loadbalancer1: + hosts: + instance-1: + loadbalancer2: + hosts: + instance-2: + kerberos1: + hosts: + instance-3: + kerberos2: + hosts: + instance-4: + ldap1: + hosts: + instance-3: + ldap2: + hosts: + instance-4: + namenode1: + hosts: + instance-5: + namenode2: + hosts: + instance-6: + namenode3: + hosts: + instance-7: + resourcemanager1: + hosts: + instance-5: + resourcemanager2: + hosts: + instance-6: + resourcemanager3: + hosts: + instance-7: + zookeeper1: + hosts: + instance-5: + zookeeper2: + hosts: + instance-6: + zookeeper3: + hosts: + instance-7: + datanode1: + hosts: + instance-8: + datanode2: + hosts: + instance-9: + datanode3: + hosts: + instance-10: + nodemanager1: + hosts: + instance-8: + nodemanager2: + hosts: + instance-9: + nodemanager3: + hosts: + instance-10: + postgres1: + hosts: + instance-8: + postgres2: + hosts: + instance-9: + postgres3: + hosts: + instance-10: + hubs: + children: + hub1: + hub2: + loadbalancers: + children: + loadbalancer1: + loadbalancer2: + ldap: + children: + ldap1: + ldap2: + kerberos: + children: + kerberos1: + kerberos2: + namenodes: + children: + namenode1: + namenode2: + namenode3: + resourcemanagers: + children: + resourcemanager1: + resourcemanager2: + resourcemanager3: + zookeeper: + children: + zookeeper1: + zookeeper2: + zookeeper3: + datanodes: + children: + datanode1: + datanode2: + datanode3: + nodemanagers: + children: + nodemanager1: + nodemanager2: + nodemanager3: + spark: + children: + nodemanager1: + nodemanager2: + nodemanager3: + jupyterlab: + children: + nodemanager1: + nodemanager2: + nodemanager3: + postgres: + children: + postgres1: + postgres2: + postgres3: + ansible.builtin.set_fact: + molecule_inventory: > + {{ molecule_inventory | combine(inventory_partial_yaml | from_yaml, recursive=true) }} + + - name: Dump molecule_inventory + ansible.builtin.copy: + content: | + {{ molecule_inventory | to_yaml }} + dest: "{{ molecule_ephemeral_directory }}/inventory/molecule_inventory.yml" + mode: "0600" + + - name: Force inventory refresh + ansible.builtin.meta: refresh_inventory + +# we want to avoid errors like "Failed to create temporary directory" +- name: Validate that inventory was refreshed + hosts: all + gather_facts: false + tasks: + - name: Check uname + ansible.builtin.raw: uname -a + register: result + changed_when: false + + - name: Display uname info + ansible.builtin.debug: + msg: "{{ result.stdout }}" diff --git a/collections/ansible_collections/jupyter/hub/extensions/molecule/ha_setup/destroy.yml b/collections/ansible_collections/jupyter/hub/extensions/molecule/ha_setup/destroy.yml new file mode 100644 index 0000000..e6af2ce --- /dev/null +++ b/collections/ansible_collections/jupyter/hub/extensions/molecule/ha_setup/destroy.yml @@ -0,0 +1,22 @@ +--- +- name: Destroy molecule containers + hosts: all + gather_facts: false + tasks: + - name: Stop and remove container + delegate_to: localhost + become: true + community.docker.docker_container: + name: "{{ inventory_hostname }}" + state: absent + auto_remove: true + +- name: Remove dynamic molecule inventory + hosts: localhost + gather_facts: false + tasks: + - name: Remove dynamic inventory file + become: true + ansible.builtin.file: + path: "{{ molecule_ephemeral_directory }}/inventory/molecule_inventory.yml" + state: absent diff --git a/collections/ansible_collections/jupyter/hub/extensions/molecule/ha_setup/molecule.yml b/collections/ansible_collections/jupyter/hub/extensions/molecule/ha_setup/molecule.yml new file mode 100644 index 0000000..5829860 --- /dev/null +++ b/collections/ansible_collections/jupyter/hub/extensions/molecule/ha_setup/molecule.yml @@ -0,0 +1,266 @@ +--- +dependency: + name: galaxy + options: + requirements-file: requirements.yml +platforms: + - name: instance-1 + image: docker.io/geerlingguy/docker-ubuntu2404-ansible + command: /usr/lib/systemd/systemd + pre_build_image: true + cgroupns_mode: host + ipv4_address: 172.23.27.3 + privileged: true + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + - /var/lib/containerd + - name: instance-2 + image: docker.io/geerlingguy/docker-ubuntu2404-ansible + command: /usr/lib/systemd/systemd + pre_build_image: true + cgroupns_mode: host + ipv4_address: 172.23.27.4 + privileged: true + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + - /var/lib/containerd + - name: instance-3 + image: docker.io/geerlingguy/docker-ubuntu2404-ansible + command: /usr/lib/systemd/systemd + pre_build_image: true + cgroupns_mode: host + ipv4_address: 172.23.27.5 + privileged: true + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + - /var/lib/containerd + - name: instance-4 + image: docker.io/geerlingguy/docker-ubuntu2404-ansible + command: /usr/lib/systemd/systemd + pre_build_image: true + cgroupns_mode: host + ipv4_address: 172.23.27.6 + privileged: true + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + - /var/lib/containerd + - name: instance-5 + image: docker.io/geerlingguy/docker-ubuntu2404-ansible + command: /usr/lib/systemd/systemd + pre_build_image: true + cgroupns_mode: host + ipv4_address: 172.23.27.7 + privileged: true + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + - /var/lib/containerd + - name: instance-6 + image: docker.io/geerlingguy/docker-ubuntu2404-ansible + command: /usr/lib/systemd/systemd + pre_build_image: true + cgroupns_mode: host + ipv4_address: 172.23.27.8 + privileged: true + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + - /var/lib/containerd + - name: instance-7 + image: docker.io/geerlingguy/docker-ubuntu2404-ansible + command: /usr/lib/systemd/systemd + pre_build_image: true + cgroupns_mode: host + ipv4_address: 172.23.27.9 + privileged: true + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + - /var/lib/containerd + - name: instance-8 + image: docker.io/geerlingguy/docker-ubuntu2404-ansible + command: /usr/lib/systemd/systemd + pre_build_image: true + cgroupns_mode: host + ipv4_address: 172.23.27.10 + privileged: true + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + - /var/lib/containerd + - name: instance-9 + image: docker.io/geerlingguy/docker-ubuntu2404-ansible + command: /usr/lib/systemd/systemd + pre_build_image: true + cgroupns_mode: host + ipv4_address: 172.23.27.11 + privileged: true + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + - /var/lib/containerd + - name: instance-10 + image: docker.io/geerlingguy/docker-ubuntu2404-ansible + command: /usr/lib/systemd/systemd + pre_build_image: true + cgroupns_mode: host + ipv4_address: 172.23.27.12 + privileged: true + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw + - /var/lib/containerd +provisioner: + name: ansible + log: false + options: + vvv: false + playbooks: + converge: converge.yml + inventory: + group_vars: + all: + molecule_deployment: true + tls_external: false + kerberos_external: false + jupyterhub_user: jupyterhub + jupyterhub_group: jupyterhub + hdfs_user: hdfs + hdfs_group: hadoop + yarn_user: yarn + yarn_group: hadoop + self_signed_certificates: true + domain: example.com + keystore_file: /etc/ssl/private/{{ ansible_fqdn }}.jks + keystore_password: changeit + truststore_file: /etc/ssl/certs/truststore.jks + truststore_password: changeit + keytab_folder: /etc/keytabs + ide_services_group: hadoop + ide_users_group: ide_users + ide_users_gid: "5001" + keytab_user_yarn: yarn + keytab_user_hdfs: hdfs + keytab_user_http: HTTP + keytab_user_jupyter: jupyterhub + hadoop_nameservice: communitylab + keytab_user_journalnode: journalnode + ldap_server_address: "{% for host in groups.ldap %}ldaps://{{ host }}.example.com:636{% if not loop.last %},{% endif %}{% endfor %}" + ldap_organization: dc=example,dc=com + ldap_user_search_base: ou=people,{{ ldap_organization }} + ldap_group_search_base: ou=groups,{{ ldap_organization }} + ldap_bind_user: cn=admin,{{ ldap_organization }} + ldap_password: changeit + ldap_bind_dn_template: uid={username},{{ ldap_user_search_base }} + jupyterhub_user_password: changeit + ldap_users: + - name: teppler + uid_number: 6000 + gid_number: "{{ ide_users_gid }}" + password: albstadt + - name: anolle + uid_number: 6001 + gid_number: "{{ ide_users_gid }}" + password: sigmaringen + - name: gschulz + uid_number: 6002 + gid_number: "{{ ide_users_gid }}" + password: datascience + ldap_replication_user: cn=replicator,{{ ldap_organization }} + ldap_replication_password: changeit + ldap_kdc_service_password: changeit + ldap_kadmin_service_password: changeit + realm: COMMUNITY.LAB + realm_password: changeit + jupyterhub_domain_ip_address: 172.23.27.25 + postgres_host: "{{ jupyterhub_domain_ip_address }}" + postgres_user_password: changeit + repl_user_password: changeit + postgres_schemes: + - username: postgres + database: postgres + password: "{{ postgres_user_password }}" + port: 5432 + hostname: "*" + - username: jupyterhub + database: jupyterhub + password: "{{ jupyterhub_user_password }}" + port: 5432 + hostname: "*" + scheme: jupyterhub + loadbalancer1: + keepalived_state: MASTER + keepalived_virtual_router_id: 80 + keepalived_priority: 100 + loadbalancer2: + keepalived_state: BACKUP + keepalived_virtual_router_id: 80 + keepalived_priority: 99 + hubs: + service_user: jupyterhub + service_name: JupyterHub + service_uid: "5001" + service_group: jupyterhub + service_gid: "4001" + tls_user: jupyterhub + tls_group: jupyterhub + keytab_group: jupyterhub + cert_file: /etc/ssl/private/cert.pem + key_file: /etc/ssl/private/key.pem + haproxy_pem_file: /etc/ssl/private/haproxy.pem + jupyterhub_domain_ip: true + haproxy_admin_password: changeit + ldap: + ldap_user: openldap + ldap_uid: "5001" + ldap_group: openldap + ldap_gid: "4001" + service_user: "{{ ldap_user }}" + service_name: OpenLdap + service_uid: "{{ ldap_uid }}" + service_group: "{{ ldap_group }}" + service_gid: "{{ ldap_gid }}" + tls_user: "{{ ldap_user }}" + tls_group: "{{ ldap_group }}" + resourcemanagers: + service_user: yarn + service_name: Apache Hadoop + service_uid: "5004" + service_group: hadoop + service_gid: "4001" + tls_user: yarn + tls_group: hadoop + nodemanagers: + service_user: yarn + service_name: Apache Hadoop + service_uid: "5004" + service_group: hadoop + service_gid: "4001" + tls_user: yarn + tls_group: hadoop + postgres: + certs_source: + - /tmp/selfsigned_certs/{{ ansible_fqdn }}/cert.pem + - /tmp/selfsigned_certs/{{ ansible_fqdn }}/chain.pem + - /tmp/selfsigned_certs/{{ ansible_fqdn }}/key.pem + certs_dest: + - cert.pem + - chain.pem + - key.pem + certs_mode: + - "0660" + - "0660" + - "0400" + cert_file_postgres: /etc/ssl/private/cert_postgres.pem + key_file_postgres: /etc/ssl/private/key_postgres.pem + chain_file_postgres: /etc/ssl/private/chain_postgres.pem + certs_dest_postgres: + - cert_postgres.pem + - chain_postgres.pem + - key_postgres.pem + postgres1: + patroni_id: 1 + postgres2: + patroni_id: 2 + postgres3: + patroni_id: 3 + zookeeper1: + zookeeper_id: 1 + zookeeper2: + zookeeper_id: 2 + zookeeper3: + zookeeper_id: 3 diff --git a/collections/ansible_collections/jupyter/hub/extensions/molecule/ha_setup/requirements.yml b/collections/ansible_collections/jupyter/hub/extensions/molecule/ha_setup/requirements.yml new file mode 100644 index 0000000..660f775 --- /dev/null +++ b/collections/ansible_collections/jupyter/hub/extensions/molecule/ha_setup/requirements.yml @@ -0,0 +1,3 @@ +--- +collections: + - name: community.docker diff --git a/collections/ansible_collections/jupyter/hub/extensions/molecule/ha_setup/tasks/create-fail.yml b/collections/ansible_collections/jupyter/hub/extensions/molecule/ha_setup/tasks/create-fail.yml new file mode 100644 index 0000000..4227c46 --- /dev/null +++ b/collections/ansible_collections/jupyter/hub/extensions/molecule/ha_setup/tasks/create-fail.yml @@ -0,0 +1,14 @@ +--- +- name: Retrieve container log + ansible.builtin.command: + cmd: >- + {% raw %} + docker logs + {% endraw %} + {{ item.stdout_lines[0] }} + changed_when: false + register: logfile_cmd + +- name: Display container log + ansible.builtin.fail: + msg: "{{ logfile_cmd.stderr }}" diff --git a/collections/ansible_collections/jupyter/hub/roles/install/files/jupyterhubenvironment.txt b/collections/ansible_collections/jupyter/hub/roles/install/files/jupyterhubenvironment.txt index 6f2aac1..722b378 100644 --- a/collections/ansible_collections/jupyter/hub/roles/install/files/jupyterhubenvironment.txt +++ b/collections/ansible_collections/jupyter/hub/roles/install/files/jupyterhubenvironment.txt @@ -11,16 +11,16 @@ attrs 24.2.0 pyh71513ae_0 conda-forge blinker 1.8.2 pyhd8ed1ab_0 conda-forge brotli-python 1.1.0 py310hf71b8c6_2 conda-forge bzip2 1.0.8 h4bc722e_7 conda-forge -c-ares 1.33.1 heb4867d_0 conda-forge +c-ares 1.34.2 heb4867d_0 conda-forge ca-certificates 2024.8.30 hbcca054_0 conda-forge cached-property 1.5.2 hd8ed1ab_1 conda-forge cached_property 1.5.2 pyha770c72_1 conda-forge certifi 2024.8.30 pyhd8ed1ab_0 conda-forge certipy 0.2.1 pyhd8ed1ab_0 conda-forge cffi 1.17.1 py310h8deb56e_0 conda-forge -charset-normalizer 3.3.2 pyhd8ed1ab_0 conda-forge +charset-normalizer 3.4.0 pyhd8ed1ab_0 conda-forge configurable-http-proxy 4.6.2 hbf95b10_1 conda-forge -cryptography 43.0.1 py310h6c63255_0 conda-forge +cryptography 43.0.3 py310h6c63255_0 conda-forge fqdn 1.5.1 pyhd8ed1ab_0 conda-forge greenlet 3.1.1 py310hf71b8c6_0 conda-forge grpc-cpp 1.48.0 hc2bec63_4 conda-forge @@ -36,12 +36,12 @@ isoduration 20.11.0 pyhd8ed1ab_0 conda-forge jinja2 3.1.4 pyhd8ed1ab_0 conda-forge jsonpointer 3.0.0 py310hff52083_1 conda-forge jsonschema 4.23.0 pyhd8ed1ab_0 conda-forge -jsonschema-specifications 2023.12.1 pyhd8ed1ab_0 conda-forge +jsonschema-specifications 2024.10.1 pyhd8ed1ab_0 conda-forge jsonschema-with-format-nongpl 4.23.0 hd8ed1ab_0 conda-forge jupyter_events 0.10.0 pyhd8ed1ab_0 conda-forge jupyterhub 5.1.0 pyh31011fe_0 conda-forge jupyterhub-base 5.1.0 pyh31011fe_0 conda-forge -jupyterhub-ldapauthenticator 2.0.0b2 pypi_0 pypi +jupyterhub-ldapauthenticator 2.0.1.dev0 pypi_0 pypi jupyterhub-yarnspawner 0.4.0 py310hff52083_2 conda-forge keyutils 1.6.1 h166bdaf_0 conda-forge krb5 1.21.3 h659f571_0 conda-forge @@ -52,22 +52,22 @@ libcurl 8.10.1 hbbe4b11_0 conda-forge libedit 3.1.20191231 he28a2e2_2 conda-forge libev 4.33 hd590300_2 conda-forge libffi 3.4.2 h7f98852_5 conda-forge -libgcc 14.1.0 h77fa898_1 conda-forge -libgcc-ng 14.1.0 h69a702a_1 conda-forge -libgomp 14.1.0 h77fa898_1 conda-forge +libgcc 14.2.0 h77fa898_1 conda-forge +libgcc-ng 14.2.0 h69a702a_1 conda-forge +libgomp 14.2.0 h77fa898_1 conda-forge libnghttp2 1.58.0 h47da74e_1 conda-forge libnsl 2.0.1 hd590300_0 conda-forge libprotobuf 3.20.3 h3eb15da_0 conda-forge libsqlite 3.46.1 hadc24fc_0 conda-forge libssh2 1.11.0 h0841786_0 conda-forge -libstdcxx 14.1.0 hc0a3c3a_1 conda-forge -libstdcxx-ng 14.1.0 h4852527_1 conda-forge +libstdcxx 14.2.0 hc0a3c3a_1 conda-forge +libstdcxx-ng 14.2.0 h4852527_1 conda-forge libuuid 2.38.1 h0b41bf4_0 conda-forge -libuv 1.49.0 hb9d3cd8_0 conda-forge +libuv 1.49.2 hb9d3cd8_0 conda-forge libxcrypt 4.4.36 hd590300_1 conda-forge -libzlib 1.3.1 h4ab18f5_1 conda-forge +libzlib 1.3.1 hb9d3cd8_2 conda-forge mako 1.3.5 pyhd8ed1ab_0 conda-forge -markupsafe 2.1.5 py310ha75aee5_1 conda-forge +markupsafe 3.0.2 py310h89163eb_0 conda-forge ncurses 6.5 he02047a_1 conda-forge nodejs 22.5.1 h6d9b948_0 conda-forge oauthlib 3.2.2 pyhd8ed1ab_0 conda-forge @@ -78,7 +78,7 @@ pip 24.2 pyh8b19718_1 conda-forge pkgutil-resolve-name 1.3.10 pyhd8ed1ab_1 conda-forge prometheus_client 0.21.0 pyhd8ed1ab_0 conda-forge protobuf 3.20.3 py310heca2aa9_1 conda-forge -psutil 6.0.0 py310ha75aee5_1 conda-forge +psutil 6.0.0 py310ha75aee5_2 conda-forge psycopg2-binary 2.9.9 pypi_0 pypi pyasn1 0.6.1 pypi_0 pypi pycparser 2.22 pyhd8ed1ab_0 conda-forge @@ -87,7 +87,7 @@ pydantic 2.9.2 pyhd8ed1ab_0 conda-forge pydantic-core 2.23.4 py310h505e2c1_0 conda-forge pyjwt 2.9.0 pyhd8ed1ab_1 conda-forge pysocks 1.7.1 pyha2e5f31_6 conda-forge -python 3.10.15 h4a871b0_0_cpython conda-forge +python 3.10.15 h4a871b0_2_cpython conda-forge python-dateutil 2.9.0 pyhd8ed1ab_0 conda-forge python-json-logger 2.0.7 pyhd8ed1ab_0 conda-forge python_abi 3.10 5_cp310 conda-forge @@ -102,14 +102,14 @@ rpds-py 0.20.0 py310h505e2c1_1 conda-forge setuptools 75.1.0 pyhd8ed1ab_0 conda-forge six 1.16.0 pyh6c4a22f_0 conda-forge skein 0.8.1 py310hff52083_4 conda-forge -sqlalchemy 2.0.35 py310ha75aee5_0 conda-forge +sqlalchemy 2.0.36 py310ha75aee5_0 conda-forge tk 8.6.13 noxft_h4845f30_101 conda-forge tornado 6.4.1 py310ha75aee5_1 conda-forge traitlets 5.14.3 pyhd8ed1ab_0 conda-forge -types-python-dateutil 2.9.0.20240906 pyhd8ed1ab_0 conda-forge +types-python-dateutil 2.9.0.20241003 pyhff2d567_0 conda-forge typing-extensions 4.12.2 hd8ed1ab_0 conda-forge typing_extensions 4.12.2 pyha770c72_0 conda-forge -tzdata 2024a h8827d51_1 conda-forge +tzdata 2024b hc8b5060_0 conda-forge uri-template 1.3.0 pyhd8ed1ab_0 conda-forge urllib3 2.2.3 pyhd8ed1ab_0 conda-forge webcolors 24.8.0 pyhd8ed1ab_0 conda-forge @@ -117,6 +117,6 @@ wheel 0.44.0 pyhd8ed1ab_0 conda-forge xz 5.2.6 h166bdaf_0 conda-forge yaml 0.2.5 h7f98852_2 conda-forge zipp 3.20.2 pyhd8ed1ab_0 conda-forge -zlib 1.3.1 h4ab18f5_1 conda-forge +zlib 1.3.1 hb9d3cd8_2 conda-forge zstandard 0.23.0 py310ha39cb0e_1 conda-forge zstd 1.5.6 ha6fb4c9_0 conda-forge diff --git a/collections/ansible_collections/jupyter/lab/roles/setup/files/jupyterlabenvironment.txt b/collections/ansible_collections/jupyter/lab/roles/setup/files/jupyterlabenvironment.txt index 9a2e74b..126bd96 100644 --- a/collections/ansible_collections/jupyter/lab/roles/setup/files/jupyterlabenvironment.txt +++ b/collections/ansible_collections/jupyter/lab/roles/setup/files/jupyterlabenvironment.txt @@ -3,36 +3,35 @@ # Name Version Build Channel _libgcc_mutex 0.1 conda_forge conda-forge _openmp_mutex 4.5 2_kmp_llvm conda-forge -_sysroot_linux-64_curr_repodata_hack 3 h69a702a_16 conda-forge absl-py 2.1.0 pyhd8ed1ab_0 conda-forge alembic 1.13.3 pyhd8ed1ab_0 conda-forge alsa-lib 1.2.12 h4ab18f5_0 conda-forge annotated-types 0.7.0 pyhd8ed1ab_0 conda-forge -anyio 4.6.0 pyhd8ed1ab_1 conda-forge +anyio 4.6.2.post1 pyhd8ed1ab_0 conda-forge appdirs 1.4.4 pyh9f0ad1d_0 conda-forge archspec 0.2.3 pyhd8ed1ab_0 conda-forge argon2-cffi 23.1.0 pyhd8ed1ab_0 conda-forge argon2-cffi-bindings 21.2.0 py310ha75aee5_5 conda-forge arrow 1.3.0 pyhd8ed1ab_0 conda-forge asttokens 2.4.1 pyhd8ed1ab_0 conda-forge -astunparse 1.6.3 pyhd8ed1ab_0 conda-forge +astunparse 1.6.3 pyhd8ed1ab_2 conda-forge async-lru 2.0.4 pyhd8ed1ab_0 conda-forge async_generator 1.10 pyhd8ed1ab_1 conda-forge attrs 24.2.0 pyh71513ae_0 conda-forge automat 24.8.1 pyhd8ed1ab_0 conda-forge -aws-c-auth 0.7.31 h57bd9a3_0 conda-forge -aws-c-cal 0.7.4 hfd43aa1_1 conda-forge -aws-c-common 0.9.28 hb9d3cd8_0 conda-forge -aws-c-compression 0.2.19 h756ea98_1 conda-forge -aws-c-event-stream 0.4.3 h29ce20c_2 conda-forge -aws-c-http 0.8.10 h5e77a74_0 conda-forge -aws-c-io 0.14.18 h33ff4e5_10 conda-forge -aws-c-mqtt 0.10.6 h02abb05_0 conda-forge -aws-c-s3 0.6.6 h834ce55_0 conda-forge -aws-c-sdkutils 0.1.19 h756ea98_3 conda-forge -aws-checksums 0.1.20 h756ea98_0 conda-forge -aws-crt-cpp 0.28.3 h469002c_5 conda-forge -aws-sdk-cpp 1.11.407 h9f1560d_0 conda-forge +aws-c-auth 0.7.31 he1a10d6_2 conda-forge +aws-c-cal 0.7.4 hae4d56a_2 conda-forge +aws-c-common 0.9.29 hb9d3cd8_0 conda-forge +aws-c-compression 0.2.19 h2bff981_2 conda-forge +aws-c-event-stream 0.4.3 h19b0707_4 conda-forge +aws-c-http 0.8.10 h14a7884_2 conda-forge +aws-c-io 0.14.19 hc9e6898_1 conda-forge +aws-c-mqtt 0.10.7 hb8d5873_2 conda-forge +aws-c-s3 0.6.7 h666547d_0 conda-forge +aws-c-sdkutils 0.1.19 h2bff981_4 conda-forge +aws-checksums 0.1.20 h2bff981_1 conda-forge +aws-crt-cpp 0.28.3 hbe26082_8 conda-forge +aws-sdk-cpp 1.11.407 h25d6d5c_1 conda-forge azure-core-cpp 1.13.0 h935415a_0 conda-forge azure-identity-cpp 1.8.0 hd126650_2 conda-forge azure-storage-blobs-cpp 12.12.0 hd2e3451_0 conda-forge @@ -52,7 +51,7 @@ brotli 1.1.0 hb9d3cd8_2 conda-forge brotli-bin 1.1.0 hb9d3cd8_2 conda-forge brotli-python 1.1.0 py310hf71b8c6_2 conda-forge bzip2 1.0.8 h4bc722e_7 conda-forge -c-ares 1.33.1 heb4867d_0 conda-forge +c-ares 1.34.2 heb4867d_0 conda-forge ca-certificates 2024.8.30 hbcca054_0 conda-forge cached-property 1.5.2 hd8ed1ab_1 conda-forge cached_property 1.5.2 pyha770c72_1 conda-forge @@ -60,21 +59,22 @@ cairo 1.18.0 hbb29018_2 conda-forge certifi 2024.8.30 pyhd8ed1ab_0 conda-forge certipy 0.2.1 pyhd8ed1ab_0 conda-forge cffi 1.17.1 py310h8deb56e_0 conda-forge -charset-normalizer 3.3.2 pyhd8ed1ab_0 conda-forge +charset-normalizer 3.4.0 pyhd8ed1ab_0 conda-forge colorama 0.4.6 pyhd8ed1ab_0 conda-forge comm 0.2.2 pyhd8ed1ab_0 conda-forge conda 24.7.1 py310hff52083_0 conda-forge conda-libmamba-solver 24.9.0 pyhd8ed1ab_0 conda-forge -conda-package-handling 2.3.0 pyh7900ff3_0 conda-forge -conda-package-streaming 0.10.0 pyhd8ed1ab_0 conda-forge +conda-package-handling 2.4.0 pyh7900ff3_0 conda-forge +conda-package-streaming 0.11.0 pyhd8ed1ab_0 conda-forge configurable-http-proxy 4.6.2 hbf95b10_1 conda-forge constantly 15.1.0 py_0 conda-forge contourpy 1.3.0 py310h3788b33_2 conda-forge -cryptography 43.0.1 py310h6c63255_0 conda-forge +cpython 3.10.15 py310hd8ed1ab_2 conda-forge +cryptography 43.0.3 py310h6c63255_0 conda-forge cssselect 1.2.0 pyhd8ed1ab_0 conda-forge cycler 0.12.1 pyhd8ed1ab_0 conda-forge dbus 1.13.6 h5008d03_3 conda-forge -debugpy 1.8.6 py310hf71b8c6_0 conda-forge +debugpy 1.8.7 py310hf71b8c6_0 conda-forge decorator 5.1.1 pyhd8ed1ab_0 conda-forge defusedxml 0.7.1 pyhd8ed1ab_0 conda-forge distro 1.9.0 pyhd8ed1ab_0 conda-forge @@ -85,18 +85,18 @@ executing 2.1.0 pyhd8ed1ab_0 conda-forge expat 2.6.3 h5888daf_0 conda-forge filelock 3.16.1 pyhd8ed1ab_0 conda-forge flatbuffers 24.3.25 h59595ed_0 conda-forge -fmt 10.2.1 h00ab1b0_0 conda-forge +fmt 11.0.2 h434a139_0 conda-forge font-ttf-dejavu-sans-mono 2.37 hab24e00_0 conda-forge font-ttf-inconsolata 3.000 h77eed37_0 conda-forge font-ttf-source-code-pro 2.038 h77eed37_0 conda-forge -font-ttf-ubuntu 0.83 h77eed37_2 conda-forge +font-ttf-ubuntu 0.83 h77eed37_3 conda-forge fontconfig 2.14.2 h14ed4e7_0 conda-forge fonts-conda-ecosystem 1 0 conda-forge fonts-conda-forge 1 0 conda-forge fonttools 4.54.1 py310ha75aee5_0 conda-forge fqdn 1.5.1 pyhd8ed1ab_0 conda-forge freetype 2.12.1 h267a509_2 conda-forge -frozendict 2.4.4 py310ha75aee5_1 conda-forge +frozendict 2.4.6 py310ha75aee5_0 conda-forge fsspec 2024.9.0 pyhff2d567_0 conda-forge gast 0.5.5 pyhd8ed1ab_0 conda-forge gcc_impl_linux-64 10.4.0 h5231bdf_19 conda-forge @@ -116,11 +116,11 @@ gxx_impl_linux-64 10.4.0 h5231bdf_19 conda-forge gxx_linux-64 10.4.0 h6e491c6_13 conda-forge h11 0.14.0 pyhd8ed1ab_0 conda-forge h2 4.1.0 pyhd8ed1ab_0 conda-forge -h5py 3.11.0 nompi_py310hf054cd7_102 conda-forge +h5py 3.12.1 nompi_py310h60e0fe6_101 conda-forge harfbuzz 9.0.0 hfac3d4d_0 conda-forge hdf5 1.14.3 nompi_hdf9ad27_105 conda-forge hpack 4.0.0 pyh9f0ad1d_0 conda-forge -httpcore 1.0.5 pyhd8ed1ab_0 conda-forge +httpcore 1.0.6 pyhd8ed1ab_0 conda-forge httpx 0.27.2 pyhd8ed1ab_0 conda-forge hyperframe 6.0.1 pyhd8ed1ab_0 conda-forge hyperlink 21.0.0 pyhd3deb0d_0 conda-forge @@ -132,10 +132,10 @@ importlib_resources 6.4.5 pyhd8ed1ab_0 conda-forge incremental 24.7.2 pyhd8ed1ab_0 conda-forge iniconfig 2.0.0 pyhd8ed1ab_0 conda-forge ipykernel 6.29.5 pyh3099207_0 conda-forge -ipython 8.27.0 pyh707e725_0 conda-forge +ipython 8.28.0 pyh707e725_0 conda-forge isoduration 20.11.0 pyhd8ed1ab_0 conda-forge itemadapter 0.9.0 pyhd8ed1ab_0 conda-forge -itemloaders 1.3.1 pyhd8ed1ab_0 conda-forge +itemloaders 1.3.2 pyhd8ed1ab_0 conda-forge jedi 0.19.1 pyhd8ed1ab_0 conda-forge jinja2 3.1.4 pyhd8ed1ab_0 conda-forge jmespath 1.0.1 pyhd8ed1ab_0 conda-forge @@ -144,7 +144,7 @@ json5 0.9.25 pyhd8ed1ab_0 conda-forge jsonpatch 1.33 pyhd8ed1ab_0 conda-forge jsonpointer 3.0.0 py310hff52083_1 conda-forge jsonschema 4.23.0 pyhd8ed1ab_0 conda-forge -jsonschema-specifications 2023.12.1 pyhd8ed1ab_0 conda-forge +jsonschema-specifications 2024.10.1 pyhd8ed1ab_0 conda-forge jsonschema-with-format-nongpl 4.23.0 hd8ed1ab_0 conda-forge jupyter-lsp 2.2.5 pyhd8ed1ab_0 conda-forge jupyter-server-mathjax 0.2.6 pyh5bfe37b_1 conda-forge @@ -162,7 +162,7 @@ jupyterlab-git 0.50.1 pyhd8ed1ab_1 conda-forge jupyterlab_pygments 0.3.0 pyhd8ed1ab_1 conda-forge jupyterlab_server 2.27.3 pyhd8ed1ab_0 conda-forge keras 3.4.1 pyhd8ed1ab_2 conda-forge -kernel-headers_linux-64 3.10.0 h4a8ded7_16 conda-forge +kernel-headers_linux-64 3.10.0 he073ed8_17 conda-forge keyutils 1.6.1 h166bdaf_0 conda-forge kiwisolver 1.4.7 py310h3788b33_0 conda-forge krb5 1.21.3 h659f571_0 conda-forge @@ -182,29 +182,29 @@ libbrotlidec 1.1.0 hb9d3cd8_2 conda-forge libbrotlienc 1.1.0 hb9d3cd8_2 conda-forge libcblas 3.9.0 24_linux64_openblas conda-forge libclang-cpp18.1 18.1.8 default_hf981a13_5 conda-forge -libclang13 19.1.0 default_h9c6a7e4_0 conda-forge +libclang13 19.1.2 default_h9c6a7e4_1 conda-forge libcrc32c 1.1.2 h9c3ff4c_0 conda-forge libcups 2.3.3 h4637d8d_4 conda-forge libcurl 8.10.1 hbbe4b11_0 conda-forge -libdeflate 1.21 h4bc722e_0 conda-forge +libdeflate 1.22 hb9d3cd8_0 conda-forge libdrm 2.4.123 hb9d3cd8_0 conda-forge libedit 3.1.20191231 he28a2e2_2 conda-forge -libegl 1.7.0 ha4b6fd6_0 conda-forge +libegl 1.7.0 ha4b6fd6_1 conda-forge libev 4.33 hd590300_2 conda-forge libevent 2.1.12 hf998b51_1 conda-forge libexpat 2.6.3 h5888daf_0 conda-forge libffi 3.4.2 h7f98852_5 conda-forge -libgcc 14.1.0 h77fa898_1 conda-forge +libgcc 14.2.0 h77fa898_1 conda-forge libgcc-devel_linux-64 10.4.0 hd38fd1e_19 conda-forge -libgcc-ng 14.1.0 h69a702a_1 conda-forge -libgfortran 14.1.0 h69a702a_1 conda-forge -libgfortran-ng 14.1.0 h69a702a_1 conda-forge -libgfortran5 14.1.0 hc5f4f2c_1 conda-forge -libgl 1.7.0 ha4b6fd6_0 conda-forge +libgcc-ng 14.2.0 h69a702a_1 conda-forge +libgfortran 14.2.0 h69a702a_1 conda-forge +libgfortran-ng 14.2.0 h69a702a_1 conda-forge +libgfortran5 14.2.0 hd5240d6_1 conda-forge +libgl 1.7.0 ha4b6fd6_1 conda-forge libglib 2.80.2 hf974151_0 conda-forge -libglvnd 1.7.0 ha4b6fd6_0 conda-forge -libglx 1.7.0 ha4b6fd6_0 conda-forge -libgomp 14.1.0 h77fa898_1 conda-forge +libglvnd 1.7.0 ha4b6fd6_1 conda-forge +libglx 1.7.0 ha4b6fd6_1 conda-forge +libgomp 14.2.0 h77fa898_1 conda-forge libgoogle-cloud 2.29.0 h435de7b_0 conda-forge libgoogle-cloud-storage 2.29.0 h0121fbd_0 conda-forge libgpuarray 0.7.6 h7f98852_1003 conda-forge @@ -214,17 +214,17 @@ libiconv 1.17 hd590300_2 conda-forge libjpeg-turbo 3.0.0 hd590300_1 conda-forge liblapack 3.9.0 24_linux64_openblas conda-forge libllvm18 18.1.8 h8b73ec9_2 conda-forge -libllvm19 19.1.0 ha7bfdaf_0 conda-forge -libmamba 1.5.10 h4cc3d14_0 conda-forge -libmambapy 1.5.10 py310h86cbe3b_0 conda-forge +libllvm19 19.1.2 ha7bfdaf_0 conda-forge +libmamba 1.5.10 hf72d635_1 conda-forge +libmambapy 1.5.10 py310h6639945_1 conda-forge libnghttp2 1.58.0 h47da74e_1 conda-forge libnsl 2.0.1 hd590300_0 conda-forge libopenblas 0.3.27 pthreads_hac2b453_1 conda-forge -libopengl 1.7.0 ha4b6fd6_0 conda-forge +libopengl 1.7.0 ha4b6fd6_1 conda-forge libparquet 17.0.0 h39682fd_16_cpu conda-forge libpciaccess 0.18 hd590300_0 conda-forge libpng 1.6.44 hadc24fc_0 conda-forge -libpq 16.4 h2d7952a_1 conda-forge +libpq 16.4 h2d7952a_3 conda-forge libprotobuf 4.25.3 hd5b35b9_1 conda-forge libre2-11 2023.09.01 h5a48ba9_2 conda-forge libsanitizer 10.4.0 h5246dfb_19 conda-forge @@ -232,23 +232,23 @@ libsodium 1.0.20 h4ab18f5_0 conda-forge libsolv 0.7.30 h3509ff9_0 conda-forge libsqlite 3.46.1 hadc24fc_0 conda-forge libssh2 1.11.0 h0841786_0 conda-forge -libstdcxx 14.1.0 hc0a3c3a_1 conda-forge +libstdcxx 14.2.0 hc0a3c3a_1 conda-forge libstdcxx-devel_linux-64 10.4.0 hd38fd1e_19 conda-forge -libstdcxx-ng 14.1.0 h4852527_1 conda-forge +libstdcxx-ng 14.2.0 h4852527_1 conda-forge libthrift 0.20.0 h0e7cc3e_1 conda-forge -libtiff 4.7.0 h6565414_0 conda-forge +libtiff 4.7.0 he137b08_1 conda-forge libtorch 2.3.1 cpu_mkl_h0bb0d08_100 conda-forge libutf8proc 2.8.0 h166bdaf_0 conda-forge libuuid 2.38.1 h0b41bf4_0 conda-forge -libuv 1.49.0 hb9d3cd8_0 conda-forge +libuv 1.49.2 hb9d3cd8_0 conda-forge libwebp-base 1.4.0 hd590300_0 conda-forge libxcb 1.17.0 h8a09558_0 conda-forge libxcrypt 4.4.36 hd590300_1 conda-forge libxkbcommon 1.7.0 h2c5496b_1 conda-forge libxml2 2.12.7 h4c95cb1_3 conda-forge libxslt 1.1.39 h76b75d6_0 conda-forge -libzlib 1.3.1 h4ab18f5_1 conda-forge -llvm-openmp 18.1.8 hf5423f3_1 conda-forge +libzlib 1.3.1 hb9d3cd8_2 conda-forge +llvm-openmp 19.1.2 h024ca30_0 conda-forge lxml 5.3.0 py310h6ee67d5_1 conda-forge lz4-c 1.9.4 hcb278e6_0 conda-forge lzo 2.10 hd590300_1001 conda-forge @@ -256,7 +256,7 @@ mako 1.3.5 pyhd8ed1ab_0 conda-forge mamba_gator 5.2.1 pyhd8ed1ab_0 conda-forge markdown 3.6 pyhd8ed1ab_0 conda-forge markdown-it-py 3.0.0 pyhd8ed1ab_0 conda-forge -markupsafe 2.1.5 py310ha75aee5_1 conda-forge +markupsafe 3.0.2 py310h89163eb_0 conda-forge matplotlib 3.9.1 py310hff52083_2 conda-forge matplotlib-base 3.9.1 py310hf02ac8c_2 conda-forge matplotlib-inline 0.1.7 pyhd8ed1ab_0 conda-forge @@ -279,7 +279,7 @@ nbdime 4.0.2 pyhd8ed1ab_0 conda-forge nbformat 5.10.4 pyhd8ed1ab_0 conda-forge ncurses 6.5 he02047a_1 conda-forge nest-asyncio 1.6.0 pyhd8ed1ab_0 conda-forge -networkx 3.3 pyhd8ed1ab_1 conda-forge +networkx 3.4.1 pyhd8ed1ab_0 conda-forge nodejs 22.5.1 h6d9b948_0 conda-forge notebook 7.2.2 pyhd8ed1ab_0 conda-forge notebook-shim 0.2.4 pyhd8ed1ab_0 conda-forge @@ -287,8 +287,8 @@ numpy 1.26.4 py310hb13e2d6_0 conda-forge oauthlib 3.2.2 pyhd8ed1ab_0 conda-forge openjpeg 2.5.2 h488ebb8_0 conda-forge openssl 3.3.2 hb9d3cd8_0 conda-forge -opt_einsum 3.3.0 pyhc1e730c_2 conda-forge -optree 0.12.1 py310h3788b33_1 conda-forge +opt_einsum 3.4.0 pyhd8ed1ab_0 conda-forge +optree 0.13.0 py310h3788b33_0 conda-forge orc 2.0.2 h669347b_0 conda-forge outcome 1.3.0.post0 pyhd8ed1ab_0 conda-forge overrides 7.7.0 pyhd8ed1ab_0 conda-forge @@ -312,7 +312,7 @@ prometheus_client 0.21.0 pyhd8ed1ab_0 conda-forge prompt-toolkit 3.0.48 pyha770c72_0 conda-forge protego 0.3.1 pyhd8ed1ab_0 conda-forge protobuf 4.25.3 py310h0e2eeba_1 conda-forge -psutil 6.0.0 py310ha75aee5_1 conda-forge +psutil 6.0.0 py310ha75aee5_2 conda-forge pthread-stubs 0.4 hb9d3cd8_1002 conda-forge ptyprocess 0.7.0 pyhd3deb0d_0 conda-forge pure_eval 0.2.3 pyhd8ed1ab_0 conda-forge @@ -332,12 +332,12 @@ pygments 2.18.0 pyhd8ed1ab_0 conda-forge pygpu 0.7.6 py310h96516ba_1003 conda-forge pyjwt 2.9.0 pyhd8ed1ab_1 conda-forge pyopenssl 24.2.1 pyhd8ed1ab_2 conda-forge -pyparsing 3.1.4 pyhd8ed1ab_0 conda-forge +pyparsing 3.2.0 pyhd8ed1ab_1 conda-forge pyside6 6.7.2 py310hfd10a26_4 conda-forge pysocks 1.7.1 pyha2e5f31_6 conda-forge pyspark 3.5.1 pyhd8ed1ab_0 conda-forge pytest 8.3.2 pyhd8ed1ab_0 conda-forge -python 3.10.15 h4a871b0_0_cpython conda-forge +python 3.10.15 h4a871b0_2_cpython conda-forge python-dateutil 2.9.0 pyhd8ed1ab_0 conda-forge python-fastjsonschema 2.20.0 pyhd8ed1ab_0 conda-forge python-flatbuffers 24.3.25 pyh59ac667_0 conda-forge @@ -347,7 +347,7 @@ python_abi 3.10 5_cp310 conda-forge pytorch 2.3.1 cpu_mkl_py310h75865b9_100 conda-forge pytz 2024.2 pyhd8ed1ab_0 conda-forge pyyaml 6.0.2 py310ha75aee5_1 conda-forge -pyzmq 26.2.0 py310h71f11fc_2 conda-forge +pyzmq 26.2.0 py310h71f11fc_3 conda-forge qhull 2020.2 h434a139_5 conda-forge qt6-main 6.7.2 h7d13b96_3 conda-forge queuelib 1.7.0 pyhd8ed1ab_1 conda-forge @@ -360,11 +360,11 @@ requests 2.32.3 pyhd8ed1ab_0 conda-forge requests-file 2.1.0 pyhd8ed1ab_0 conda-forge rfc3339-validator 0.1.4 pyhd8ed1ab_0 conda-forge rfc3986-validator 0.1.1 pyh9f0ad1d_0 conda-forge -rich 13.8.1 pyhd8ed1ab_0 conda-forge +rich 13.9.2 pyhd8ed1ab_0 conda-forge rpds-py 0.20.0 py310h505e2c1_1 conda-forge -ruamel.yaml 0.18.6 py310h2372a71_0 conda-forge -ruamel.yaml.clib 0.2.8 py310h2372a71_0 conda-forge -s2n 1.5.3 h7b32b05_0 conda-forge +ruamel.yaml 0.18.6 py310ha75aee5_1 conda-forge +ruamel.yaml.clib 0.2.8 py310ha75aee5_1 conda-forge +s2n 1.5.5 h3931f03_0 conda-forge scikit-learn 1.5.1 py310h27f47ee_1 conda-forge scipy 1.14.0 py310h3fe0a75_2 conda-forge scrapy 2.11.2 py310hff52083_0 conda-forge @@ -384,37 +384,37 @@ snappy 1.2.1 ha2e4443_0 conda-forge sniffio 1.3.1 pyhd8ed1ab_0 conda-forge sortedcontainers 2.4.0 pyhd8ed1ab_0 conda-forge soupsieve 2.5 pyhd8ed1ab_1 conda-forge -sqlalchemy 2.0.35 py310ha75aee5_0 conda-forge +sqlalchemy 2.0.36 py310ha75aee5_0 conda-forge stack_data 0.6.2 pyhd8ed1ab_0 conda-forge -statsmodels 0.14.3 py310hf462985_1 conda-forge -sympy 1.13.2 pypyh2585a3b_103 conda-forge -sysroot_linux-64 2.17 h4a8ded7_16 conda-forge +statsmodels 0.14.4 py310hf462985_0 conda-forge +sympy 1.13.3 pyh2585a3b_104 conda-forge +sysroot_linux-64 2.17 h4a8ded7_17 conda-forge tbb 2021.13.0 h84d6215_0 conda-forge tensorboard 2.16.2 pyhd8ed1ab_0 conda-forge -tensorboard-data-server 0.7.0 py310h75e40e8_1 conda-forge +tensorboard-data-server 0.7.0 py310h6c63255_2 conda-forge tensorflow 2.16.2 cpu_py310h42475c5_0 conda-forge tensorflow-base 2.16.2 cpu_py310h273d231_0 conda-forge tensorflow-estimator 2.16.2 cpu_py310hc6dcfef_0 conda-forge -termcolor 2.4.0 pyhd8ed1ab_0 conda-forge +termcolor 2.5.0 pyhd8ed1ab_0 conda-forge terminado 0.18.1 pyh0d859eb_0 conda-forge theano 1.0.5 py310hd8f1fbe_3 conda-forge threadpoolctl 3.5.0 pyhc1e730c_0 conda-forge tinycss2 1.3.0 pyhd8ed1ab_0 conda-forge tk 8.6.13 noxft_h4845f30_101 conda-forge tldextract 5.1.2 pyhd8ed1ab_0 conda-forge -tomli 2.0.1 pyhd8ed1ab_0 conda-forge +tomli 2.0.2 pyhd8ed1ab_0 conda-forge tornado 6.4.1 py310ha75aee5_1 conda-forge tqdm 4.66.5 pyhd8ed1ab_0 conda-forge traitlets 5.14.3 pyhd8ed1ab_0 conda-forge -trio 0.26.2 py310hff52083_1 conda-forge +trio 0.27.0 py310hff52083_0 conda-forge trio-websocket 0.11.1 pyhd8ed1ab_0 conda-forge truststore 0.9.2 pyhd8ed1ab_0 conda-forge -twisted 24.7.0 py310h5b4e0ec_0 conda-forge -types-python-dateutil 2.9.0.20240906 pyhd8ed1ab_0 conda-forge +twisted 24.7.0 py310ha75aee5_1 conda-forge +types-python-dateutil 2.9.0.20241003 pyhff2d567_0 conda-forge typing-extensions 4.12.2 hd8ed1ab_0 conda-forge typing_extensions 4.12.2 pyha770c72_0 conda-forge typing_utils 0.1.0 pyhd8ed1ab_0 conda-forge -tzdata 2024a h8827d51_1 conda-forge +tzdata 2024b hc8b5060_0 conda-forge unicodedata2 15.1.0 py310h2372a71_0 conda-forge uri-template 1.3.0 pyhd8ed1ab_0 conda-forge urllib3 2.2.3 pyhd8ed1ab_0 conda-forge @@ -434,23 +434,21 @@ xcb-util-image 0.4.0 hb711507_2 conda-forge xcb-util-keysyms 0.4.1 hb711507_0 conda-forge xcb-util-renderutil 0.3.10 hb711507_0 conda-forge xcb-util-wm 0.4.2 hb711507_0 conda-forge -xkeyboard-config 2.42 h4ab18f5_0 conda-forge -xorg-libice 1.1.1 hd590300_0 conda-forge -xorg-libsm 1.2.4 h7391055_0 conda-forge +xkeyboard-config 2.43 hb9d3cd8_0 conda-forge +xorg-libice 1.1.1 hb9d3cd8_1 conda-forge +xorg-libsm 1.2.4 he73a12e_1 conda-forge xorg-libx11 1.8.10 h4f16b4b_0 conda-forge xorg-libxau 1.0.11 hb9d3cd8_1 conda-forge -xorg-libxdmcp 1.1.3 hb9d3cd8_1 conda-forge -xorg-libxext 1.3.4 h0b41bf4_2 conda-forge -xorg-libxrender 0.9.11 hd590300_0 conda-forge -xorg-renderproto 0.11.1 hb9d3cd8_1003 conda-forge -xorg-xextproto 7.3.0 hb9d3cd8_1004 conda-forge +xorg-libxdmcp 1.1.5 hb9d3cd8_0 conda-forge +xorg-libxext 1.3.6 hb9d3cd8_0 conda-forge +xorg-libxrender 0.9.11 hb9d3cd8_1 conda-forge xorg-xorgproto 2024.1 hb9d3cd8_1 conda-forge xz 5.2.6 h166bdaf_0 conda-forge yaml 0.2.5 h7f98852_2 conda-forge yaml-cpp 0.8.0 h59595ed_0 conda-forge -zeromq 4.3.5 ha4adb4c_5 conda-forge +zeromq 4.3.5 h3b0a872_6 conda-forge zipp 3.20.2 pyhd8ed1ab_0 conda-forge -zlib 1.3.1 h4ab18f5_1 conda-forge -zope.interface 7.0.3 py310ha75aee5_1 conda-forge +zlib 1.3.1 hb9d3cd8_2 conda-forge +zope.interface 7.1.0 py310ha75aee5_0 conda-forge zstandard 0.23.0 py310ha39cb0e_1 conda-forge zstd 1.5.6 ha6fb4c9_0 conda-forge diff --git a/collections/ansible_collections/loadbalancing/keepalived/roles/setup/tasks/main.yml b/collections/ansible_collections/loadbalancing/keepalived/roles/setup/tasks/main.yml index 2465835..7791038 100644 --- a/collections/ansible_collections/loadbalancing/keepalived/roles/setup/tasks/main.yml +++ b/collections/ansible_collections/loadbalancing/keepalived/roles/setup/tasks/main.yml @@ -6,11 +6,9 @@ update_cache: true - name: Get ip address of remote host - use bash to avoid dns python conflict for dig lookup - when: - - groups.hubs is defined - - jupyterhub_domain_ip is defined + when: jupyterhub_domain_ip_address is not defined ansible.builtin.command: dig +short jupyterhub.{{ domain }} - register: jupyterhub_domain_ip_address + register: get_jupyterhub_domain_ip_address - name: Configure keepalived for Hetzner Cloud when: diff --git a/collections/ansible_collections/loadbalancing/keepalived/roles/setup/templates/failover.sh b/collections/ansible_collections/loadbalancing/keepalived/roles/setup/templates/failover.sh index 846f3ad..5bfa7d4 100644 --- a/collections/ansible_collections/loadbalancing/keepalived/roles/setup/templates/failover.sh +++ b/collections/ansible_collections/loadbalancing/keepalived/roles/setup/templates/failover.sh @@ -1,7 +1,7 @@ #!/bin/bash export HCLOUD_TOKEN={{ hetzner_token }} -JUPYTERHUB_FLOATING_IP_ID=`hcloud floating-ip list | grep {{ jupyterhub_domain_ip_address.stdout }} | awk {'print $1'}` +JUPYTERHUB_FLOATING_IP_ID=`hcloud floating-ip list | grep {{ get_jupyterhub_domain_ip_address.stdout }} | awk {'print $1'}` # assign floating ip of domain jupyterhub.{{ domain }} hcloud floating-ip assign $JUPYTERHUB_FLOATING_IP_ID {{ inventory_hostname_short }} diff --git a/collections/ansible_collections/loadbalancing/keepalived/roles/setup/templates/keepalived.conf b/collections/ansible_collections/loadbalancing/keepalived/roles/setup/templates/keepalived.conf index 66fe9a1..b2cfc83 100644 --- a/collections/ansible_collections/loadbalancing/keepalived/roles/setup/templates/keepalived.conf +++ b/collections/ansible_collections/loadbalancing/keepalived/roles/setup/templates/keepalived.conf @@ -28,7 +28,7 @@ vrrp_instance jupyterhub_frontend { {% endfor %} {% endif %} virtual_ipaddress { - {{ jupyterhub_domain_ip_address.stdout }} + {{ jupyterhub_domain_ip_address if molecule_deployment is defined and molecule_deployment else get_jupyterhub_domain_ip_address.stdout }} } track_script { check_haproxy diff --git a/collections/ansible_collections/rdbms/postgres/extensions/molecule/ha_setup/molecule.yml b/collections/ansible_collections/rdbms/postgres/extensions/molecule/ha_setup/molecule.yml index dd14e97..e8bea59 100644 --- a/collections/ansible_collections/rdbms/postgres/extensions/molecule/ha_setup/molecule.yml +++ b/collections/ansible_collections/rdbms/postgres/extensions/molecule/ha_setup/molecule.yml @@ -86,9 +86,9 @@ provisioner: tls_user: root tls_group: root certs_source: - - /opt/selfsigned/{{ ansible_fqdn }}/cert.pem - - /opt/selfsigned/{{ ansible_fqdn }}/RootCA.pem - - /opt/selfsigned/{{ ansible_fqdn }}/key.pem + - /tmp/selfsigned_certs/{{ ansible_fqdn }}/cert.pem + - /tmp/selfsigned_certs/{{ ansible_fqdn }}/chain.pem + - /tmp/selfsigned_certs/{{ ansible_fqdn }}/key.pem certs_mode: - "0660" - "0660" diff --git a/collections/ansible_collections/rdbms/postgres/roles/install/tasks/install-postgres.yml b/collections/ansible_collections/rdbms/postgres/roles/install/tasks/install-postgres.yml index 71151a0..b518d35 100644 --- a/collections/ansible_collections/rdbms/postgres/roles/install/tasks/install-postgres.yml +++ b/collections/ansible_collections/rdbms/postgres/roles/install/tasks/install-postgres.yml @@ -34,6 +34,9 @@ update_cache: true - name: Add service user to group ssl-cert + when: + - service_user is defined + - service_group is defined ansible.builtin.user: name: "{{ service_user }}" group: "{{ service_group }}" diff --git a/collections/ansible_collections/rdbms/postgres/roles/schemes/tasks/main.yml b/collections/ansible_collections/rdbms/postgres/roles/schemes/tasks/main.yml index c1167a1..371c269 100644 --- a/collections/ansible_collections/rdbms/postgres/roles/schemes/tasks/main.yml +++ b/collections/ansible_collections/rdbms/postgres/roles/schemes/tasks/main.yml @@ -1,4 +1,20 @@ --- +- name: Get ip address of remote host if jupyterhub_domain_ip_address is undefined - use bash to avoid dns python conflict for dig lookup + when: + - jupyterhub_domain_ip_address is not defined + - groups.hubs is defined + - groups.hubs | length == 2 + ansible.builtin.command: dig +short jupyterhub.{{ domain }} + register: get_jupyterhub_domain_ip_address + +- name: Set fact if jupyterhub_domain_ip_address is undefined + when: + - jupyterhub_domain_ip_address is not defined + - groups.hubs is defined + - groups.hubs | length == 2 + ansible.builtin.set_fact: + jupyterhub_domain_ip_address: "{{ get_jupyterhub_domain_ip_address.stdout }}" + - name: Create postgres user and schemes ansible.builtin.include_tasks: create-user-and-schemes.yml loop: "{{ postgres_schemes }}" diff --git a/collections/ansible_collections/tls/certs/roles/distribute/defaults/main.yml b/collections/ansible_collections/tls/certs/roles/distribute/defaults/main.yml index 63922da..6f0bd6d 100644 --- a/collections/ansible_collections/tls/certs/roles/distribute/defaults/main.yml +++ b/collections/ansible_collections/tls/certs/roles/distribute/defaults/main.yml @@ -1,17 +1,11 @@ --- self_signed_certificates: true -# Location of issued self signed certificate and key on localhost - available after Ansible Role tls.certs.issue -certs_source: - - /opt/selfsigned/{{ ansible_fqdn }}/cert.pem - - /opt/selfsigned/{{ ansible_fqdn }}/RootCA.pem - - /opt/selfsigned/{{ ansible_fqdn }}/key.pem - # Names of issued self signed certificate and key on remote hosts - used for Ansible Role tls.certs.distribute certs_dest: - - cert.pem - - chain.pem - - key.pem + - "{{ ansible_fqdn }}.cert.pem" + - "{{ ansible_fqdn }}.csr.pem" + - "{{ ansible_fqdn }}.key" # Mode of issued self signed certificate and key on remote hosts - used for Ansible Role tls.certs.distribute certs_mode: diff --git a/collections/ansible_collections/tls/certs/roles/distribute/tasks/main.yml b/collections/ansible_collections/tls/certs/roles/distribute/tasks/main.yml index 18b6d1b..87c1b2a 100644 --- a/collections/ansible_collections/tls/certs/roles/distribute/tasks/main.yml +++ b/collections/ansible_collections/tls/certs/roles/distribute/tasks/main.yml @@ -10,7 +10,22 @@ group: "{{ tls_group }}" mode: "0770" + - name: Change owner of certs and key on remote server + when: self_signed_certificates + ansible.builtin.file: + path: "/etc/ssl/private/{{ item.0 }}" + state: file + owner: "{{ tls_user }}" + group: "{{ tls_group }}" + mode: "{{ item.1 }}" + with_together: + - "{{ certs_dest }}" + - "{{ certs_mode }}" + loop_control: + label: "{{ item.0 }}" + - name: Copy cert, fullchain and private key to server + when: not self_signed_certificates ansible.builtin.copy: src: "{{ item.0 }}" dest: /etc/ssl/private/{{ item.1 }} diff --git a/collections/ansible_collections/tls/certs/roles/issue/tasks/issue-selfsigned-certs.yml b/collections/ansible_collections/tls/certs/roles/issue/tasks/issue-selfsigned-certs.yml index a0f092e..69d15f6 100644 --- a/collections/ansible_collections/tls/certs/roles/issue/tasks/issue-selfsigned-certs.yml +++ b/collections/ansible_collections/tls/certs/roles/issue/tasks/issue-selfsigned-certs.yml @@ -1,77 +1,221 @@ --- -- name: Set fact for no root user - use command to avoid ansible_fact_caching for root user - become: false - ansible.builtin.command: whoami - register: no_root_user - -- name: Create folder for certificate and key - delegate_to: localhost - run_once: true - become: true - ansible.builtin.file: - path: /opt/selfsigned - state: directory - owner: "{{ no_root_user.stdout }}" - mode: "0755" - -- name: Check if certificate and key are already present on ansible server - delegate_to: localhost +- name: Check if self signed certificates are already present ansible.builtin.stat: - path: /opt/selfsigned/{{ ansible_fqdn }} - register: cert_dir + path: /tmp/selfsigned_certs + register: selfsigned_remote_path -- name: Issue and fetch certificates only if not present on ansible server - become: true - when: not cert_dir.stat.exists +- name: Create self signed certificates if not present + when: not selfsigned_remote_path.stat.exists block: - - name: Create CA Key - ansible.builtin.command: openssl genrsa -out /tmp/RootCA.key 4096 - - - name: Create Root CA - ansible.builtin.command: openssl req -x509 -new -nodes -key /tmp/RootCA.key -sha256 \ - -days 365 -out /tmp/RootCA.pem -subj '/CN=Root CA/C=GE/ST=Hessen/L=Kassel/O=FreelancerMap' + # https://docs.securosys.com/openssl/osslv3/Use-Cases/self_signed_certificate + - name: Create directories + ansible.builtin.file: + path: "{{ item }}" + state: directory + mode: "0755" + loop: + - /tmp/selfsigned_certs/myCA/rootCA/certs + - /tmp/selfsigned_certs/myCA/rootCA/crl + - /tmp/selfsigned_certs/myCA/rootCA/newcerts + - /tmp/selfsigned_certs/myCA/rootCA/private + - /tmp/selfsigned_certs/myCA/rootCA/csr + - /tmp/selfsigned_certs/myCA/intermediateCA/certs + - /tmp/selfsigned_certs/myCA/intermediateCA/crl + - /tmp/selfsigned_certs/myCA/intermediateCA/newcerts + - /tmp/selfsigned_certs/myCA/intermediateCA/private + - /tmp/selfsigned_certs/myCA/intermediateCA/csr - - name: Install ca-certificates - ansible.builtin.apt: - name: ca-certificates - state: present - update_cache: true + - name: Touch initial files + ansible.builtin.file: + path: "{{ item }}" + state: touch + mode: "0644" + modification_time: preserve + access_time: preserve + loop: + - /tmp/selfsigned_certs/myCA/rootCA/index.txt + - /tmp/selfsigned_certs/myCA/intermediateCA/index.txt - - name: Copy Root CA to /usr/local/share/ca-certificates/ + - name: Create initial files ansible.builtin.copy: - src: /tmp/RootCA.pem - dest: /usr/local/share/ca-certificates/RootCA.crt - remote_src: true - owner: root - group: root + content: 1000 + dest: "{{ item }}" mode: "0644" + loop: + - /tmp/selfsigned_certs/myCA/rootCA/serial + - /tmp/selfsigned_certs/myCA/rootCA/crlnumber + - /tmp/selfsigned_certs/myCA/intermediateCA/serial + - /tmp/selfsigned_certs/myCA/intermediateCA/crlnumber - - name: Update ca-certificates - ansible.builtin.command: update-ca-certificates + - name: Download configuration for root CA + ansible.builtin.get_url: + url: https://docs.securosys.com/assets/files/openssl_root-d562bd505a4ae334dfa0664b2b13676b.cnf + dest: /tmp/selfsigned_certs/openssl_root.cnf + mode: "0644" - - name: Create cert for host - ansible.builtin.command: openssl req -new -nodes -out /tmp/{{ ansible_fqdn }}.csr -newkey rsa:4096 \ - -keyout /tmp/{{ ansible_fqdn }}.key -subj '/CN={{ ansible_fqdn }}/C=GE/ST=Hessen/L=Kassel/O=FreelancerMap' + - name: Download configuration for intermediate CA + ansible.builtin.get_url: + url: https://docs.securosys.com/assets/files/openssl_intermediate-a244856f0122c19165cd5ed89017588b.cnf + dest: /tmp/selfsigned_certs/openssl_intermediate.cnf + mode: "0644" - - name: Sign cert for host - ansible.builtin.command: openssl x509 -req -in /tmp/{{ ansible_fqdn }}.csr \ - -CA /tmp/RootCA.pem -CAkey /tmp/RootCA.key -CAcreateserial \ - -out /tmp/{{ ansible_fqdn }}.crt -days 730 -sha256 + - name: Create a key - Root CA certificate + args: + chdir: /tmp/selfsigned_certs + ansible.builtin.command: openssl genpkey \ + -algorithm rsa -pkeyopt rsa_keygen_bits:4096 \ + -out rootCA.key - - name: Create folder for host on ansible server - delegate_to: localhost - ansible.builtin.file: - path: /opt/selfsigned/{{ ansible_fqdn }} - state: directory - owner: "{{ no_root_user.stdout }}" - mode: "0777" - - - name: Fetch self signed certificate and key to ansible server - become: true - ansible.builtin.fetch: - src: /tmp/{{ item.0 }} - dest: /opt/selfsigned/{{ ansible_fqdn }}/{{ item.1 }} - flat: true - with_together: - - ["{{ ansible_fqdn }}.crt", RootCA.pem, "{{ ansible_fqdn }}.key"] - - [cert.pem, RootCA.pem, key.pem] + # The v3_ca extensions defined in the openssl_root.cnf configuration are applied which enable the "CA:true" property + - name: Create a self-signed certificate + args: + chdir: /tmp/selfsigned_certs + ansible.builtin.command: openssl req -config openssl_root.cnf \ + -key rootCA.key -new -x509 -days 7300 \ + -sha256 -extensions v3_ca -out myCA/rootCA/certs/ca.cert.pem \ + -subj "/C=GE/ST=Hessen/L=Kassel/O=FreelancerMap/OU=Georg Schulz/CN=Root CA" + + - name: Create a key - Intermediate CA certificate + args: + chdir: /tmp/selfsigned_certs + ansible.builtin.command: openssl genpkey \ + -algorithm rsa -pkeyopt rsa_keygen_bits:4096 \ + -out intermediateCA.key + + - name: Create a certificate signing request (CSR) + args: + chdir: /tmp/selfsigned_certs + ansible.builtin.command: openssl req -config openssl_intermediate.cnf \ + -key intermediateCA.key \ + -new -sha256 \ + -out myCA/intermediateCA/certs/intermediate.csr.pem \ + -subj "/C=GE/ST=Hessen/L=Kassel/O=FreelancerMap/OU=Georg Schulz/CN=Intermediate CA" + + # The v3_intermediate_ca extensions defined in the openssl_root.cnf configuration are applied + - name: As root CA, sign the CSR for the intermediate CA with the root CA key + args: + chdir: /tmp/selfsigned_certs + ansible.builtin.command: openssl ca -batch -config openssl_root.cnf \ + -keyfile rootCA.key \ + -extensions v3_intermediate_ca -days 750 -notext -md sha256 \ + -in myCA/intermediateCA/certs/intermediate.csr.pem \ + -out myCA/intermediateCA/certs/intermediate.cert.pem + + - name: Verify the signature on the intermediate certificate + args: + chdir: /tmp/selfsigned_certs + ansible.builtin.command: openssl verify -CAfile myCA/rootCA/certs/ca.cert.pem \ + myCA/intermediateCA/certs/intermediate.cert.pem + + - name: Create a key - Server certificate + args: + chdir: /tmp/selfsigned_certs + ansible.builtin.command: openssl genpkey \ + -algorithm rsa -pkeyopt rsa_keygen_bits:4096 \ + -out {{ ansible_fqdn }}.key + + - name: Create a certificate signing request (CSR) + args: + chdir: /tmp/selfsigned_certs + ansible.builtin.command: openssl req -copy_extensions=copyall \ + -key {{ ansible_fqdn }}.key \ + -new -sha256 \ + -out {{ ansible_fqdn }}.csr.pem \ + -subj "/C=GE/ST=Hessen/L=Kassel/O=FreelancerMap/OU=Georg Schulz/CN={{ ansible_fqdn }}" \ + -addext "subjectAltName = DNS:example.com, DNS:*.example.com, DNS:{{ inventory_hostname }}" + + - name: As intermediate CA, sign the server CSR with the intermediate CA key + args: + chdir: /tmp/selfsigned_certs + ansible.builtin.command: openssl ca -batch -config openssl_intermediate.cnf \ + -extensions v3_server_cert \ + -keyfile intermediateCA.key \ + -days 40 -notext -md sha256 \ + -in {{ ansible_fqdn }}.csr.pem \ + -out {{ ansible_fqdn }}.cert.pem + + - name: Verify the certificate by checking the signatures using OpenSSL + args: + chdir: /tmp/selfsigned_certs + ansible.builtin.shell: | + cat myCA/intermediateCA/certs/intermediate.cert.pem \ + myCA/rootCA/certs/ca.cert.pem \ + > chain.pem + openssl verify -show_chain \ + -trusted myCA/rootCA/certs/ca.cert.pem \ + --untrusted chain.pem \ + {{ ansible_fqdn }}.cert.pem + +- name: Install ca-certificates + ansible.builtin.apt: + name: ca-certificates + state: present + update_cache: true + +- name: Copy Root CA to /usr/local/share/ca-certificates/ + ansible.builtin.copy: + src: /tmp/selfsigned_certs/chain.pem + dest: /usr/local/share/ca-certificates/RootCA.crt + remote_src: true + owner: root + group: root + mode: "0644" + +- name: Update ca-certificates + ansible.builtin.command: update-ca-certificates + +- name: Change ownership of /etc/ssl/private + ansible.builtin.file: + path: /etc/ssl/private + state: directory + owner: "{{ tls_user }}" + group: "{{ tls_group }}" + mode: "0770" + +- name: Copy server cert.pem to /etc/ssl/private + ansible.builtin.copy: + src: /tmp/selfsigned_certs/{{ ansible_fqdn }}.cert.pem + dest: /etc/ssl/private/cert.pem + remote_src: true + owner: "{{ tls_user }}" + group: "{{ tls_group }}" + mode: "0660" + +- name: Copy chain.pem to /etc/ssl/private + ansible.builtin.copy: + src: /tmp/selfsigned_certs/chain.pem + dest: /etc/ssl/private/chain.pem + remote_src: true + owner: "{{ tls_user }}" + group: "{{ tls_group }}" + mode: "0660" + +- name: Copy server key file to /etc/ssl/private + ansible.builtin.copy: + src: /tmp/selfsigned_certs/{{ ansible_fqdn }}.key + dest: /etc/ssl/private/key.pem + remote_src: true + owner: "{{ tls_user }}" + group: "{{ tls_group }}" + mode: "0400" + +- name: Fetch server cert.pem to ansible server + become: true + ansible.builtin.fetch: + src: /etc/ssl/private/cert.pem + dest: /tmp/selfsigned_certs/{{ ansible_fqdn }}/cert.pem + flat: true + +- name: Fetch server key.pem to ansible server + become: true + ansible.builtin.fetch: + src: /etc/ssl/private/key.pem + dest: /tmp/selfsigned_certs/{{ ansible_fqdn }}/key.pem + flat: true + +- name: Fetch server chain.pem to ansible server + become: true + ansible.builtin.fetch: + src: /etc/ssl/private/chain.pem + dest: /tmp/selfsigned_certs/{{ ansible_fqdn }}/chain.pem + flat: true diff --git a/collections/ansible_collections/tls/certs/roles/setup/meta/main.yml b/collections/ansible_collections/tls/certs/roles/setup/meta/main.yml index d2d2d24..0feabbb 100644 --- a/collections/ansible_collections/tls/certs/roles/setup/meta/main.yml +++ b/collections/ansible_collections/tls/certs/roles/setup/meta/main.yml @@ -4,5 +4,6 @@ dependencies: - name: ide.environment.variables - name: tls.certs.issue - name: tls.certs.distribute + when: not self_signed_certificates - name: tls.java.keystore - name: tls.java.truststore diff --git a/collections/ansible_collections/tls/java/roles/truststore/defaults/main.yml b/collections/ansible_collections/tls/java/roles/truststore/defaults/main.yml index f499bc9..275fc36 100644 --- a/collections/ansible_collections/tls/java/roles/truststore/defaults/main.yml +++ b/collections/ansible_collections/tls/java/roles/truststore/defaults/main.yml @@ -1,3 +1,3 @@ --- -local_cert_path: "{{ '/opt/selfsigned' if self_signed_certificates is defined and self_signed_certificates == true else '/opt/letsencrypt' }}" +local_cert_path: "{{ '/tmp/selfsigned_certs' if self_signed_certificates is defined and self_signed_certificates == true else '/opt/letsencrypt' }}" cert_name: "{{ 'cert.pem' if self_signed_certificates is defined and self_signed_certificates == true else 'cert1.pem' }}" diff --git a/requirements.sh b/requirements.sh index 1ef9779..2f5782b 100644 --- a/requirements.sh +++ b/requirements.sh @@ -10,7 +10,7 @@ conda_env_python_version=3.12 ansible_vault_file=~/.vault_pass.txt echo "====================================================================================================================" -echo "======================== Idempotent installation script for Terraform, Go and Ansible ========================" +echo "====================== Idempotent installation script for Terraform, Go and Ansible ======================" echo "====================================================================================================================" echo "" @@ -149,6 +149,6 @@ EOF fi echo "====================================================================================================================" -echo "======================== Installation process for Terraform, Go and Ansible finished =========================" +echo "====================== Installation process for Terraform, Go and Ansible finished =======================" echo "====================================================================================================================" echo "" diff --git a/setup.yml b/setup.yml index 6448c44..6310715 100644 --- a/setup.yml +++ b/setup.yml @@ -162,8 +162,7 @@ - role: hadoop.hdfs.common - role: hadoop.hdfs.journalnode when: - - ide_ha_setup is defined - - ide_ha_setup + - groups.namenodes | length == 3 - role: hadoop.hdfs.namenode - name: Install and configure HDFS Datanodes diff --git a/terraform/inventory_ha_ide.tpl b/terraform/inventory_ha_ide.tpl index 0101eaa..e88f9c3 100644 --- a/terraform/inventory_ha_ide.tpl +++ b/terraform/inventory_ha_ide.tpl @@ -58,7 +58,6 @@ all: hub1: hub2: vars: - jupyterhub_domain_ip: true hetzner_token: "${hetzner_token}" hetznerdns_token: "${hetznerdns_token}" haproxy_admin_password: "changeit" @@ -66,7 +65,11 @@ all: namenode1: children: master1: + namenode2: + children: master2: + namenode3: + children: master3: namenodes: children: @@ -84,7 +87,11 @@ all: resourcemanager1: children: master1: + resourcemanager2: + children: master2: + resourcemanager3: + children: master3: resourcemanagers: children: @@ -182,7 +189,6 @@ all: vars: realm_password: "changeit" vars: - ide_ha_setup: true molecule_deployment: false custom_inventory_file: false self_signed_certificates: false @@ -197,9 +203,6 @@ all: hdfs_group: "hadoop" yarn_user: "yarn" yarn_group: "hadoop" - journalnode_user: "journalnode" - journalnode_uid: "5006" - journalnode_group: "hadoop" domain: "${domain}" certs_source: - "/opt/letsencrypt/{{ inventory_hostname }}/cert1.pem" @@ -242,4 +245,5 @@ all: realm: "COMMUNITY.LAB" keytab_folder: "/etc/keytabs" hadoop_nameservice: "communitylab" - postgres_host: "{{ groups.hub1[0] }}" + jupyterhub_domain_ip: true + postgres_host: "{{ jupyterhub_domain_ip_address }}" diff --git a/terraform/inventory_non_ha_ide.tpl b/terraform/inventory_non_ha_ide.tpl index bdf63c0..2ffdb24 100644 --- a/terraform/inventory_non_ha_ide.tpl +++ b/terraform/inventory_non_ha_ide.tpl @@ -111,7 +111,6 @@ all: vars: realm_password: "changeit" vars: - ide_ha_setup: false molecule_deployment: false custom_inventory_file: false self_signed_certificates: false