diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eebfa5191..fab937bd2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -124,6 +124,7 @@ jobs: with: distro-slug: macos-12 display-name: macOS 12 + container-slug: macos-12 timeout: 20 runs-on: macos-12 instances: '["stable-3006", "stable-3006-8", "stable-3007", "stable-3007-1", "latest"]' @@ -139,194 +140,44 @@ jobs: with: distro-slug: macos-13 display-name: macOS 13 + container-slug: macos-13 timeout: 20 runs-on: macos-13 instances: '["stable-3006", "stable-3006-8", "stable-3007", "stable-3007-1", "latest"]' - - windows-2022: - name: Windows 2022 + macos-14: + name: macOS 14 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-windows.yml + uses: ./.github/workflows/test-macos.yml needs: - lint - generate-actions-workflow with: - distro-slug: windows-2022 - display-name: Windows 2022 + distro-slug: macos-14 + display-name: macOS 14 + container-slug: macOS 14 timeout: 20 - runs-on: windows-2022 + runs-on: macos-14 instances: '["stable-3006", "stable-3006-8", "stable-3007", "stable-3007-1", "latest"]' - almalinux-8: - name: AlmaLinux 8 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: almalinux-8 - display-name: AlmaLinux 8 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' - - - almalinux-9: - name: AlmaLinux 9 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: almalinux-9 - display-name: AlmaLinux 9 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - - - amazon-2: - name: Amazon 2 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: amazon-2 - display-name: Amazon 2 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' - - - centos-stream9: - name: CentOS Stream 9 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: centos-stream9 - display-name: CentOS Stream 9 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - - - debian-11: - name: Debian 11 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: debian-11 - display-name: Debian 11 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - - - debian-12: - name: Debian 12 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: debian-12 - display-name: Debian 12 - timeout: 20 - instances: '["stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - - - fedora-39: - name: Fedora 39 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: fedora-39 - display-name: Fedora 39 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "latest", "default"]' - - - fedora-40: - name: Fedora 40 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: fedora-40 - display-name: Fedora 40 - timeout: 20 - instances: '["stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - - - opensuse-15: - name: Opensuse 15 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: opensuse-15 - display-name: Opensuse 15 - timeout: 20 - instances: '["latest", "default"]' - - - oraclelinux-8: - name: Oracle Linux 8 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: oraclelinux-8 - display-name: Oracle Linux 8 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' - - - oraclelinux-9: - name: Oracle Linux 9 + windows-2022: + name: Windows 2022 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml + uses: ./.github/workflows/test-windows.yml needs: - lint - generate-actions-workflow with: - distro-slug: oraclelinux-9 - display-name: Oracle Linux 9 + distro-slug: windows-2022 + display-name: Windows 2022 + container-slug: windows-2022 timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - + runs-on: windows-2022 + instances: '["stable-3006", "stable-3006-8", "stable-3007", "stable-3007-1", "latest"]' - photon-4: - name: Photon OS 4 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: photon-4 - display-name: Photon OS 4 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "latest", "default"]' photon-5: @@ -339,6 +190,7 @@ jobs: with: distro-slug: photon-5 display-name: Photon OS 5 + container-slug: systemd-photon-5 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "latest", "default"]' @@ -353,6 +205,7 @@ jobs: with: distro-slug: rockylinux-8 display-name: Rocky Linux 8 + container-slug: systemd-rockylinux-8 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' @@ -367,22 +220,9 @@ jobs: with: distro-slug: rockylinux-9 display-name: Rocky Linux 9 + container-slug: systemd-rockylinux-9 timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - - - ubuntu-2004: - name: Ubuntu 20.04 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: ubuntu-2004 - display-name: Ubuntu 20.04 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' + instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' ubuntu-2204: @@ -395,24 +235,11 @@ jobs: with: distro-slug: ubuntu-2204 display-name: Ubuntu 22.04 + container-slug: systemd-ubuntu-22.04 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - ubuntu-2404: - name: Ubuntu 24.04 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: ubuntu-2404 - display-name: Ubuntu 24.04 - timeout: 20 - instances: '["stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - - set-pipeline-exit-status: # This step is just so we can make github require this step, to pass checks # on a pull request instead of requiring all @@ -423,25 +250,12 @@ jobs: - generate-actions-workflow - macos-12 - macos-13 + - macos-14 - windows-2022 - - almalinux-8 - - almalinux-9 - - amazon-2 - - centos-stream9 - - debian-11 - - debian-12 - - fedora-39 - - fedora-40 - - opensuse-15 - - oraclelinux-8 - - oraclelinux-9 - - photon-4 - photon-5 - rockylinux-8 - rockylinux-9 - - ubuntu-2004 - ubuntu-2204 - - ubuntu-2404 if: always() steps: diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index bfdb47b13..c776a44cc 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -41,61 +41,3 @@ jobs: echo "${MSG}" >> "${GITHUB_STEP_SUMMARY}" echo "requirements-met=true" >> "${GITHUB_OUTPUT}" fi - - update-s3-bucket: - name: Update S3 Bucket - if: ${{ fromJSON(needs.workflow-requirements.outputs.requirements-met) }} - runs-on: - - self-hosted - - linux - - repo-release - needs: - - workflow-requirements - environment: release - - steps: - - uses: actions/checkout@v4 - - - name: Get Salt Project GitHub Actions Bot Environment - run: | - TOKEN=$(curl -sS -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 30") - SPB_ENVIRONMENT=$(curl -sS -f -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/spb:environment) - echo "SPB_ENVIRONMENT=$SPB_ENVIRONMENT" >> "$GITHUB_ENV" - - - name: Setup GnuPG - run: | - sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg - GNUPGHOME="$(mktemp -d -p /run/gpg)" - echo "GNUPGHOME=${GNUPGHOME}" >> "$GITHUB_ENV" - cat < "${GNUPGHOME}/gpg.conf" - batch - no-tty - pinentry-mode loopback - EOF - - - name: Get Secrets - id: get-secrets - env: - SECRETS_KEY: ${{ secrets.SECRETS_KEY }} - run: | - SECRETS_KEY_FILE=$(mktemp /tmp/output.XXXXXXXXXX) - echo "$SECRETS_KEY" > "$SECRETS_KEY_FILE" - aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ - --query SecretString --output text | jq .default_key -r | base64 -d \ - | gpg --passphrase-file "${SECRETS_KEY_FILE}" -d - \ - | gpg --import - - sync - aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ - --query SecretString --output text| jq .default_passphrase -r | base64 -d \ - | gpg --passphrase-file "${SECRETS_KEY_FILE}" -o "${GNUPGHOME}/passphrase" -d - - sync - rm "$SECRETS_KEY_FILE" - echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" - - - name: Install Requirements - run: | - python3 -m pip install -r requirements/release.txt - - - name: Upload Develop to S3 - run: | - tools release s3-publish --key-id 64CBBC8173D76B3F develop diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 935b0f70f..f31d6c470 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -51,11 +51,7 @@ jobs: update-develop: name: Update CHANGELOG.md and bootstrap-salt.sh - runs-on: - - ubuntu-latest - ## - self-hosted - ## - linux - ## - repo-release + runs-on: ubuntu-latest permissions: contents: write # To be able to publish the release environment: release @@ -75,36 +71,6 @@ jobs: python3 -m pip install -r requirements/release.txt pre-commit install --install-hooks - ## - name: Setup GnuPG - ## run: | - ## sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg - ## GNUPGHOME="$(mktemp -d -p /run/gpg)" - ## echo "GNUPGHOME=${GNUPGHOME}" >> "$GITHUB_ENV" - ## cat < "${GNUPGHOME}/gpg.conf" - ## batch - ## no-tty - ## pinentry-mode loopback - ## EOF - - ## - name: Get Secrets - ## id: get-secrets - ## env: - ## SECRETS_KEY: ${{ secrets.SECRETS_KEY }} - ## run: | - ## SECRETS_KEY_FILE=$(mktemp /tmp/output.XXXXXXXXXX) - ## echo "$SECRETS_KEY" > "$SECRETS_KEY_FILE" - ## aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ - ## --query SecretString --output text | jq .default_key -r | base64 -d \ - ## | gpg --passphrase-file "${SECRETS_KEY_FILE}" -d - \ - ## | gpg --import - - ## sync - ## aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ - ## --query SecretString --output text| jq .default_passphrase -r | base64 -d \ - ## | gpg --passphrase-file "${SECRETS_KEY_FILE}" -o "${GNUPGHOME}/passphrase" -d - - ## sync - ## rm "$SECRETS_KEY_FILE" - ## echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" - - name: Configure Git shell: bash run: | @@ -112,8 +78,6 @@ jobs: git config --global user.name "Salt Project Packaging" git config --global user.email saltproject-packaging@vmware.com git config --global commit.gpgsign false - ## git config --global user.signingkey 64CBBC8173D76B3F - ## git config --global commit.gpgsign true - name: Update Repository id: update-repo @@ -151,11 +115,7 @@ jobs: merge-develop-into-stable: name: Merge develop into stable - runs-on: - - ubuntu-latest - ## - self-hosted - ## - linux - ## - repo-release + runs-on: ubuntu-latest needs: - update-develop environment: release @@ -169,36 +129,6 @@ jobs: ssh-key: ${{ secrets.SALT_BOOTSTRAP_RELEASE_KEY }} fetch-depth: 0 - ## - name: Setup GnuPG - ## run: | - ## sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg - ## GNUPGHOME="$(mktemp -d -p /run/gpg)" - ## echo "GNUPGHOME=${GNUPGHOME}" >> "$GITHUB_ENV" - ## cat < "${GNUPGHOME}/gpg.conf" - ## batch - ## no-tty - ## pinentry-mode loopback - ## EOF - - ## - name: Get Secrets - ## id: get-secrets - ## env: - ## SECRETS_KEY: ${{ secrets.SECRETS_KEY }} - ## run: | - ## SECRETS_KEY_FILE=$(mktemp /tmp/output.XXXXXXXXXX) - ## echo "$SECRETS_KEY" > "$SECRETS_KEY_FILE" - ## aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ - ## --query SecretString --output text | jq .default_key -r | base64 -d \ - ## | gpg --passphrase-file "${SECRETS_KEY_FILE}" -d - \ - ## | gpg --import - - ## sync - ## aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ - ## --query SecretString --output text| jq .default_passphrase -r | base64 -d \ - ## | gpg --passphrase-file "${SECRETS_KEY_FILE}" -o "${GNUPGHOME}/passphrase" -d - - ## sync - ## rm "$SECRETS_KEY_FILE" - ## echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" - - name: Configure Git shell: bash run: | @@ -206,8 +136,6 @@ jobs: git config --global user.name "Salt Project Packaging" git config --global user.email saltproject-packaging@vmware.com git config --global commit.gpgsign false - ## git config --global user.signingkey 64CBBC8173D76B3F - ## git config --global commit.gpgsign true - name: Download Release Details uses: actions/download-artifact@v4 @@ -250,10 +178,7 @@ jobs: publish-release: name: Create GitHub Release - runs-on: - - ubuntu-latest - ## - self-hosted - ## - linux + runs-on: ubuntu-latest needs: - merge-develop-into-stable environment: release @@ -299,75 +224,9 @@ jobs: name: release-details failOnError: false - ## update-s3-bucket: - ## name: Update S3 Bucket - ## runs-on: - ## - ubuntu-latest - ## ## - self-hosted - ## ## - linux - ## ## - repo-release - ## needs: - ## - publish-release - ## environment: release - - ## steps: - ## - uses: actions/checkout@v4 - ## with: - ## ref: stable - ## repository: ${{ github.repository }} - ## ssh-key: ${{ secrets.SALT_BOOTSTRAP_RELEASE_KEY }} - - ## - name: Get Salt Project GitHub Actions Bot Environment - ## run: | - ## TOKEN=$(curl -sS -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 30") - ## SPB_ENVIRONMENT=$(curl -sS -f -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/spb:environment) - ## echo "SPB_ENVIRONMENT=$SPB_ENVIRONMENT" >> "$GITHUB_ENV" - - ## ## - name: Setup GnuPG - ## ## run: | - ## ## sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg - ## ## GNUPGHOME="$(mktemp -d -p /run/gpg)" - ## ## echo "GNUPGHOME=${GNUPGHOME}" >> "$GITHUB_ENV" - ## ## cat < "${GNUPGHOME}/gpg.conf" - ## ## batch - ## ## no-tty - ## ## pinentry-mode loopback - ## ## EOF - - ## ## - name: Get Secrets - ## ## id: get-secrets - ## ## env: - ## ## SECRETS_KEY: ${{ secrets.SECRETS_KEY }} - ## ## run: | - ## ## SECRETS_KEY_FILE=$(mktemp /tmp/output.XXXXXXXXXX) - ## ## echo "$SECRETS_KEY" > "$SECRETS_KEY_FILE" - ## ## aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ - ## ## --query SecretString --output text | jq .default_key -r | base64 -d \ - ## ## | gpg --passphrase-file "${SECRETS_KEY_FILE}" -d - \ - ## ## | gpg --import - - ## ## sync - ## ## aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ - ## ## --query SecretString --output text| jq .default_passphrase -r | base64 -d \ - ## ## | gpg --passphrase-file "${SECRETS_KEY_FILE}" -o "${GNUPGHOME}/passphrase" -d - - ## ## sync - ## ## rm "$SECRETS_KEY_FILE" - ## ## echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" - - ## ## - name: Install Requirements - ## ## run: | - ## ## python3 -m pip install -r requirements/release.txt - - ## ## - name: Upload Stable Release to S3 - ## ## run: | - ## ## tools release s3-publish --key-id 64CBBC8173D76B3F stable - update-develop-checksums: name: Update Release Checksums on Develop - runs-on: - - ubuntu-latest - ## - self-hosted - ## - linux - ## - repo-release + runs-on: ubuntu-latest needs: - publish-release environment: release @@ -393,36 +252,6 @@ jobs: repository: ${{ github.repository }} ssh-key: ${{ secrets.SALT_BOOTSTRAP_RELEASE_KEY }} - ## - name: Setup GnuPG - ## run: | - ## sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg - ## GNUPGHOME="$(mktemp -d -p /run/gpg)" - ## echo "GNUPGHOME=${GNUPGHOME}" >> "$GITHUB_ENV" - ## cat < "${GNUPGHOME}/gpg.conf" - ## batch - ## no-tty - ## pinentry-mode loopback - ## EOF - - ## - name: Get Secrets - ## id: get-secrets - ## env: - ## SECRETS_KEY: ${{ secrets.SECRETS_KEY }} - ## run: | - ## SECRETS_KEY_FILE=$(mktemp /tmp/output.XXXXXXXXXX) - ## echo "$SECRETS_KEY" > "$SECRETS_KEY_FILE" - ## aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ - ## --query SecretString --output text | jq .default_key -r | base64 -d \ - ## | gpg --passphrase-file "${SECRETS_KEY_FILE}" -d - \ - ## | gpg --import - - ## sync - ## aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ - ## --query SecretString --output text| jq .default_passphrase -r | base64 -d \ - ## | gpg --passphrase-file "${SECRETS_KEY_FILE}" -o "${GNUPGHOME}/passphrase" -d - - ## sync - ## rm "$SECRETS_KEY_FILE" - ## echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" - - name: Configure Git shell: bash run: | @@ -430,8 +259,6 @@ jobs: git config --global user.name "Salt Project Packaging" git config --global user.email saltproject-packaging@vmware.com git config --global commit.gpgsign false - ## git config --global user.signingkey 64CBBC8173D76B3F - ## git config --global commit.gpgsign true - name: Update Latest Release on README run: | diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index 537d1da5e..8a312c5af 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -6,25 +6,22 @@ os.chdir(os.path.abspath(os.path.dirname(__file__))) +# only test against current containers with systemd +# will add these when they become available with systemd +# "amazonlinux-2", +# "amazonlinux-2023", +# "debian-11", +# "debian-12", +# "debian-13", +# "fedora-40", +# "photon-4", +# "ubuntu-2004", +# "ubuntu-2404", LINUX_DISTROS = [ - "almalinux-8", - "almalinux-9", - "amazon-2", - "centos-stream9", - "debian-11", - "debian-12", - "fedora-39", - "fedora-40", - "opensuse-15", - "oraclelinux-8", - "oraclelinux-9", - "photon-4", "photon-5", "rockylinux-8", "rockylinux-9", - "ubuntu-2004", "ubuntu-2204", - "ubuntu-2404", ] WINDOWS = [ @@ -34,84 +31,78 @@ OSX = [ "macos-12", "macos-13", + "macos-14", ] +# only test against current containers with systemd +# will add these when they become available with systemd +# "amazonlinux-2", +# "amazonlinux-2023", +# "centos-stream9", +# "debian-11", +# "debian-12", +# "debian-13", +# "fedora-40", +# "photon-4", +# "ubuntu-2004", +# "ubuntu-2404", STABLE_DISTROS = [ - "almalinux-8", - "almalinux-9", - "amazon-2", - "centos-stream9", - "debian-11", - "debian-12", - "fedora-39", - "fedora-40", - "opensuse-15", - "oraclelinux-8", - "oraclelinux-9", - "photon-4", "photon-5", "rockylinux-8", "rockylinux-9", - "ubuntu-2004", "ubuntu-2204", - "ubuntu-2404", ] +# only test against current containers with systemd +# will add these when they become available with systemd +# "amazonlinux-2", +# "amazonlinux-2023", +# "debian-11", +# "debian-12", +# "debian-13", +# "fedora-40", +# "photon-4", +# "ubuntu-2004", +# "ubuntu-2404", ONEDIR_DISTROS = [ - "almalinux-8", - "almalinux-9", - "amazon-2", - "centos-stream9", - "debian-11", - "debian-12", - "fedora-39", - "fedora-40", - "oraclelinux-8", - "oraclelinux-9", - "photon-4", "photon-5", "rockylinux-8", "rockylinux-9", - "ubuntu-2004", "ubuntu-2204", - "ubuntu-2404", ] +# only test against current containers with systemd +# will add these when they become available with systemd +# "amazonlinux-2", +# "amazonlinux-2023", +# "debian-12", +# "photon-4", +# "photon-5", +# "rockylinux-9", +# "ubuntu-2404", ONEDIR_RC_DISTROS = [ - "almalinux-9", - "amazon-2", - "centos-stream9", - "debian-12", - "oraclelinux-9", - "photon-4", "photon-5", "rockylinux-9", - "ubuntu-2404", + "ubuntu-2204", ] BLACKLIST_3006 = [ "debian-12", "fedora-40", - "opensuse-15", "ubuntu-2404", ] BLACKLIST_3007 = [ - "fedora-39", - "opensuse-15", "photon-4", "photon-5", ] BLACKLIST_GIT_3006 = [ - "almalinux-9", - "amazon-2", - "centos-stream9", + "amazonlinux-2", + "amazonlinux-2023", "debian-11", "debian-12", "fedora-40", - "opensuse-15", - "oraclelinux-9", "photon-4", "photon-5", "rockylinux-9", @@ -121,15 +112,12 @@ ] BLACKLIST_GIT_3007 = [ - "almalinux-9", - "amazon-2", - "centos-stream9", + "amazonlinux-2", + "amazonlinux-2023", "debian-11", "debian-12", - "fedora-39", + "debian-13", "fedora-40", - "opensuse-15", - "oraclelinux-9", "photon-4", "photon-5", "rockylinux-9", @@ -139,10 +127,11 @@ ] BLACKLIST_GIT_MASTER = [ - "amazon-2", - "fedora-39", + "amazonlinux-2", + "amazonlinux-2023", "photon-4", "photon-5", + "rockylinux-9", ] SALT_VERSIONS = [ @@ -199,27 +188,18 @@ # Use build and pip and other standards-based tools. # GIT_DISTRO_BLACKLIST = [ - "almalinux-8", - "fedora-39", - "opensuse-15", - "oraclelinux-8", "rockylinux-8", ] LATEST_PKG_BLACKLIST = [] DISTRO_DISPLAY_NAMES = { - "almalinux-8": "AlmaLinux 8", - "almalinux-9": "AlmaLinux 9", - "amazon-2": "Amazon 2", - "centos-stream9": "CentOS Stream 9", + "amazonlinux-2": "Amazon 2", + "amazonlinux-2023": "Amazon 2023", "debian-11": "Debian 11", "debian-12": "Debian 12", - "fedora-39": "Fedora 39", + "debian-13": "Debian 13", "fedora-40": "Fedora 40", - "opensuse-15": "Opensuse 15", - "oraclelinux-8": "Oracle Linux 8", - "oraclelinux-9": "Oracle Linux 9", "photon-4": "Photon OS 4", "photon-5": "Photon OS 5", "rockylinux-8": "Rocky Linux 8", @@ -229,9 +209,30 @@ "ubuntu-2404": "Ubuntu 24.04", "macos-12": "macOS 12", "macos-13": "macOS 13", + "macos-14": "macOS 14", "windows-2022": "Windows 2022", } +CONTAINER_SLUG_NAMES = { + "amazonlinux-2": "systemd-amazonlinux-2", + "amazonlinux-2023": "systemd-amazonlinux-2023", + "debian-11": "systemd-debian-11", + "debian-12": "systemd-debian-12", + "debian-13": "systemd-debian-13", + "fedora-40": "systemd-fedora-40", + "photon-4": "systemd-photon-4", + "photon-5": "systemd-photon-5", + "rockylinux-8": "systemd-rockylinux-8", + "rockylinux-9": "systemd-rockylinux-9", + "ubuntu-2004": "systemd-ubuntu-20.04", + "ubuntu-2204": "systemd-ubuntu-22.04", + "ubuntu-2404": "systemd-ubuntu-24.04", + "macos-12": "macos-12", + "macos-13": "macos-13", + "macos-14": "macOS 14", + "windows-2022": "windows-2022", +} + TIMEOUT_DEFAULT = 20 TIMEOUT_OVERRIDES = {} VERSION_ONLY_OVERRIDES = [] @@ -246,6 +247,7 @@ with: distro-slug: {distro} display-name: {display_name} + container-slug: {container_name} timeout: {timeout_minutes}{runs_on} instances: '{instances}' """ @@ -296,6 +298,7 @@ def generate_test_jobs(): ifcheck=ifcheck, instances=json.dumps(instances), display_name=DISTRO_DISPLAY_NAMES[distro], + container_name=CONTAINER_SLUG_NAMES[distro], timeout_minutes=timeout_minutes, ) @@ -341,6 +344,7 @@ def generate_test_jobs(): ifcheck=ifcheck, instances=json.dumps(instances), display_name=DISTRO_DISPLAY_NAMES[distro], + container_name=CONTAINER_SLUG_NAMES[distro], timeout_minutes=timeout_minutes, ) @@ -440,6 +444,7 @@ def generate_test_jobs(): ifcheck=ifcheck, instances=json.dumps(instances), display_name=DISTRO_DISPLAY_NAMES[distro], + container_name=CONTAINER_SLUG_NAMES[distro], timeout_minutes=timeout_minutes, ) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index d31ef758f..120f23eca 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -17,6 +17,11 @@ on: required: true description: The Instances To Test + container-slug: + type: string + required: true + description: The Container Slug + timeout: type: number required: false @@ -26,10 +31,10 @@ on: jobs: Test: name: ${{ matrix.instance }} - ## runs-on: ubuntu-20.04 runs-on: ubuntu-latest container: - image: ghcr.io/saltstack/salt-ci-containers/packaging:ubuntu-22.04 + image: ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} + options: --privileged timeout-minutes: ${{ inputs.timeout }} strategy: @@ -41,18 +46,36 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Install Pytest + - name: Install Python Dependencies with pip breakage + if: ${{ ( inputs.distro-slug == 'debian-11' ) || ( inputs.distro-slug == 'debian-12' ) || ( inputs.distro-slug == 'debian-13' ) || ( inputs.distro-slug == 'ubuntu-2404' ) }} run: | - pip install -U pytest + python3 -m pip install --break-system-packages -r tests/requirements.txt - ## TBD Disabled until CI/CD is functional - ## - name: Bootstrap Salt - ## run: | - ## sh -x ./bootstrap-salt.sh + - name: Install Python Dependencies without pip breakage + if: ${{ ( inputs.distro-slug != 'debian-11' ) && ( inputs.distro-slug != 'debian-12' ) && ( inputs.distro-slug != 'debian-13' ) && ( inputs.distro-slug != 'ubuntu-2404' ) }} + run: | + python3 -m pip install -r tests/requirements.txt - ## - name: Test Bootstrap - ## run: | - ## pytest --cache-clear -v -s -ra --log-cli-level=debug tests/integration/ + - name: Get Version + run: | + # We need to get the version here and make it an environment variable + # It is used to install via bootstrap and in the test + # The version is in the instance name + # sed 1st - becomes space, 2nd - becomes dot + vt_parm_ver=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./' | awk -F ' ' '{print $2}') + echo "SaltVersion=$vt_parm_ver" >> $GITHUB_ENV + + - name: Bootstrap Salt + run: | + # sed 1st - becomes space, 2nd - becomes dot + bt_arg1=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./' | awk -F ' ' '{print $1}') + bt_arg2=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./' | awk -F ' ' '{print $2}') + echo "bt parms ,$bt_parms, bt_arg1 ,$bt_arg1, bt_arg2 ,$bt_arg2," + sh -x ./bootstrap-salt.sh "$bt_arg1" "$bt_arg2" + + - name: Test Bootstrap + run: | + pytest --cache-clear -v -s -ra --log-cli-level=debug tests/integration/ - name: Set Exit Status if: always() diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index 504f8b3e7..870fb5965 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -22,6 +22,11 @@ on: required: true description: The Instances To Test + container-slug: + type: string + required: true + description: The Container Slug + timeout: type: number required: false @@ -31,8 +36,8 @@ on: jobs: Test: name: ${{ matrix.instance }} - ## runs-on: ${{ inputs.runs-on }} - runs-on: macos-13 + runs-on: ${{ inputs.runs-on }} + ## runs-on: macos-13 timeout-minutes: ${{ inputs.timeout }} strategy: fail-fast: false @@ -42,18 +47,37 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Install Pytest + - name: Install Python Dependencies with pip breakage + if: ${{ ( inputs.distro-slug != 'macos-12' ) && ( inputs.distro-slug != 'macos-13' ) }} + run: | + python3 -m pip install --break-system-packages -r tests/requirements.txt + + - name: Install Python Dependencies without pip breakage + if: ${{ ( inputs.distro-slug == 'macos-12' ) || ( inputs.distro-slug == 'macos-13' ) }} + run: | + python3 -m pip install -r tests/requirements.txt + + - name: Get Version run: | - pip install -U pytest + # We need to get the version here and make it an environment variable + # It is used to install via bootstrap and in the test + # The version is in the instance name + # sed 1st - becomes space, 2nd - becomes dot + vt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./') + vt_parm_ver=$(echo "$vt_parms" | awk -F ' ' '{print $2}') + echo "SaltVersion=$vt_parm_ver" >> $GITHUB_ENV - name: Bootstrap Salt run: | - sudo sh -x ./bootstrap-salt.sh + # sed 1st - becomes space, 2nd - becomes dot + bt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./') + bt_arg1=$(echo "$bt_parms" | awk -F ' ' '{print $1}') + bt_arg2=$(echo "$bt_parms" | awk -F ' ' '{print $2}') + sudo sh ./bootstrap-salt.sh "$bt_arg1" "$bt_arg2" - ## TBD Disabled until CI/CD is functional - ## - name: Test Bootstrap - ## run: | - ## pytest --cache-clear -v -s -ra --log-cli-level=debug tests/integration/ + - name: Test Bootstrap + run: | + pytest --cache-clear -v -s -ra --log-cli-level=debug tests/integration/ - name: Set Exit Status if: always() diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index e4d4dde34..9515dae8d 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -22,6 +22,11 @@ on: required: true description: The Instances To Test + container-slug: + type: string + required: true + description: The Container Slug + timeout: type: number required: false @@ -46,9 +51,9 @@ jobs: with: python-version: "3.10" - - name: Install Pytest + - name: Install Python Dependencies run: | - pip install -r tests\requirements.txt + python3 -m pip install -r tests/requirements.txt - name: Get Version run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 725022f71..2a70b9fa1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,17 @@ +# v2024.11.21 + +## What's Changed + +- Make universal repo url by @twangboy in https://github.com/saltstack/salt-bootstrap/pull/2045 +- Update bootstrap shell CI/CD to utilize containers and clean up by @dmurphy18 in https://github.com/saltstack/salt-bootstrap/pull/2039 +- \[WIP\] hwclock check missing additional condition on major version by @ruslantum in https://github.com/saltstack/salt-bootstrap/pull/2048 + +## New Contributors + +- @ruslantum made their first contribution in https://github.com/saltstack/salt-bootstrap/pull/2048 + +**Full Changelog**: https://github.com/saltstack/salt-bootstrap/compare/v2024.11.07...v2024.11.21 + # v2024.11.07 ## What's Changed diff --git a/README.rst b/README.rst index aa7609d33..7c524e3a7 100644 --- a/README.rst +++ b/README.rst @@ -37,6 +37,7 @@ sum** of the downloaded ``bootstrap-salt.sh`` file. The SHA256 sum of the ``bootstrap-salt.sh`` file, per release, is: +- 2024.11.07: ``70a9783649e129985563d1a86cf28b8984499643e62ae1dc47dc008bd204fcbb`` - 2024.09.24: ``88e4e4cad4b115a7b721dd9c21d5ee5df390b5b73b63de48f99399146f43f371`` - 2024.07.23: ``7212b6b497b5c3d2bf15bfe5301625ec7bc1bf3e2949cd47d8e2073614935bf8`` - 2024.07.18: ``92a74e7ff8a9032a7713c2b3955991d66aaca08a4eb9494ce3dd66b5044f6bc3`` @@ -413,21 +414,19 @@ Debian and derivatives - Devuan GNU/Linux 1/2/3/4/5 - Kali Linux 1.0 (based on Debian 7) - Linux Mint Debian Edition 1 (based on Debian 8) -- Raspbian 8 (``armhf`` packages) and 9 (using ``git`` installation mode only) Red Hat family ~~~~~~~~~~~~~~ -- Amazon Linux 2012.3 and later - Amazon Linux 2 - Amazon Linux 2023 -- CentOS 7/8/9 +- CentOS 8/9 - Cloud Linux 6/7 -- Fedora 38/39/40 (install latest stable from standard repositories) -- Oracle Linux 7/8/9 -- Red Hat Enterprise Linux 7/8/9 -- Scientific Linux 7/8/9 +- Fedora 40 (install latest stable from standard repositories) +- Oracle Linux 8/9 +- Red Hat Enterprise Linux 8/9 +- Scientific Linux 8/9 SUSE family @@ -460,8 +459,8 @@ repositories are not provided on the `Salt Project's Ubuntu repository`_ for the the bootstrap script will attempt to install the packages for the most closely related LTS Ubuntu release instead. -For example, when installing Salt on Ubuntu 21.10, the bootstrap script will setup the repository -for Ubuntu 20.04 from the `Salt Project's Ubuntu repository`_ and install the 20.04 packages. +For example, when installing Salt on Ubuntu 24.10, the bootstrap script will setup the repository +for Ubuntu 24.04 from the `Salt Project's Ubuntu repository`_ and install the 24.04 packages. Non-LTS Ubuntu releases are not supported once the release reaches End-of-Life as defined by `Ubuntu's release schedule`_. diff --git a/bootstrap-salt.ps1 b/bootstrap-salt.ps1 index 85223d371..815a0eb74 100644 --- a/bootstrap-salt.ps1 +++ b/bootstrap-salt.ps1 @@ -48,7 +48,6 @@ [CmdletBinding()] param( [Parameter(Mandatory=$false, ValueFromPipeline=$True)] - [ValidatePattern('^(\d{4}(\.\d{1,2}){0,2}(\-\d{1})?)|(latest)$', Options=1)] [Alias("v")] # The version of the Salt minion to install. Default is "latest" which will # install the latest version of Salt minion available. Doesn't support @@ -108,7 +107,7 @@ if ($help) { exit 0 } -$__ScriptVersion = "2024.11.07" +$__ScriptVersion = "2024.11.21" $ScriptName = $myInvocation.MyCommand.Name # We'll check for the Version next, because it also has no requirements @@ -153,34 +152,116 @@ function Get-MajorVersion { return ( $Version -split "\." )[0] } -function Convert-PSObjectToHashtable { - param ( - [Parameter(ValueFromPipeline)] - $InputObject - ) - if ($null -eq $InputObject) { return $null } - - $is_enum = $InputObject -is [System.Collections.IEnumerable] - $not_string = $InputObject -isnot [string] - if ($is_enum -and $not_string) { - $collection = @( - foreach ($object in $InputObject) { - Convert-PSObjectToHashtable $object - } - ) +function Get-AvailableVersions { + # Get available versions from a remote location specified in the Source + # Parameter + Write-Verbose "Getting version information from the repo" + Write-Verbose "base_url: $base_url" + + $available_versions = [System.Collections.ArrayList]@() + + if ( $base_url.StartsWith("http") -or $base_url.StartsWith("ftp") ) { + # We're dealing with HTTP, HTTPS, or FTP + $response = Invoke-WebRequest "$base_url" -UseBasicParsing + try { + $response = Invoke-WebRequest "$base_url" -UseBasicParsing + } catch { + Write-Host "Failed to get version information" -ForegroundColor Red + exit 1 + } - Write-Host -NoEnumerate $collection - } elseif ($InputObject -is [PSObject]) { - $hash = @{} + if ( $response.StatusCode -ne 200 ) { + Write-Host "There was an error getting version information" -ForegroundColor Red + Write-Host "Error: $($response.StatusCode)" -ForegroundColor red + exit 1 + } - foreach ($property in $InputObject.PSObject.Properties) { - $hash[$property.Name] = Convert-PSObjectToHashtable $property.Value + $response.links | ForEach-Object { + if ( $_.href.Length -gt 8) { + Write-Host "The content at this location is unexpected" -ForegroundColor Red + Write-Host "Should be a list of directories where the name is a version of Salt" -ForegroundColor Red + exit 1 + } } - $hash + # Getting available versions from response + Write-Verbose "Getting available versions from response" + $filtered = $response.Links | Where-Object -Property href -NE "../" + $filtered | Select-Object -Property href | ForEach-Object { + $available_versions.Add($_.href.Trim("/")) | Out-Null + } + } elseif ( $base_url.StartsWith("\\") -or $base_url -match "^[A-Za-z]:\\" ) { + # We're dealing with a local directory or SMB source + Get-ChildItem -Path $base_url -Directory | ForEach-Object { + $available_versions.Add($_.Name) | Out-Null + } } else { - $InputObject + Write-Host "Unknown Source Type" -ForegroundColor Red + Write-Host "Must be one of HTTP, HTTPS, FTP, SMB Share, Local Directory" -ForegroundColor Red + exit 1 + } + + Write-Verbose "Available versions:" + $available_versions | ForEach-Object { + Write-Verbose "- $_" + } + + # Get the latest version, should be the last in the list + Write-Verbose "Getting latest available version" + $latest = $available_versions | Select-Object -Last 1 + Write-Verbose "Latest available version: $latest" + + # Create a versions table + # This will have the latest version available, the latest version available + # for each major version, and every version available. This makes the + # version lookup logic easier. The contents of the versions table can be + # found by running -Verbose + Write-Verbose "Populating the versions table" + $versions_table = [ordered]@{"latest"=$latest} + $available_versions | ForEach-Object { + $versions_table[$(Get-MajorVersion $_)] = $_ + $versions_table[$_.ToLower()] = $_.ToLower() + } + + Write-Verbose "Versions Table:" + $versions_table | Sort-Object Name | Out-String | ForEach-Object { + Write-Verbose "$_" + } + + return $versions_table +} + +function Get-HashFromArtifactory { + # This function uses the artifactory API to get the SHA265 Hash for the file + # If Source is NOT artifactory, the sha will not be checked + [CmdletBinding()] + param( + [Parameter(Mandatory=$true)] + [String] $SaltVersion, + + [Parameter(Mandatory=$true)] + [String] $SaltFileName + ) + if ( $api_url ) { + $full_url = "$api_url/$SaltVersion/$SaltFileName" + Write-Verbose "Querying Artifactory API for hash:" + Write-Verbose $full_url + try { + $response = Invoke-RestMethod $full_url -UseBasicParsing + return $response.checksums.sha256 + } catch { + Write-Verbose "Artifactory API Not available or file not" + Write-Verbose "available at specified location" + Write-Verbose "Hash will not be checked" + return "" + } + Write-Verbose "No hash found for this file: $SaltFileName" + Write-Verbose "Hash will not be checked" + return "" } + Write-Verbose "No artifactory API defined" + Write-Verbose "Hash will not be checked" + return "" } function Get-FileHash { @@ -313,10 +394,8 @@ if ($majorVersion -lt "3006") { #=============================================================================== $ConfDir = "$RootDir\conf" $PkiDir = "$ConfDir\pki\minion" -$RootDir = "$env:ProgramData\Salt Project\Salt" -$DfltUrl = "https://packages.broadcom.com/artifactory/saltproject-generic/windows/" -$ApiUrl = "https://packages.broadcom.com/artifactory/api/storage/saltproject-generic/windows" +$RootDir = "$env:ProgramData\Salt Project\Salt" # Check for existing installation where RootDir is stored in the registry $SaltRegKey = "HKLM:\SOFTWARE\Salt Project\Salt" if (Test-Path -Path $SaltRegKey) { @@ -325,6 +404,18 @@ if (Test-Path -Path $SaltRegKey) { } } +# Get repo and api URLs. An artifactory URL will have "artifactory" in it +$domain, $target = $RepoUrl -split "/artifactory/" +if ( $target ) { + # Create $base_url and $api_url + $base_url = "$domain/artifactory/$target" + $api_url = "$domain/artifactory/api/storage/$target" +} else { + # This is a non-artifactory url, there is no api + $base_url = $domain + $api_url = "" +} + #=============================================================================== # Verify Parameters #=============================================================================== @@ -335,8 +426,8 @@ Write-Verbose "version: $Version" Write-Verbose "runservice: $RunService" Write-Verbose "master: $Master" Write-Verbose "minion: $Minion" -Write-Verbose "repourl: $RepoUrl" -Write-Verbose "apiurl: $ApiUrl" +Write-Verbose "repourl: $base_url" +Write-Verbose "apiurl: $api_url" Write-Verbose "ConfDir: $ConfDir" Write-Verbose "RootDir: $RootDir" @@ -393,90 +484,34 @@ if ( $ConfigureOnly ) { #=============================================================================== # Detect architecture #=============================================================================== -if ([IntPtr]::Size -eq 4) { - $arch = "x86" -} else { - $arch = "AMD64" -} +if ([IntPtr]::Size -eq 4) { $arch = "x86" } else { $arch = "AMD64" } #=============================================================================== # Getting version information from the repo #=============================================================================== -if ( $RepoUrl -eq $DfltUrl ) { - Write-Verbose "Getting version information from Artifactory" - $response = Invoke-WebRequest $ApiUrl -UseBasicParsing - # Convert the output to a powershell object - $psobj = $response.ToString() | ConvertFrom-Json - - # Filter the object for folders - $filtered = $psobj.children | Where-Object -Property folder -EQ $true - - # Get each uri and add it to the list of versions - $available_versions = [System.Collections.ArrayList]@() - $filtered | Select-Object -Property uri | ForEach-Object { - $available_versions.Add($_.uri.Trim("/")) | Out-Null - } +$versions = Get-AvailableVersions - # Create a versions table, similar to repo.json - # This will have the latest version available, the latest version available for - # each major version, and every version available. This makes the version - # lookup logic easier. You can view the contents of the versions table by - # passing the -Verbose command - $latest = $available_versions | Select-Object -Last 1 - $versions_table = [ordered]@{"latest"=$latest} - - $available_versions | ForEach-Object { - $versions_table[$(Get-MajorVersion $_)] = $_ - $versions_table[$_.ToLower()] = $_.ToLower() - } - - Write-Verbose "Available versions:" - $available_versions | ForEach-Object { - Write-Verbose "- $_" - } - Write-Verbose "Versions Table:" - $versions_table | Sort-Object Name | Out-String | Write-Verbose - - #=============================================================================== - # Validate passed version - #=============================================================================== - if ( $versions_table.Contains($Version.ToLower()) ) { - $Version = $versions_table[$Version.ToLower()] - } else { - Write-Host "Version $Version is not available" -ForegroundColor Red - Write-Host "Available versions are:" -ForegroundColor Yellow - $available_versions | ForEach-Object { Write-Host "- $_" -ForegroundColor Yellow } - exit 1 - } - - #=============================================================================== - # Get file url and sha256 - #=============================================================================== - $saltFileName = "Salt-Minion-$Version-Py3-$arch-Setup.exe" - $response = Invoke-WebRequest "$ApiUrl/$Version/$saltFileName" -UseBasicParsing - $psobj = $response.ToString() | ConvertFrom-Json - $saltFileUrl = $psobj.downloadUri - $saltSha256 = $psobj.checksums.sha256 - - if ( $saltFileName -and $saltVersion -and $saltSha256) { - Write-Verbose "Found Name, Version, and Sha" - } else { - # We will guess the name of the installer - Write-Verbose "Failed to get Name, Version, and Sha from Artifactory API" - Write-Verbose "We'll try to find the file in standard paths" - $saltFileName = "Salt-Minion-$Version-Py3-$arch-Setup.exe" - $saltVersion = $Version - } +#=============================================================================== +# Validate passed version +#=============================================================================== +Write-Verbose "Looking up version: $Version" +if ( $versions.Contains($Version.ToLower()) ) { + $Version = $versions[$Version.ToLower()] + Write-Verbose "Found version: $Version" } else { - # If we're using a custom RepoUrl, we're going to assum that the binary is - # in the reoot of the RepoUrl/Version. We will not check the sha on custom - # repos - $saltFileName = "Salt-Minion-$Version-Py3-$arch-Setup.exe" - $saltFileUrl = "$RepoUrl/$Version/$saltFileName" - $saltVersion = $Version - $saltSha256 = "" + Write-Host "Version $Version is not available" -ForegroundColor Red + Write-Host "Available versions are:" -ForegroundColor Yellow + $versions + exit 1 } +#=============================================================================== +# Get file url and sha256 +#=============================================================================== +$saltFileName = "Salt-Minion-$Version-Py3-$arch-Setup.exe" +$saltFileUrl = "$base_url/$Version/$saltFileName" +$saltSha256 = Get-HashFromArtifactory -SaltVersion $Version -SaltFileName $saltFileName + #=============================================================================== # Download minion setup file #=============================================================================== @@ -484,7 +519,7 @@ Write-Host "==================================================================== Write-Host " Bootstrapping Salt Minion" -ForegroundColor Green Write-Host " - version: $Version" Write-Host " - file name: $saltFileName" -Write-Host " - file url: $saltFileUrl" +Write-Host " - file url : $saltFileUrl" Write-Host " - file hash: $saltSha256" Write-Host " - master: $Master" Write-Host " - minion id: $Minion" @@ -498,9 +533,11 @@ Write-Verbose "" Write-Verbose "Salt File URL: $saltFileUrl" Write-Verbose "Local File: $localFile" -if ( Test-Path -Path $localFile ) {Remove-Item -Path $localFile -Force} -Invoke-WebRequest -Uri $saltFileUrl -OutFile $localFile +# Remove existing local file +if ( Test-Path -Path $localFile ) { Remove-Item -Path $localFile -Force } +# Download the file +Invoke-WebRequest -Uri $saltFileUrl -OutFile $localFile if ( Test-Path -Path $localFile ) { Write-Host "Success" -ForegroundColor Green } else { @@ -508,6 +545,7 @@ if ( Test-Path -Path $localFile ) { exit 1 } +# Compare the hash if there is a hash to compare if ( $saltSha256 ) { $localSha256 = (Get-FileHash -Path $localFile -Algorithm SHA256).Hash Write-Host "Comparing Hash: " -NoNewline @@ -546,13 +584,13 @@ $process = Start-Process $localFile ` -NoNewWindow -PassThru # Sometimes the installer hangs... we'll wait 5 minutes and then kill it -Write-Verbose "" Write-Verbose "Waiting for installer to finish" $process | Wait-Process -Timeout 300 -ErrorAction SilentlyContinue $process.Refresh() if ( !$process.HasExited ) { - Write-Host "Installer Timeout" -ForegroundColor Yellow + Write-Verbose "Installer Timeout" + Write-Host "" Write-Host "Killing hung installer: " -NoNewline $process | Stop-Process $process.Refresh() diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index e6b25f2d3..390ee7b5c 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -26,7 +26,7 @@ #====================================================================================================================== set -o nounset # Treat unset variables as an error -__ScriptVersion="2024.11.07" +__ScriptVersion="2024.11.21" __ScriptName="bootstrap-salt.sh" __ScriptFullName="$0" @@ -221,6 +221,7 @@ __check_config_dir() { # DESCRIPTION: Checks the placed after the install arguments #---------------------------------------------------------------------------------------------------------------------- __check_unparsed_options() { + shellopts="$1" # grep alternative for SunOS if [ -f /usr/xpg4/bin/grep ]; then @@ -618,13 +619,24 @@ if [ "$#" -gt 0 ];then fi # Check installation type -if [ "$(echo "$ITYPE" | grep -E '(stable|testing|git|onedir|onedir_rc)')" = "" ]; then +if [ "$(echo "$ITYPE" | grep -E '(latest|default|stable|testing|git|onedir|onedir_rc)')" = "" ]; then echoerror "Installation type \"$ITYPE\" is not known..." exit 1 fi +## allows GitHub Actions CI/CD easier handling of latest and default +if [ "$ITYPE" = "latest" ] || [ "$ITYPE" = "default" ]; then + STABLE_REV="latest" + ONEDIR_REV="latest" + _ONEDIR_REV="latest" + ITYPE="onedir" + if [ "$#" -gt 0 ];then + shift + fi + echodebug "using ITYPE onedir for input 'latest' or 'default', cmd args left ,$#," + # If doing a git install, check what branch/tag/sha will be checked out -if [ "$ITYPE" = "git" ]; then +elif [ "$ITYPE" = "git" ]; then if [ "$#" -eq 0 ];then GIT_REV="master" else @@ -649,7 +661,7 @@ elif [ "$ITYPE" = "stable" ]; then _ONEDIR_REV="$1" ITYPE="onedir" shift - elif [ "$(echo "$1" | grep -E '^([3-9][0-5]{2}[5-9](\.[0-9]*)?)')" != "" ]; then + elif [ "$(echo "$1" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then STABLE_REV="$1" ONEDIR_REV="$1" _ONEDIR_REV="$1" @@ -849,7 +861,7 @@ if [ "$ITYPE" != "git" ]; then fi fi -# Set the _REPO_URL value based on if -R was passed or not. Defaults to packages.broadcom.com +# Set the _REPO_URL value based on if -R was passed or not. Defaults to packages.broadcom.com/artifactory if [ "$_CUSTOM_REPO_URL" != "null" ]; then _REPO_URL="$_CUSTOM_REPO_URL" @@ -920,6 +932,7 @@ fi # DESCRIPTION: Retrieves a URL and writes it to a given path #---------------------------------------------------------------------------------------------------------------------- __fetch_url() { + # shellcheck disable=SC2086 curl $_CURL_ARGS -L -s -f -o "$1" "$2" >/dev/null 2>&1 || wget $_WGET_ARGS -q -O "$1" "$2" >/dev/null 2>&1 || @@ -1924,6 +1937,7 @@ __function_defined() { # process is finished so the script doesn't exit on a locked proc. #---------------------------------------------------------------------------------------------------------------------- __wait_for_apt(){ + # Timeout set at 15 minutes WAIT_TIMEOUT=900 @@ -2003,7 +2017,6 @@ __temp_gpg_pub() { #---------------------------------------------------------------------------------------------------------------------- __apt_key_fetch() { - url=$1 tempfile="$(__temp_gpg_pub)" @@ -2437,10 +2450,12 @@ __check_services_systemd() { _SYSTEMD_ACTIVE=$(/bin/systemctl daemon-reload 2>&1 | grep 'System has not been booted with systemd') echodebug "__check_services_systemd _SYSTEMD_ACTIVE result ,$_SYSTEMD_ACTIVE," - if [ "$_SYSTEMD_ACTIVE" != "" ]; then + if [ -n "$_SYSTEMD_ACTIVE" ]; then _SYSTEMD_FUNCTIONAL=$BS_FALSE echodebug "systemd is not functional, despite systemctl being present, setting _SYSTEMD_FUNCTIONAL false, $_SYSTEMD_FUNCTIONAL" return 1 + else + echodebug "systemd is functional, _SYSTEMD_FUNCTIONAL true, $_SYSTEMD_FUNCTIONAL" fi servicename=$1 @@ -2611,6 +2626,7 @@ __activate_virtualenv() { #---------------------------------------------------------------------------------------------------------------------- __install_pip_pkgs() { + _pip_pkgs="$1" _py_exe="$2" _py_pkg=$(echo "$_py_exe" | sed -E "s/\\.//g") @@ -2632,8 +2648,10 @@ __install_pip_pkgs() { else __PACKAGES="${__PACKAGES} ${_py_pkg}-devel" if [ "$DISTRO_NAME_L" = "fedora" ];then + dnf makecache || return 1 __dnf_install_noinput ${__PACKAGES} || return 1 else + yum makecache || return 1 __yum_install_noinput ${__PACKAGES} || return 1 fi fi @@ -2652,6 +2670,7 @@ __install_pip_pkgs() { # PARAMETERS: requirements_file #---------------------------------------------------------------------------------------------------------------------- __install_pip_deps() { + # Install virtualenv to system pip before activating virtualenv if thats going to be used # We assume pip pkg is installed since that is distro specific if [ "$_VIRTUALENV_DIR" != "null" ]; then @@ -2794,7 +2813,6 @@ EOM rm -f /tmp/git/deps/* echoinfo "Building Salt Python Wheel" - if [ "$_ECHO_DEBUG" -eq $BS_TRUE ]; then SETUP_PY_INSTALL_ARGS="-v" fi @@ -2937,6 +2955,8 @@ fi # __enable_universe_repository() { + echodebug "__enable_universe_repository() entry" + if [ "$(grep -R universe /etc/apt/sources.list /etc/apt/sources.list.d/ | grep -v '#')" != "" ]; then # The universe repository is already enabled return 0 @@ -2950,6 +2970,7 @@ __enable_universe_repository() { } __install_saltstack_ubuntu_repository() { + # Workaround for latest non-LTS Ubuntu echodebug "__install_saltstack_ubuntu_repository() entry" @@ -2993,14 +3014,13 @@ __install_saltstack_ubuntu_repository() { if [ "$STABLE_REV" != "latest" ]; then # latest is default - STABLE_REV_MAJOR=$(echo "$STABLE_REV" | cut -d '.' -f 1) - if [ "$STABLE_REV_MAJOR" -eq "3006" ]; then + if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - echo "Pin: version 3006.*" >> /etc/apt/preferences.d/salt-pin-1001 + echo "Pin: version $STABLE_REV.*" >> /etc/apt/preferences.d/salt-pin-1001 echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 - elif [ "$STABLE_REV_MAJOR" -eq "3007" ]; then + elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - echo "Pin: version 3007.*" >> /etc/apt/preferences.d/salt-pin-1001 + echo "Pin: version $STABLE_REV" >> /etc/apt/preferences.d/salt-pin-1001 echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 fi fi @@ -3010,6 +3030,7 @@ __install_saltstack_ubuntu_repository() { __install_saltstack_ubuntu_onedir_repository() { echodebug "__install_saltstack_ubuntu_onedir_repository() entry" + # Workaround for latest non-LTS Ubuntu if { [ "$DISTRO_MAJOR_VERSION" -eq 20 ] && [ "$DISTRO_MINOR_VERSION" -eq 10 ]; } || \ { [ "$DISTRO_MAJOR_VERSION" -eq 22 ] && [ "$DISTRO_MINOR_VERSION" -eq 10 ]; } || \ @@ -3031,7 +3052,7 @@ __install_saltstack_ubuntu_onedir_repository() { fi ## include hwclock if not part of base OS (23.10 and up) - if [ ! -f /usr/sbin/hwclock ]; then + if [ "$DISTRO_MAJOR_VERSION" -ge 23 ] && [ ! -f /usr/sbin/hwclock ]; then __PACKAGES="${__PACKAGES} util-linux-extra" fi @@ -3045,14 +3066,13 @@ __install_saltstack_ubuntu_onedir_repository() { if [ "$ONEDIR_REV" != "latest" ]; then # latest is default - ONEDIR_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) - if [ "$ONEDIR_REV_MAJOR" -eq "3006" ]; then + if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - echo "Pin: version 3006.*" >> /etc/apt/preferences.d/salt-pin-1001 + echo "Pin: version $ONEDIR_REV.*" >> /etc/apt/preferences.d/salt-pin-1001 echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 - elif [ "$ONEDIR_REV_MAJOR" -eq "3007" ]; then + elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - echo "Pin: version 3007.*" >> /etc/apt/preferences.d/salt-pin-1001 + echo "Pin: version $ONEDIR_REV" >> /etc/apt/preferences.d/salt-pin-1001 echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 fi fi @@ -3099,6 +3119,9 @@ install_ubuntu_deps() { # Additionally install procps and pciutils which allows for Docker bootstraps. See 366#issuecomment-39666813 __PACKAGES="${__PACKAGES} procps pciutils" + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + ## include hwclock if not part of base OS (23.10 and up) if [ ! -f /usr/sbin/hwclock ]; then __PACKAGES="${__PACKAGES} util-linux-extra" @@ -3182,6 +3205,9 @@ install_ubuntu_git_deps() { __PACKAGES="${__PACKAGES} util-linux-extra" fi + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + # shellcheck disable=SC2086 __apt_get_install_noinput ${__PACKAGES} || return 1 @@ -3229,6 +3255,8 @@ install_ubuntu_onedir_deps() { install_ubuntu_stable() { + __wait_for_apt apt-get update || return 1 + __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then @@ -3286,6 +3314,8 @@ install_ubuntu_git() { install_ubuntu_onedir() { + __wait_for_apt apt-get update || return 1 + __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then @@ -3322,7 +3352,6 @@ install_ubuntu_stable_post() { [ $fname = "master" ] && [ "$_INSTALL_MASTER" -eq $BS_FALSE ] && continue [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue - ## if [ -f /bin/systemctl ]; then if [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ]; then # Using systemd /bin/systemctl is-enabled salt-$fname.service > /dev/null 2>&1 || ( @@ -3480,25 +3509,19 @@ __install_saltstack_debian_repository() { # shellcheck disable=SC2086,SC2090 __apt_get_install_noinput ${__PACKAGES} || return 1 - ## SALTSTACK_DEBIAN_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/debian/${DEBIAN_RELEASE}/${__REPO_ARCH}/${STABLE_REV}" - ## echo "$__REPO_ARCH_DEB $SALTSTACK_DEBIAN_URL $DEBIAN_CODENAME main" > "/etc/apt/sources.list.d/salt.list" - ## __apt_key_fetch "$SALTSTACK_DEBIAN_URL/SALT-PROJECT-GPG-PUBKEY-2023.gpg" || return 1 - ## __wait_for_apt apt-get update || return 1 - __fetch_url "/etc/apt/sources.list.d/salt.sources" "https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.sources" __apt_key_fetch "${HTTP_VAL}://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" || return 1 __wait_for_apt apt-get update || return 1 if [ "$STABLE_REV" != "latest" ]; then # latest is default - STABLE_REV_MAJOR=$(echo "$STABLE_REV" | cut -d '.' -f 1) - if [ "$STABLE_REV_MAJOR" -eq "3006" ]; then + if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - echo "Pin: version 3006.*" >> /etc/apt/preferences.d/salt-pin-1001 + echo "Pin: version $STABLE_REV.*" >> /etc/apt/preferences.d/salt-pin-1001 echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 - elif [ "$STABLE_REV_MAJOR" -eq "3007" ]; then + elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - echo "Pin: version 3007.*" >> /etc/apt/preferences.d/salt-pin-1001 + echo "Pin: version $STABLE_REV" >> /etc/apt/preferences.d/salt-pin-1001 echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 fi fi @@ -3535,14 +3558,13 @@ __install_saltstack_debian_onedir_repository() { if [ "$ONEDIR_REV" != "latest" ]; then # latest is default - ONEDIR_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) - if [ "$ONEDIR_REV_MAJOR" -eq "3006" ]; then + if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - echo "Pin: version 3006.*" >> /etc/apt/preferences.d/salt-pin-1001 + echo "Pin: version $ONEDIR_REV.*" >> /etc/apt/preferences.d/salt-pin-1001 echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 - elif [ "$ONEDIR_REV_MAJOR" -eq "3007" ]; then + elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - echo "Pin: version 3007.*" >> /etc/apt/preferences.d/salt-pin-1001 + echo "Pin: version $ONEDIR_REV" >> /etc/apt/preferences.d/salt-pin-1001 echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 fi fi @@ -3586,6 +3608,9 @@ install_debian_onedir_deps() { # YAML module is used for generating custom master/minion configs __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-yaml" + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + # shellcheck disable=SC2086 __apt_get_install_noinput ${__PACKAGES} || return 1 @@ -3627,6 +3652,9 @@ install_debian_git_deps() { __PACKAGES="python${PY_PKG_VER}-dev python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc" echodebug "install_debian_git_deps() Installing ${__PACKAGES}" + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + # shellcheck disable=SC2086 __apt_get_install_noinput ${__PACKAGES} || return 1 @@ -3641,6 +3669,8 @@ install_debian_git_deps() { install_debian_stable() { + __wait_for_apt apt-get update || return 1 + __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then @@ -3720,6 +3750,8 @@ install_debian_12_git() { install_debian_onedir() { + __wait_for_apt apt-get update || return 1 + __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then @@ -3862,12 +3894,27 @@ __install_saltstack_fedora_onedir_repository() { FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" if [ "$ONEDIR_REV" != "latest" ]; then - # 3006.x is default - REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) - if [ "$REPO_REV_MAJOR" -eq "3007" ]; then - # Enable the Salt 3007 STS repo - dnf config-manager --set-disable salt-repo-* - dnf config-manager --set-enabled salt-repo-3007-sts + # 3006.x is default, and latest for 3006.x branch + if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # latest version for branch 3006 | 3007 + REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) + if [ "$REPO_REV_MAJOR" -eq "3007" ]; then + # Enable the Salt 3007 STS repo + dnf config-manager --set-disable salt-repo-* + dnf config-manager --set-enabled salt-repo-3007-sts + fi + elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # using minor version + echo "[salt-repo-${ONEDIR_REV}-lts]" > "${YUM_REPO_FILE}" + # shellcheck disable=SC2129 + echo "name=Salt Repo for Salt v${ONEDIR_REV} LTS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" fi else # Enable the Salt LATEST repo @@ -3875,6 +3922,7 @@ __install_saltstack_fedora_onedir_repository() { dnf config-manager --set-enabled salt-repo-latest fi dnf clean expire-cache || return 1 + dnf makecache || return 1 elif [ "$ONEDIR_REV" != "latest" ]; then echowarn "salt.repo already exists, ignoring salt version argument." @@ -3909,6 +3957,9 @@ install_fedora_deps() { echoinfo "Installing the following extra packages as requested: ${_EXTRA_PACKAGES}" fi + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + # shellcheck disable=SC2086 __dnf_install_noinput ${__PACKAGES} ${_EXTRA_PACKAGES} || return 1 @@ -3939,6 +3990,10 @@ install_fedora_git_deps() { __git_clone_and_checkout || return 1 __PACKAGES="python${PY_PKG_VER}-devel python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc gcc-c++" + + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + # shellcheck disable=SC2086 __dnf_install_noinput ${__PACKAGES} || return 1 @@ -4068,6 +4123,9 @@ install_fedora_onedir_deps() { __PACKAGES="dnf-utils chkconfig procps-ng" + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -4086,27 +4144,37 @@ install_fedora_onedir() { STABLE_REV=$ONEDIR_REV #install_fedora_stable || return 1 + if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # Major version Salt, config and repo already setup + MINOR_VER_STRG="" + elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # Minor version Salt, need to add specific minor version + MINOR_VER_STRG="-$STABLE_REV" + else + MINOR_VER_STRG="" + fi __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-cloud" + __PACKAGES="${__PACKAGES} salt-cloud$MINOR_VER_STRG" fi if [ "$_INSTALL_MASTER" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-master" + __PACKAGES="${__PACKAGES} salt-master$MINOR_VER_STRG" fi if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-minion" + __PACKAGES="${__PACKAGES} salt-minion$MINOR_VER_STRG" fi if [ "$_INSTALL_SYNDIC" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-syndic" + __PACKAGES="${__PACKAGES} salt-syndic$MINOR_VER_STRG" fi if [ "$_INSTALL_SALT_API" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-api" + __PACKAGES="${__PACKAGES} salt-api$MINOR_VER_STRG" fi # shellcheck disable=SC2086 + dnf makecache || return 1 __yum_install_noinput ${__PACKAGES} || return 1 return 0 @@ -4153,12 +4221,27 @@ __install_saltstack_rhel_onedir_repository() { FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" if [ "$ONEDIR_REV" != "latest" ]; then - # 3006.x is default - REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) - if [ "$REPO_REV_MAJOR" -eq "3007" ]; then - # Enable the Salt 3007 STS repo - dnf config-manager --set-disable salt-repo-* - dnf config-manager --set-enabled salt-repo-3007-sts + # 3006.x is default, and latest for 3006.x branch + if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # latest version for branch 3006 | 3007 + REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) + if [ "$REPO_REV_MAJOR" -eq "3007" ]; then + # Enable the Salt 3007 STS repo + dnf config-manager --set-disable salt-repo-* + dnf config-manager --set-enabled salt-repo-3007-sts + fi + elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # using minor version + echo "[salt-repo-${ONEDIR_REV}-lts]" > "${YUM_REPO_FILE}" + # shellcheck disable=SC2129 + echo "name=Salt Repo for Salt v${ONEDIR_REV} LTS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" fi else # Enable the Salt LATEST repo @@ -4166,6 +4249,7 @@ __install_saltstack_rhel_onedir_repository() { dnf config-manager --set-enabled salt-repo-latest fi dnf clean expire-cache || return 1 + dnf makecache || return 1 elif [ "$ONEDIR_REV" != "latest" ]; then echowarn "salt.repo already exists, ignoring salt version argument." echowarn "Use -F (forced overwrite) to install $ONEDIR_REV." @@ -4200,6 +4284,9 @@ install_centos_stable_deps() { __PACKAGES="yum-utils chkconfig procps-ng findutils" + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -4214,26 +4301,40 @@ install_centos_stable_deps() { install_centos_stable() { + if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # Major version Salt, config and repo already setup + MINOR_VER_STRG="" + elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # Minor version Salt, need to add specific minor version + MINOR_VER_STRG="-$STABLE_REV" + else + MINOR_VER_STRG="" + fi + __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-cloud" + __PACKAGES="${__PACKAGES} salt-cloud$MINOR_VER_STRG" fi if [ "$_INSTALL_MASTER" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-master" + __PACKAGES="${__PACKAGES} salt-master$MINOR_VER_STRG" fi if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-minion" + __PACKAGES="${__PACKAGES} salt-minion$MINOR_VER_STRG" fi if [ "$_INSTALL_SYNDIC" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-syndic" + __PACKAGES="${__PACKAGES} salt-syndic$MINOR_VER_STRG" fi if [ "$_INSTALL_SALT_API" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-api" + __PACKAGES="${__PACKAGES} salt-api$MINOR_VER_STRG" fi + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + # shellcheck disable=SC2086 + yum makecache || return 1 __yum_install_noinput ${__PACKAGES} || return 1 # Workaround for 3.11 broken on CentOS Stream 8.x @@ -4311,6 +4412,9 @@ install_centos_git_deps() { __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-devel python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc" + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -4407,6 +4511,9 @@ install_centos_onedir_deps() { __PACKAGES="yum-utils chkconfig procps-ng findutils" + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -4421,26 +4528,38 @@ install_centos_onedir_deps() { install_centos_onedir() { + if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # Major version Salt, config and repo already setup + MINOR_VER_STRG="" + elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # Minor version Salt, need to add specific minor version + MINOR_VER_STRG="-$ONEDIR_REV" + else + MINOR_VER_STRG="" + fi + __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-cloud" + __PACKAGES="${__PACKAGES} salt-cloud$MINOR_VER_STRG" fi if [ "$_INSTALL_MASTER" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-master" + __PACKAGES="${__PACKAGES} salt-master$MINOR_VER_STRG" fi if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-minion" + __PACKAGES="${__PACKAGES} salt-minion$MINOR_VER_STRG" fi if [ "$_INSTALL_SYNDIC" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-syndic" + __PACKAGES="${__PACKAGES} salt-syndic$MINOR_VER_STRG" fi if [ "$_INSTALL_SALT_API" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-api" + __PACKAGES="${__PACKAGES} salt-api$MINOR_VER_STRG" fi # shellcheck disable=SC2086 + dnf makecache || return 1 + dnf list salt-minion || return 1 __yum_install_noinput ${__PACKAGES} || return 1 return 0 @@ -4992,79 +5111,79 @@ install_oracle_linux_check_services() { ####################################################################################################################### # -# AlmaLinux Install Functions +# RockyLinux Install Functions # -install_almalinux_stable_deps() { +install_rockylinux_stable_deps() { install_centos_stable_deps || return 1 return 0 } -install_almalinux_git_deps() { +install_rockylinux_git_deps() { install_centos_git_deps || return 1 return 0 } -install_almalinux_onedir_deps() { +install_rockylinux_onedir_deps() { install_centos_onedir_deps || return 1 return 0 } -install_almalinux_testing_deps() { +install_rockylinux_testing_deps() { install_centos_testing_deps || return 1 return 0 } -install_almalinux_stable() { +install_rockylinux_stable() { install_centos_stable || return 1 return 0 } -install_almalinux_git() { +install_rockylinux_git() { install_centos_git || return 1 return 0 } -install_almalinux_onedir() { +install_rockylinux_onedir() { install_centos_onedir || return 1 return 0 } -install_almalinux_testing() { +install_rockylinux_testing() { install_centos_testing || return 1 return 0 } -install_almalinux_stable_post() { +install_rockylinux_stable_post() { install_centos_stable_post || return 1 return 0 } -install_almalinux_git_post() { +install_rockylinux_git_post() { install_centos_git_post || return 1 return 0 } -install_almalinux_onedir_post() { +install_rockylinux_onedir_post() { install_centos_onedir_post || return 1 return 0 } -install_almalinux_testing_post() { +install_rockylinux_testing_post() { install_centos_testing_post || return 1 return 0 } -install_almalinux_restart_daemons() { +install_rockylinux_restart_daemons() { install_centos_restart_daemons || return 1 return 0 } -install_almalinux_check_services() { +install_rockylinux_check_services() { install_centos_check_services || return 1 return 0 } # -# Ended AlmaLinux Install Functions +# Ended RockyLinux Install Functions # ####################################################################################################################### @@ -5493,6 +5612,9 @@ install_amazon_linux_ami_2_git_deps() { __PACKAGES="python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools python${PY_PKG_VER}-devel gcc" + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -5513,7 +5635,12 @@ install_amazon_linux_ami_2_deps() { # We need to install yum-utils before doing anything else when installing on # Amazon Linux ECS-optimized images. See issue #974. - __yum_install_noinput yum-utils + __PACKAGES="yum-utils" + + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + + __yum_install_noinput ${__PACKAGES} # Do upgrade early if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then @@ -5525,12 +5652,27 @@ install_amazon_linux_ami_2_deps() { FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" if [ "$STABLE_REV" != "latest" ]; then - # 3006.x is default - REPO_REV_MAJOR=$(echo "$STABLE_REV" | cut -d '.' -f 1) - if [ "$REPO_REV_MAJOR" -eq "3007" ]; then - # Enable the Salt 3007 STS repo - dnf config-manager --set-disable salt-repo-* - dnf config-manager --set-enabled salt-repo-3007-sts + # 3006.x is default, and latest for 3006.x branch + if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # latest version for branch 3006 | 3007 + REPO_REV_MAJOR=$(echo "$STABLE_REV" | cut -d '.' -f 1) + if [ "$REPO_REV_MAJOR" -eq "3007" ]; then + # Enable the Salt 3007 STS repo + dnf config-manager --set-disable salt-repo-* + dnf config-manager --set-enabled salt-repo-3007-sts + fi + elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # using minor version + echo "[salt-repo-${STABLE_REV}-lts]" > "${YUM_REPO_FILE}" + # shellcheck disable=SC2129 + echo "name=Salt Repo for Salt v${STABLE_REV} LTS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" fi else # Enable the Salt LATEST repo @@ -5538,6 +5680,7 @@ install_amazon_linux_ami_2_deps() { dnf config-manager --set-enabled salt-repo-latest fi dnf clean expire-cache || return 1 + dnf makecache || return 1 fi fi @@ -5556,7 +5699,12 @@ install_amazon_linux_ami_2_onedir_deps() { # We need to install yum-utils before doing anything else when installing on # Amazon Linux ECS-optimized images. See issue #974. - __yum_install_noinput yum-utils + __PACKAGES="yum-utils" + + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + + __yum_install_noinput ${__PACKAGES} # Do upgrade early if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then @@ -5568,12 +5716,27 @@ install_amazon_linux_ami_2_onedir_deps() { FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" if [ "$ONEDIR_REV" != "latest" ]; then - # 3006.x is default - REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) - if [ "$REPO_REV_MAJOR" -eq "3007" ]; then - # Enable the Salt 3007 STS repo - dnf config-manager --set-disable salt-repo-* - dnf config-manager --set-enabled salt-repo-3007-sts + # 3006.x is default, and latest for 3006.x branch + if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # latest version for branch 3006 | 3007 + REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) + if [ "$REPO_REV_MAJOR" -eq "3007" ]; then + # Enable the Salt 3007 STS repo + dnf config-manager --set-disable salt-repo-* + dnf config-manager --set-enabled salt-repo-3007-sts + fi + elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # using minor version + echo "[salt-repo-${ONEDIR_REV}-lts]" > "${YUM_REPO_FILE}" + # shellcheck disable=SC2129 + echo "name=Salt Repo for Salt v${ONEDIR_REV} LTS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" fi else # Enable the Salt LATEST repo @@ -5581,6 +5744,7 @@ install_amazon_linux_ami_2_onedir_deps() { dnf config-manager --set-enabled salt-repo-latest fi dnf clean expire-cache || return 1 + dnf makecache || return 1 fi fi @@ -5667,6 +5831,9 @@ install_amazon_linux_ami_2023_git_deps() { __PACKAGES="python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools python${PY_PKG_VER}-devel gcc" + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -5683,7 +5850,12 @@ install_amazon_linux_ami_2023_onedir_deps() { # We need to install yum-utils before doing anything else when installing on # Amazon Linux ECS-optimized images. See issue #974. - __yum_install_noinput yum-utils + __PACKAGES="yum-utils" + + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + + __yum_install_noinput ${__PACKAGES} # Do upgrade early if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then @@ -5695,12 +5867,27 @@ install_amazon_linux_ami_2023_onedir_deps() { FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" if [ "$ONEDIR_REV" != "latest" ]; then - # 3006.x is default - REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) - if [ "$REPO_REV_MAJOR" -eq "3007" ]; then - # Enable the Salt 3007 STS repo - dnf config-manager --set-disable salt-repo-* - dnf config-manager --set-enabled salt-repo-3007-sts + # 3006.x is default, and latest for 3006.x branch + if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # latest version for branch 3006 | 3007 + REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) + if [ "$REPO_REV_MAJOR" -eq "3007" ]; then + # Enable the Salt 3007 STS repo + dnf config-manager --set-disable salt-repo-* + dnf config-manager --set-enabled salt-repo-3007-sts + fi + elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # using minor version + echo "[salt-repo-${ONEDIR_REV}-lts]" > "${YUM_REPO_FILE}" + # shellcheck disable=SC2129 + echo "name=Salt Repo for Salt v${ONEDIR_REV} LTS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" fi else # Enable the Salt LATEST repo @@ -5708,6 +5895,7 @@ install_amazon_linux_ami_2023_onedir_deps() { dnf config-manager --set-enabled salt-repo-latest fi dnf clean expire-cache || return 1 + dnf makecache || return 1 fi fi @@ -6024,6 +6212,37 @@ install_arch_linux_onedir_post() { # Photon OS Install Functions # +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: __rpm_get_packagesite_onedir_latest +# DESCRIPTION: Set _GENERIC_PKG_VERSION to the latest for RPM or latest for major version input +#---------------------------------------------------------------------------------------------------------------------- +__get_packagesite_onedir_latest() { + + echodebug "Find latest rpm release from repository" + + # get dir listing from url, sort and pick highest + generic_versions_tmpdir=$(mktemp -d) + curr_pwd=$(pwd) + cd ${generic_versions_tmpdir} || return 1 + + # leverage the windows directories since release Windows and Linux + wget -q -r -np -nH --exclude-directories=onedir,relenv,macos -x -l 1 "https://${_REPO_URL}/saltproject-generic/windows/" + if [ "$#" -gt 0 ] && [ -n "$1" ]; then + MAJOR_VER="$1" + # shellcheck disable=SC2010 + _GENERIC_PKG_VERSION=$(ls artifactory/saltproject-generic/windows/ | grep -v 'index.html' | sort -V -u | grep -E "$MAJOR_VER" | tail -n 1) + else + # shellcheck disable=SC2010 + _GENERIC_PKG_VERSION=$(ls artifactory/saltproject-generic/windows/ | grep -v 'index.html' | sort -V -u | tail -n 1) + fi + cd ${curr_pwd} || return "${_GENERIC_PKG_VERSION}" + rm -fR ${generic_versions_tmpdir} + + echodebug "latest rpm release from repository found ${_GENERIC_PKG_VERSION}" + +} + + __install_saltstack_photon_onedir_repository() { echodebug "__install_saltstack_photon_onedir_repository() entry" @@ -6038,34 +6257,48 @@ __install_saltstack_photon_onedir_repository() { ## __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" # shellcheck disable=SC2129 if [ "$ONEDIR_REV" != "latest" ]; then - # 3006.x is default - REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) - if [ "$REPO_REV_MAJOR" -eq "3007" ]; then - # Enable the Salt 3007 STS repo - ## tdnf config-manager --set-disable salt-repo-* - ## tdnf config-manager --set-enabled salt-repo-3007-sts - echo "[salt-repo-3007-sts]" > "${YUM_REPO_FILE}" - echo "name=Salt Repo for Salt v3007 STS" >> "${YUM_REPO_FILE}" - echo "baseurl=https://packages.broadcom.com/artifactory/saltproject-rpm/" >> "${YUM_REPO_FILE}" - echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" - echo "priority=10" >> "${YUM_REPO_FILE}" - echo "enabled=1" >> "${YUM_REPO_FILE}" - echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" - echo "gpgcheck=1" >> "${YUM_REPO_FILE}" - echo "exclude=*3006* *3008* *3009* *3010*" >> "${YUM_REPO_FILE}" - echo "gpgkey=https://packages.broadcom.com/artifactory/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" - else - # Salt 3006 repo - echo "[salt-repo-3006-lts]" > "${YUM_REPO_FILE}" - echo "name=Salt Repo for Salt v3006 LTS" >> "${YUM_REPO_FILE}" - echo "baseurl=https://packages.broadcom.com/artifactory/saltproject-rpm/" >> "${YUM_REPO_FILE}" + # 3006.x is default, and latest for 3006.x branch + if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # latest version for branch 3006 | 3007 + REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) + if [ "$REPO_REV_MAJOR" -eq "3007" ]; then + # Enable the Salt 3007 STS repo + ## tdnf config-manager --set-disable salt-repo-* + ## tdnf config-manager --set-enabled salt-repo-3007-sts + echo "[salt-repo-3007-sts]" > "${YUM_REPO_FILE}" + echo "name=Salt Repo for Salt v3007 STS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "exclude=*3006* *3008* *3009* *3010*" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" + else + # Salt 3006 repo + echo "[salt-repo-3006-lts]" > "${YUM_REPO_FILE}" + echo "name=Salt Repo for Salt v3006 LTS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "exclude=*3007* *3008* *3009* *3010*" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" + fi + elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # using minor version + echo "[salt-repo-${ONEDIR_REV}-lts]" > "${YUM_REPO_FILE}" + echo "name=Salt Repo for Salt v${ONEDIR_REV} LTS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" echo "priority=10" >> "${YUM_REPO_FILE}" echo "enabled=1" >> "${YUM_REPO_FILE}" echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" echo "gpgcheck=1" >> "${YUM_REPO_FILE}" - echo "exclude=*3007* *3008* *3009* *3010*" >> "${YUM_REPO_FILE}" - echo "gpgkey=https://packages.broadcom.com/artifactory/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" fi else # Enable the Salt LATEST repo @@ -6073,13 +6306,13 @@ __install_saltstack_photon_onedir_repository() { ## tdnf config-manager --set-enabled salt-repo-latest echo "[salt-repo-latest]" > "${YUM_REPO_FILE}" echo "name=Salt Repo for Salt LATEST release" >> "${YUM_REPO_FILE}" - echo "baseurl=https://packages.broadcom.com/artifactory/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" echo "priority=10" >> "${YUM_REPO_FILE}" echo "enabled=1" >> "${YUM_REPO_FILE}" echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" echo "gpgcheck=1" >> "${YUM_REPO_FILE}" - echo "gpgkey=https://packages.broadcom.com/artifactory/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" fi tdnf makecache || return 1 elif [ "$ONEDIR_REV" != "latest" ]; then @@ -6330,27 +6563,42 @@ install_photon_onedir_deps() { install_photon_onedir() { + echodebug "install_photon_onedir() entry" STABLE_REV=$ONEDIR_REV + _GENERIC_PKG_VERSION="" + + if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # Major version Salt, config and repo already setup + __get_packagesite_onedir_latest "$STABLE_REV" + MINOR_VER_STRG="-$_GENERIC_PKG_VERSION" + elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # Minor version Salt, need to add specific minor version + MINOR_VER_STRG="-$STABLE_REV" + else + # default to latest version Salt, config and repo already setup + __get_packagesite_onedir_latest + MINOR_VER_STRG="-$_GENERIC_PKG_VERSION" + fi __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-cloud" + __PACKAGES="${__PACKAGES} salt-cloud$MINOR_VER_STRG" fi if [ "$_INSTALL_MASTER" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-master" + __PACKAGES="${__PACKAGES} salt-master$MINOR_VER_STRG" fi if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-minion" + __PACKAGES="${__PACKAGES} salt-minion$MINOR_VER_STRG" fi if [ "$_INSTALL_SYNDIC" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-syndic" + __PACKAGES="${__PACKAGES} salt-syndic$MINOR_VER_STRG" fi if [ "$_INSTALL_SALT_API" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-api" + __PACKAGES="${__PACKAGES} salt-api$MINOR_VER_STRG" fi # shellcheck disable=SC2086 @@ -6534,23 +6782,33 @@ install_opensuse_onedir_deps() { } install_opensuse_stable() { + if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # Major version Salt, config and repo already setup + MINOR_VER_STRG="" + elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # Minor version Salt, need to add specific minor version + MINOR_VER_STRG="-$STABLE_REV" + else + MINOR_VER_STRG="" + fi + __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-cloud" + __PACKAGES="${__PACKAGES} salt-cloud$MINOR_VER_STRG" fi if [ "$_INSTALL_MASTER" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-master" + __PACKAGES="${__PACKAGES} salt-master$MINOR_VER_STRG" fi if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-minion" + __PACKAGES="${__PACKAGES} salt-minion$MINOR_VER_STRG" fi if [ "$_INSTALL_SYNDIC" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-syndic" + __PACKAGES="${__PACKAGES} salt-syndic$MINOR_VER_STRG" fi if [ "$_INSTALL_SALT_API" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-api" + __PACKAGES="${__PACKAGES} salt-api$MINOR_VER_STRG" fi # shellcheck disable=SC2086 @@ -7257,7 +7515,7 @@ daemons_running_voidlinux() { #--- FUNCTION ------------------------------------------------------------------------------------------------------- # NAME: __macosx_get_packagesite_onedir_latest -# DESCRIPTION: Set _PKG_VERSION to the latest for MacOS +# DESCRIPTION: Set _PKG_VERSION to the latest for MacOS or latest for major version input #---------------------------------------------------------------------------------------------------------------------- __macosx_get_packagesite_onedir_latest() { @@ -7267,9 +7525,15 @@ __macosx_get_packagesite_onedir_latest() { macos_versions_tmpdir=$(mktemp -d) curr_pwd=$(pwd) cd ${macos_versions_tmpdir} || return 1 - wget -r -np -nH --exclude-directories=onedir,relenv,windows -x -l 1 "$SALT_MACOS_PKGDIR_URL/" - # shellcheck disable=SC2010 - _PKG_VERSION=$(ls artifactory/saltproject-generic/macos/ | grep -v 'index.html' | sort -V -u | tail -n 1) + wget -q -r -np -nH --exclude-directories=onedir,relenv,windows -x -l 1 "$SALT_MACOS_PKGDIR_URL/" + if [ "$#" -gt 0 ] && [ -n "$1" ]; then + MAJOR_VER="$1" + # shellcheck disable=SC2010 + _PKG_VERSION=$(ls artifactory/saltproject-generic/macos/ | grep -v 'index.html' | sort -V -u | grep -E "$MAJOR_VER" | tail -n 1) + else + # shellcheck disable=SC2010 + _PKG_VERSION=$(ls artifactory/saltproject-generic/macos/ | grep -v 'index.html' | sort -V -u | tail -n 1) + fi cd ${curr_pwd} || return "${_PKG_VERSION}" rm -fR ${macos_versions_tmpdir} @@ -7293,11 +7557,15 @@ __macosx_get_packagesite_onedir() { _ONEDIR_TYPE="saltproject-generic" SALT_MACOS_PKGDIR_URL="https://${_REPO_URL}/${_ONEDIR_TYPE}/macos" if [ "$(echo "$_ONEDIR_REV" | grep -E '^(latest)$')" != "" ]; then - __macosx_get_packagesite_onedir_latest - elif [ "$(echo "$_ONEDIR_REV" | grep -E '^([3-9][0-9]{3}(\.[0-9]*))')" != "" ]; then - _PKG_VERSION=$_ONEDIR_REV + __macosx_get_packagesite_onedir_latest + elif [ "$(echo "$_ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # need to get latest for major version + __macosx_get_packagesite_onedir_latest "$_ONEDIR_REV" + elif [ "$(echo "$_ONEDIR_REV" | grep -E '^([3-9][0-9]{3}(\.[0-9]*)?)')" != "" ]; then + _PKG_VERSION=$_ONEDIR_REV else - __macosx_get_packagesite_onedir_latest + # default to getting latest + __macosx_get_packagesite_onedir_latest fi PKG="salt-${_PKG_VERSION}-py3-${DARWIN_ARCH}.pkg" diff --git a/tests/conftest.py b/tests/conftest.py index 154b257c0..5d5702c12 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,7 +18,6 @@ def target_python_version(): def target_salt_version(): target_salt = os.environ.get("SaltVersion", "") - html_response = requests.get(API_URL) content = json.loads(html_response.text) folders = content["children"] @@ -36,6 +35,14 @@ def target_salt_version(): target_salt = target_salt[1:] if target_salt not in versions: pytest.skip(f"Invalid testing version: {target_salt}") - if target_salt in ("default", "latest", "master", "nightly"): + if target_salt in ( + "default", + "latest", + "master", + "nightly", + "stable", + "onedir", + "git", + ): pytest.skip("Don't have a specific salt version to test against") return versions[target_salt] diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index f8427b675..714b8e166 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -15,7 +15,9 @@ def path(): salt_path = "C:\\Program Files\\Salt Project\\Salt" if salt_path not in os.environ["path"]: os.environ["path"] = f'{os.environ["path"]};{salt_path}' - yield os.environ["path"] + yield os.environ["path"] + else: + yield "" def run_salt_call(cmd): @@ -23,9 +25,33 @@ def run_salt_call(cmd): Runs salt call command and returns a dictionary Accepts cmd as a list """ - cmd.append("--out=json") - result = subprocess.run(cmd, capture_output=True, text=True) - json_data = json.loads(result.stdout) + json_data = {"local": {}} + if platform.system() == "Windows": + cmd.append("--out=json") + result = subprocess.run(cmd, capture_output=True, text=True) + print( + f"DGM run_salt_call, cmd '{cmd}', result '{result}', stdout '{result.stdout}'", + flush=True, + ) + if 0 == result.returncode: + json_data = json.loads(result.stdout) + else: + log.error(f"failed to produce output result, '{result}'") + + else: + cmdl = ["sudo"] + cmdl.extend(cmd) + cmdl.append("--out=json") + result = subprocess.run(cmdl, capture_output=True, text=True) + print( + f"DGM run_salt_call, cmdl '{cmdl}', result '{result}', stdout '{result.stdout}'", + flush=True, + ) + if 0 == result.returncode: + json_data = json.loads(result.stdout) + else: + log.error(f"failed to produce output result, '{result}'") + return json_data["local"] @@ -48,5 +74,10 @@ def test_target_salt_version(path, target_salt_version): pytest.skip(f"No target version specified") cmd = ["salt-call", "--local", "grains.item", "saltversion", "--timeout=120"] result = run_salt_call(cmd) + dgm_saltversion = result["saltversion"] + print( + f"DGM test_target_salt_version, target_salt_version '{target_salt_version}', result saltversion '{dgm_saltversion }', result '{result}'", + flush=True, + ) # Returns: {'saltversion': '3006.9+217.g53cfa53040'} - assert result["saltversion"] == target_salt_version \ No newline at end of file + assert result["saltversion"] == target_salt_version