diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 000000000..093a4fd41 --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,4 @@ +self-hosted-runner: + # Labels of self-hosted runner in array of string + labels: + - repo-release diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0e01c958d..e81e9b3d1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -173,7 +173,7 @@ jobs: display-name: macOS 10.15 timeout: 20 runs-on: macos-10.15 - instances: '["stable-3003", "stable-3004", "stable-3005", "latest"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "stable-3006", "latest"]' macos-11: @@ -188,7 +188,7 @@ jobs: display-name: macOS 11 timeout: 20 runs-on: macos-11 - instances: '["stable-3003", "stable-3004", "stable-3005", "latest"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "stable-3006", "latest"]' macos-12: @@ -203,7 +203,7 @@ jobs: display-name: macOS 12 timeout: 20 runs-on: macos-12 - instances: '["stable-3003", "stable-3004", "stable-3005", "latest"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "stable-3006", "latest"]' @@ -219,7 +219,7 @@ jobs: display-name: Windows 2019 timeout: 20 runs-on: windows-2019 - instances: '["stable-3003", "stable-3004", "stable-3005", "latest"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "stable-3006", "latest"]' windows-2022: @@ -234,7 +234,7 @@ jobs: display-name: Windows 2022 timeout: 20 runs-on: windows-2022 - instances: '["stable-3003", "stable-3004", "stable-3005", "latest"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "stable-3006", "latest"]' @@ -249,7 +249,7 @@ jobs: distro-slug: almalinux-8 display-name: AlmaLinux 8 timeout: 20 - instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "latest", "onedir-nightly", "onedir-rc-3006-0rc1", "onedir-rc-3006-0rc2"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "latest"]' almalinux-9: @@ -263,7 +263,7 @@ jobs: distro-slug: almalinux-9 display-name: AlmaLinux 9 timeout: 20 - instances: '["git-3005", "onedir-3005", "git-master", "onedir-nightly", "onedir-rc-3006-0rc1", "onedir-rc-3006-0rc2"]' + instances: '["git-3005", "onedir-3005", "onedir-3006", "git-master"]' amazon-2: @@ -277,7 +277,7 @@ jobs: distro-slug: amazon-2 display-name: Amazon 2 timeout: 20 - instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "git-master", "latest", "onedir-nightly", "onedir-rc-3006-0rc1", "onedir-rc-3006-0rc2"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "git-master", "latest"]' arch: @@ -305,7 +305,7 @@ jobs: distro-slug: centos-7 display-name: CentOS 7 timeout: 20 - instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "latest", "onedir-nightly", "onedir-rc-3006-0rc1", "onedir-rc-3006-0rc2"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "latest"]' centos-stream8: @@ -319,7 +319,7 @@ jobs: distro-slug: centos-stream8 display-name: CentOS Stream 8 timeout: 20 - instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "latest", "onedir-nightly", "onedir-rc-3006-0rc1", "onedir-rc-3006-0rc2"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "latest"]' centos-stream9: @@ -333,7 +333,7 @@ jobs: distro-slug: centos-stream9 display-name: CentOS Stream 9 timeout: 20 - instances: '["git-3005", "onedir-3005", "git-master", "onedir-nightly", "onedir-rc-3006-0rc1", "onedir-rc-3006-0rc2"]' + instances: '["git-3005", "onedir-3005", "onedir-3006", "git-master"]' debian-10: @@ -347,7 +347,7 @@ jobs: distro-slug: debian-10 display-name: Debian 10 timeout: 20 - instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "git-master", "latest", "onedir-nightly", "onedir-rc-3006-0rc1", "onedir-rc-3006-0rc2"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "git-master", "latest"]' debian-11: @@ -361,7 +361,7 @@ jobs: distro-slug: debian-11 display-name: Debian 11 timeout: 20 - instances: '["stable-3004", "stable-3005", "onedir-3005", "git-master", "latest", "onedir-nightly", "onedir-rc-3006-0rc1", "onedir-rc-3006-0rc2"]' + instances: '["stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "git-master", "latest"]' fedora-35: @@ -487,7 +487,7 @@ jobs: distro-slug: oraclelinux-7 display-name: Oracle Linux 7 timeout: 20 - instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "latest", "onedir-nightly", "onedir-rc-3006-0rc1", "onedir-rc-3006-0rc2"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "latest"]' oraclelinux-8: @@ -501,7 +501,7 @@ jobs: distro-slug: oraclelinux-8 display-name: Oracle Linux 8 timeout: 20 - instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "latest", "onedir-nightly", "onedir-rc-3006-0rc1", "onedir-rc-3006-0rc2"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "latest"]' rockylinux-8: @@ -515,7 +515,7 @@ jobs: distro-slug: rockylinux-8 display-name: Rocky Linux 8 timeout: 20 - instances: '["stable-3004", "stable-3005", "onedir-3005", "latest", "onedir-nightly", "onedir-rc-3006-0rc1", "onedir-rc-3006-0rc2"]' + instances: '["stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "latest"]' rockylinux-9: @@ -529,7 +529,7 @@ jobs: distro-slug: rockylinux-9 display-name: Rocky Linux 9 timeout: 20 - instances: '["git-3005", "onedir-3005", "git-master", "onedir-nightly", "onedir-rc-3006-0rc1", "onedir-rc-3006-0rc2"]' + instances: '["git-3005", "onedir-3005", "onedir-3006", "git-master"]' ubuntu-2004: @@ -543,7 +543,7 @@ jobs: distro-slug: ubuntu-2004 display-name: Ubuntu 20.04 timeout: 20 - instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "git-master", "latest", "onedir-nightly", "onedir-rc-3006-0rc1", "onedir-rc-3006-0rc2"]' + instances: '["stable-3003", "stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "git-master", "latest"]' ubuntu-2204: @@ -557,7 +557,7 @@ jobs: distro-slug: ubuntu-2204 display-name: Ubuntu 22.04 timeout: 20 - instances: '["stable-3004", "stable-3005", "onedir-3005", "git-master", "latest", "onedir-nightly", "onedir-rc-3006-0rc1", "onedir-rc-3006-0rc2"]' + instances: '["stable-3004", "stable-3005", "onedir-3005", "stable-3006", "onedir-3006", "git-master", "latest"]' set-pipeline-exit-status: diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 000000000..d04378611 --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,101 @@ +name: Nightly S3 Update +run-name: "Nightly S3 (branch: ${{ github.ref_name }})" + +on: + workflow_dispatch: {} + schedule: + # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#onschedule + - cron: '0 1 * * *' # Every day at 1AM + +jobs: + + workflow-requirements: + name: Check Workflow Requirements + runs-on: ubuntu-latest + outputs: + requirements-met: ${{ steps.check-requirements.outputs.requirements-met }} + steps: + - name: Check For Admin Permission + if: ${{ github.event_name != 'schedule' }} + uses: actions-cool/check-user-permission@v2 + with: + require: admin + username: ${{ github.triggering_actor }} + + - name: Check Requirements + id: check-requirements + run: | + if [ "${{ vars.RUN_SCHEDULED_BUILDS }}" = "1" ]; then + MSG="Running workflow because RUN_SCHEDULED_BUILDS=1" + echo "${MSG}" + echo "${MSG}" >> "${GITHUB_STEP_SUMMARY}" + echo "requirements-met=true" >> "${GITHUB_OUTPUT}" + elif [ "${{ github.event.repository.fork }}" = "true" ]; then + MSG="Not running workflow because ${{ github.repository }} is a fork" + echo "${MSG}" + echo "${MSG}" >> "${GITHUB_STEP_SUMMARY}" + echo "requirements-met=false" >> "${GITHUB_OUTPUT}" + else + MSG="Running workflow because ${{ github.repository }} is not a fork" + echo "${MSG}" + echo "${MSG}" >> "${GITHUB_STEP_SUMMARY}" + echo "requirements-met=true" >> "${GITHUB_OUTPUT}" + fi + + update-s3-bucket: + name: Update S3 Bucket + if: ${{ fromJSON(needs.workflow-requirements.outputs.requirements-met) }} + runs-on: + - self-hosted + - linux + - repo-release + needs: + - workflow-requirements + environment: release + + steps: + - uses: actions/checkout@v3 + + - name: Get Salt Project GitHub Actions Bot Environment + run: | + TOKEN=$(curl -sS -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 30") + SPB_ENVIRONMENT=$(curl -sS -f -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/spb:environment) + echo "SPB_ENVIRONMENT=$SPB_ENVIRONMENT" >> "$GITHUB_ENV" + + - name: Setup GnuPG + run: | + sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg + GNUPGHOME="$(mktemp -d -p /run/gpg)" + echo "GNUPGHOME=${GNUPGHOME}" >> "$GITHUB_ENV" + cat < "${GNUPGHOME}/gpg.conf" + batch + no-tty + pinentry-mode loopback + EOF + + - name: Get Secrets + id: get-secrets + env: + SECRETS_KEY: ${{ secrets.SECRETS_KEY }} + run: | + SECRETS_KEY_FILE=$(mktemp /tmp/output.XXXXXXXXXX) + echo "$SECRETS_KEY" > "$SECRETS_KEY_FILE" + aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ + --query SecretString --output text | jq .default_key -r | base64 -d \ + | gpg --passphrase-file "${SECRETS_KEY_FILE}" -d - \ + | gpg --import - + sync + aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ + --query SecretString --output text| jq .default_passphrase -r | base64 -d \ + | gpg --passphrase-file "${SECRETS_KEY_FILE}" -o "${GNUPGHOME}/passphrase" -d - + sync + rm "$SECRETS_KEY_FILE" + echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" + + - name: Install Requirements + run: | + python3 -m pip install -r requirements/release.txt + + - name: Upload Develop to S3 + run: | + tools release s3-publish --key-id 64CBBC8173D76B3F develop diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2f6550c55..fa9c12653 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,13 +3,11 @@ name: Cut Release on: workflow_dispatch jobs: - update-develop: - name: Update CHANGELOG.md and bootstrap-salt.sh + + check-requirements: + name: Check Requirements runs-on: ubuntu-latest - if: github.repository == 'saltstack/salt-bootstrap' - permissions: - contents: write # To be able to publish the release - environment: Release + environment: release-check steps: - name: Check For Admin Permission uses: actions-cool/check-user-permission@v2 @@ -17,42 +15,110 @@ jobs: require: admin username: ${{ github.triggering_actor }} - - name: Check Branch Triggering Release + - name: Check Repository run: | - if [ "${{ github.ref_name }}" != "develop" ] - then - echo "This workflow should only be triggered from the develop branch" - exit 1 + if [ "${{ vars.RUN_RELEASE_BUILDS }}" = "1" ]; then + MSG="Running workflow because RUN_RELEASE_BUILDS=1" + echo "${MSG}" + echo "${MSG}" >> "${GITHUB_STEP_SUMMARY}" + exit 0 + fi + echo "Trying to run the release workflow from repository ${{ github.repository }}" + if [ "${{ github.repository }}" != "saltstack/salt-bootstrap" ]; then + MSG="Running the release workflow from the ${{ github.repository }} repository is not allowed" + echo "${MSG}" + echo "${MSG}" >> "${GITHUB_STEP_SUMMARY}" + MSG="Allowed repository: saltstack/salt-bootstrap" + echo "${MSG}" + echo "${MSG}" >> "${GITHUB_STEP_SUMMARY}" + exit 1 + else + MSG="Allowed to release from repository ${{ github.repository }}" + echo "${MSG}" + echo "${MSG}" >> "${GITHUB_STEP_SUMMARY}" + fi + + - name: Check Branch + run: | + echo "Trying to run the release workflow from branch ${{ github.ref_name }}" + if [ "${{ github.ref_name }}" != "develop" ]; then + echo "Running the release workflow from the ${{ github.ref_name }} branch is not allowed" + echo "Allowed branches: develop" + exit 1 + else + echo "Allowed to release from branch ${{ github.ref_name }}" fi + update-develop: + name: Update CHANGELOG.md and bootstrap-salt.sh + runs-on: + - self-hosted + - linux + - repo-release + permissions: + contents: write # To be able to publish the release + environment: release + needs: + - check-requirements + outputs: + release-version: ${{ steps.update-repo.outputs.release-version }} + steps: - uses: actions/checkout@v3 with: ref: develop repository: ${{ github.repository }} ssh-key: ${{ secrets.SALT_BOOTSTRAP_RELEASE_KEY }} - - name: Update Git Settings - run: | - git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" - git config --local user.name "github-actions[bot] on behalf of ${{ github.event.sender.login }}" - - - name: Set up Python 3.7 - uses: actions/setup-python@v4 - with: - python-version: 3.9 - - name: Install Requirements run: | - python3 -m pip install requests pre-commit + python3 -m pip install -r requirements/release.txt pre-commit install --install-hooks + - name: Setup GnuPG + run: | + sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg + GNUPGHOME="$(mktemp -d -p /run/gpg)" + echo "GNUPGHOME=${GNUPGHOME}" >> "$GITHUB_ENV" + cat < "${GNUPGHOME}/gpg.conf" + batch + no-tty + pinentry-mode loopback + EOF + + - name: Get Secrets + id: get-secrets + env: + SECRETS_KEY: ${{ secrets.SECRETS_KEY }} + run: | + SECRETS_KEY_FILE=$(mktemp /tmp/output.XXXXXXXXXX) + echo "$SECRETS_KEY" > "$SECRETS_KEY_FILE" + aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ + --query SecretString --output text | jq .default_key -r | base64 -d \ + | gpg --passphrase-file "${SECRETS_KEY_FILE}" -d - \ + | gpg --import - + sync + aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ + --query SecretString --output text| jq .default_passphrase -r | base64 -d \ + | gpg --passphrase-file "${SECRETS_KEY_FILE}" -o "${GNUPGHOME}/passphrase" -d - + sync + rm "$SECRETS_KEY_FILE" + echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" + + - name: Configure Git + shell: bash + run: | + git config --global --add safe.directory "$(pwd)" + git config --global user.name "Salt Project Packaging" + git config --global user.email saltproject-packaging@vmware.com + git config --global user.signingkey 64CBBC8173D76B3F + git config --global commit.gpgsign true + - name: Update Repository + id: update-repo env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | python3 .github/workflows/scripts/cut-release.py --repo ${{ github.repository }} - export CUT_RELEASE_VERSION=$(cat .cut_release_version) - echo "CUT_RELEASE_VERSION=${CUT_RELEASE_VERSION}" >> $GITHUB_ENV - name: Show Changes run: | @@ -61,15 +127,16 @@ jobs: - name: Commit Changes run: | - git commit -am "Update develop branch for the ${CUT_RELEASE_VERSION} release" || \ - git commit -am "Update develop branch for the ${CUT_RELEASE_VERSION} release" + git commit -am "Update develop branch for the ${{ steps.update-repo.outputs.release-version }} release" || \ + git commit -am "Update develop branch for the ${{ steps.update-repo.outputs.release-version }} release" - name: Push Changes - uses: ad-m/github-push-action@master + uses: ad-m/github-push-action@b87afee92c6e70ea888be6203a3e9426fda49839 with: - repository: ${{ github.repository }} - branch: develop ssh: true + atomic: true + branch: develop + repository: ${{ github.repository }} - name: Upload Release Details uses: actions/upload-artifact@v3 @@ -81,10 +148,13 @@ jobs: merge-develop-into-stable: name: Merge develop into stable - runs-on: ubuntu-latest - if: github.repository == 'saltstack/salt-bootstrap' - needs: update-develop - environment: Release + runs-on: + - self-hosted + - linux + - repo-release + needs: + - update-develop + environment: release permissions: contents: write # To be able to publish the release steps: @@ -95,64 +165,92 @@ jobs: ssh-key: ${{ secrets.SALT_BOOTSTRAP_RELEASE_KEY }} fetch-depth: 0 - - name: Update Git Settings + - name: Setup GnuPG run: | - git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" - git config --local user.name "github-actions[bot] on behalf of @${{ github.event.sender.login }}" + sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg + GNUPGHOME="$(mktemp -d -p /run/gpg)" + echo "GNUPGHOME=${GNUPGHOME}" >> "$GITHUB_ENV" + cat < "${GNUPGHOME}/gpg.conf" + batch + no-tty + pinentry-mode loopback + EOF + + - name: Get Secrets + id: get-secrets + env: + SECRETS_KEY: ${{ secrets.SECRETS_KEY }} + run: | + SECRETS_KEY_FILE=$(mktemp /tmp/output.XXXXXXXXXX) + echo "$SECRETS_KEY" > "$SECRETS_KEY_FILE" + aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ + --query SecretString --output text | jq .default_key -r | base64 -d \ + | gpg --passphrase-file "${SECRETS_KEY_FILE}" -d - \ + | gpg --import - + sync + aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ + --query SecretString --output text| jq .default_passphrase -r | base64 -d \ + | gpg --passphrase-file "${SECRETS_KEY_FILE}" -o "${GNUPGHOME}/passphrase" -d - + sync + rm "$SECRETS_KEY_FILE" + echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" + + - name: Configure Git + shell: bash + run: | + git config --global --add safe.directory "$(pwd)" + git config --global user.name "Salt Project Packaging" + git config --global user.email saltproject-packaging@vmware.com + git config --global user.signingkey 64CBBC8173D76B3F + git config --global commit.gpgsign true - name: Download Release Details uses: actions/download-artifact@v3 with: name: release-details - - name: Update Environment - run: | - export CUT_RELEASE_VERSION=$(cat .cut_release_version) - echo "CUT_RELEASE_VERSION=${CUT_RELEASE_VERSION}" >> $GITHUB_ENV - - name: Merge develop into stable run: | - git merge --no-ff -m "Merge develop into stable for ${CUT_RELEASE_VERSION} release" origin/develop || touch .git-conflicts + git merge --no-ff -m "Merge develop into stable for ${{ needs.update-develop.outputs.release-version }} release" origin/develop || touch .git-conflicts if [ -f .git-conflicts ] then git diff for f in $(git status | grep 'both modified' | awk '{ print $3 }') do - git checkout --theirs $f - pre-commit run -av --files $f - git add $f + git checkout --theirs "$f" + pre-commit run -av --files "$f" + git add "$f" done - git commit -a -m "Merge develop into stable for ${CUT_RELEASE_VERSION} release(auto resolving conflicts to the develop version)" + git commit -a -m "Merge develop into stable for ${{ needs.update-develop.outputs.release-version }} release(auto resolving conflicts to the develop version)" fi - - name: Tag Release - uses: mathieudutour/github-tag-action@v6.0 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - custom_tag: ${{ env.CUT_RELEASE_VERSION }} - tag_prefix: "" - create_annotated_tag: true + - name: Tag The ${{ needs.update-develop.outputs.release-version }} Release + run: | + git tag -m "Release ${{ needs.update-develop.outputs.release-version }}" -as ${{ needs.update-develop.outputs.release-version }} - name: Update bootstrap-salt.sh sha256sum's run: | - echo "$(sha256sum bootstrap-salt.sh | awk '{ print $1 }')" > bootstrap-salt.sh.sha256 - echo "$(sha256sum bootstrap-salt.ps1 | awk '{ print $1 }')" > bootstrap-salt.ps1.sha256 + sha256sum bootstrap-salt.sh | awk '{ print $1 }' > bootstrap-salt.sh.sha256 + sha256sum bootstrap-salt.ps1 | awk '{ print $1 }' > bootstrap-salt.ps1.sha256 git commit -a -m "Update sha256 checksums" || git commit -a -m "Update sha256 checksums" - name: Push Changes - uses: ad-m/github-push-action@master + uses: ad-m/github-push-action@b87afee92c6e70ea888be6203a3e9426fda49839 with: - repository: ${{ github.repository }} - branch: stable - tags: true ssh: true + tags: true + atomic: true + branch: stable + repository: ${{ github.repository }} publish-release: name: Create GitHub Release - runs-on: ubuntu-latest - if: github.repository == 'saltstack/salt-bootstrap' - needs: merge-develop-into-stable - environment: Release + runs-on: + - self-hosted + - linux + needs: + - merge-develop-into-stable + environment: release permissions: contents: write # To be able to publish the release steps: @@ -168,8 +266,8 @@ jobs: - name: Update Environment run: | - export CUT_RELEASE_VERSION=$(cat .cut_release_version) - echo "CUT_RELEASE_VERSION=${CUT_RELEASE_VERSION}" >> $GITHUB_ENV + CUT_RELEASE_VERSION=$(cat .cut_release_version) + echo "CUT_RELEASE_VERSION=${CUT_RELEASE_VERSION}" >> "$GITHUB_ENV" - name: Create Github Release uses: softprops/action-gh-release@v1 @@ -189,17 +287,77 @@ jobs: LICENSE - name: Delete Release Details Artifact - uses: geekyeggo/delete-artifact@v1 + uses: geekyeggo/delete-artifact@v2 with: name: release-details failOnError: false + update-s3-bucket: + name: Update S3 Bucket + runs-on: + - self-hosted + - linux + - repo-release + needs: + - publish-release + environment: release + + steps: + - uses: actions/checkout@v3 + + - name: Get Salt Project GitHub Actions Bot Environment + run: | + TOKEN=$(curl -sS -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 30") + SPB_ENVIRONMENT=$(curl -sS -f -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/spb:environment) + echo "SPB_ENVIRONMENT=$SPB_ENVIRONMENT" >> "$GITHUB_ENV" + + - name: Setup GnuPG + run: | + sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg + GNUPGHOME="$(mktemp -d -p /run/gpg)" + echo "GNUPGHOME=${GNUPGHOME}" >> "$GITHUB_ENV" + cat < "${GNUPGHOME}/gpg.conf" + batch + no-tty + pinentry-mode loopback + EOF + + - name: Get Secrets + id: get-secrets + env: + SECRETS_KEY: ${{ secrets.SECRETS_KEY }} + run: | + SECRETS_KEY_FILE=$(mktemp /tmp/output.XXXXXXXXXX) + echo "$SECRETS_KEY" > "$SECRETS_KEY_FILE" + aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ + --query SecretString --output text | jq .default_key -r | base64 -d \ + | gpg --passphrase-file "${SECRETS_KEY_FILE}" -d - \ + | gpg --import - + sync + aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ + --query SecretString --output text| jq .default_passphrase -r | base64 -d \ + | gpg --passphrase-file "${SECRETS_KEY_FILE}" -o "${GNUPGHOME}/passphrase" -d - + sync + rm "$SECRETS_KEY_FILE" + echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" + + - name: Install Requirements + run: | + python3 -m pip install -r requirements/release.txt + + - name: Upload Stable Release to S3 + run: | + tools release s3-publish --key-id 64CBBC8173D76B3F stable + update-develop-checksums: name: Update Release Checksums on Develop - runs-on: ubuntu-latest - if: github.repository == 'saltstack/salt-bootstrap' - needs: publish-release - environment: Release + runs-on: + - self-hosted + - linux + - repo-release + needs: + - publish-release + environment: release permissions: contents: write # For action peter-evans/create-pull-request pull-requests: write # For action peter-evans/create-pull-request @@ -213,8 +371,8 @@ jobs: - name: Get bootstrap-salt.sh on stable branch sha256sum run: | - echo "SH=$(sha256sum bootstrap-salt.sh | awk '{ print $1 }')" >> $GITHUB_ENV - echo "BS_VERSION=$(sh bootstrap-salt.sh -v | awk '{ print $4 }')" >> $GITHUB_ENV + echo "SH=$(sha256sum bootstrap-salt.sh | awk '{ print $1 }')" >> "$GITHUB_ENV" + echo "BS_VERSION=$(sh bootstrap-salt.sh -v | awk '{ print $4 }')" >> "$GITHUB_ENV" - uses: actions/checkout@v3 with: @@ -222,15 +380,44 @@ jobs: repository: ${{ github.repository }} ssh-key: ${{ secrets.SALT_BOOTSTRAP_RELEASE_KEY }} - - name: Update Git Settings + - name: Setup GnuPG run: | - git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" - git config --local user.name "github-actions[bot] on behalf of @${{ github.event.sender.login }}" - - - name: Set up Python 3.7 - uses: actions/setup-python@v4 - with: - python-version: 3.7 + sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg + GNUPGHOME="$(mktemp -d -p /run/gpg)" + echo "GNUPGHOME=${GNUPGHOME}" >> "$GITHUB_ENV" + cat < "${GNUPGHOME}/gpg.conf" + batch + no-tty + pinentry-mode loopback + EOF + + - name: Get Secrets + id: get-secrets + env: + SECRETS_KEY: ${{ secrets.SECRETS_KEY }} + run: | + SECRETS_KEY_FILE=$(mktemp /tmp/output.XXXXXXXXXX) + echo "$SECRETS_KEY" > "$SECRETS_KEY_FILE" + aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ + --query SecretString --output text | jq .default_key -r | base64 -d \ + | gpg --passphrase-file "${SECRETS_KEY_FILE}" -d - \ + | gpg --import - + sync + aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ + --query SecretString --output text| jq .default_passphrase -r | base64 -d \ + | gpg --passphrase-file "${SECRETS_KEY_FILE}" -o "${GNUPGHOME}/passphrase" -d - + sync + rm "$SECRETS_KEY_FILE" + echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" + + - name: Configure Git + shell: bash + run: | + git config --global --add safe.directory "$(pwd)" + git config --global user.name "Salt Project Packaging" + git config --global user.email saltproject-packaging@vmware.com + git config --global user.signingkey 64CBBC8173D76B3F + git config --global commit.gpgsign true - name: Update Latest Release on README run: | @@ -247,19 +434,23 @@ jobs: git commit -am "Update README.rst with ${{ env.BS_VERSION }} release sha256sum" - name: Push Changes - uses: ad-m/github-push-action@master + uses: ad-m/github-push-action@b87afee92c6e70ea888be6203a3e9426fda49839 with: - repository: ${{ github.repository }} - branch: develop ssh: true + atomic: true + branch: develop + repository: ${{ github.repository }} salt: name: Update Release on Salt Repo - runs-on: ubuntu-latest - if: github.repository == 'saltstack/salt-bootstrap' - needs: update-develop-checksums - environment: Open PR Against Salt + runs-on: + - self-hosted + - linux + - repo-release + needs: + - update-develop-checksums + environment: release permissions: contents: write # For action peter-evans/create-pull-request pull-requests: write # For action peter-evans/create-pull-request @@ -272,7 +463,7 @@ jobs: - name: Get bootstrap version run: | - echo "BS_VERSION=$(sh bootstrap-salt.sh -v | awk '{ print $4 }')" >> $GITHUB_ENV + echo "BS_VERSION=$(sh bootstrap-salt.sh -v | awk '{ print $4 }')" >> "$GITHUB_ENV" - uses: actions/checkout@v3 with: @@ -281,21 +472,63 @@ jobs: path: salt-checkout token: ${{ secrets.SALT_REPO_WRITE_TOKEN }} - - name: Update Git Settings + - name: Setup GnuPG + run: | + sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg + GNUPGHOME="$(mktemp -d -p /run/gpg)" + echo "GNUPGHOME=${GNUPGHOME}" >> "$GITHUB_ENV" + cat < "${GNUPGHOME}/gpg.conf" + batch + no-tty + pinentry-mode loopback + EOF + + - name: Get Secrets + id: get-secrets + env: + SECRETS_KEY: ${{ secrets.SECRETS_KEY }} run: | - git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" - git config --local user.name "github-actions[bot] on behalf of @${{ github.event.sender.login }}" + SECRETS_KEY_FILE=$(mktemp /tmp/output.XXXXXXXXXX) + echo "$SECRETS_KEY" > "$SECRETS_KEY_FILE" + aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ + --query SecretString --output text | jq .default_key -r | base64 -d \ + | gpg --passphrase-file "${SECRETS_KEY_FILE}" -d - \ + | gpg --import - + sync + aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ + --query SecretString --output text| jq .default_passphrase -r | base64 -d \ + | gpg --passphrase-file "${SECRETS_KEY_FILE}" -o "${GNUPGHOME}/passphrase" -d - + sync + rm "$SECRETS_KEY_FILE" + echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" + + - name: Configure Git + shell: bash + run: | + git config --global --add safe.directory "$(pwd)" + git config --global user.name "Salt Project Packaging" + git config --global user.email saltproject-packaging@vmware.com + git config --global user.signingkey 64CBBC8173D76B3F + git config --global commit.gpgsign true - name: Update bootstrap script on Salt run: | cp bootstrap-salt.sh salt-checkout/salt/cloud/deploy/bootstrap-salt.sh + - name: Commit Changes + working-directory: salt-checkout/ + run: | + git commit -am "Update the bootstrap script to v${{ env.BS_VERSION }}" + - name: Create Pull Request Against Develop - uses: peter-evans/create-pull-request@v4 + uses: peter-evans/create-pull-request@v5 with: title: "Update the bootstrap script to v${{ env.BS_VERSION }}" path: salt-checkout base: master token: ${{ secrets.SALT_REPO_WRITE_TOKEN }} + author: "Salt Project Packaging " + committer: "Salt Project Packaging " commit-message: Update the bootstrap script to v${{ env.BS_VERSION }} + signoff: true delete-branch: true diff --git a/.github/workflows/scripts/cut-release.py b/.github/workflows/scripts/cut-release.py index 4cbda55f9..3b6eabffb 100644 --- a/.github/workflows/scripts/cut-release.py +++ b/.github/workflows/scripts/cut-release.py @@ -1,12 +1,13 @@ #!/usr/bin/env python +import argparse import os +import pathlib import re import sys -import pathlib -import argparse -import requests from datetime import datetime +import requests + REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent.parent.parent @@ -155,6 +156,8 @@ def main(): message=f"Unable to generate changelog. HTTP Response:\n{changelog}", ) + github_output = os.environ.get("GITHUB_OUTPUT") + cut_release_version = REPO_ROOT / ".cut_release_version" print( f"* Writing {cut_release_version.relative_to(REPO_ROOT)} ...", @@ -162,6 +165,9 @@ def main(): flush=True, ) cut_release_version.write_text(options.release_tag) + if github_output is not None: + with open(github_output, "a", encoding="utf-8") as wfh: + wfh.write(f"release-version={options.release_tag}\n") cut_release_changes = REPO_ROOT / ".cut_release_changes" print( @@ -170,7 +176,6 @@ def main(): flush=True, ) cut_release_changes.write_text(changelog["body"]) - print( f"* Updating {changelog_file.relative_to(REPO_ROOT)} ...", file=sys.stderr, diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index a966c1746..f21ace903 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -1,9 +1,8 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- -import os +import datetime import json +import os import pathlib -import datetime os.chdir(os.path.abspath(os.path.dirname(__file__))) @@ -211,32 +210,62 @@ "ubuntu-2204", ] +BLACKLIST_3006 = [ + "arch", + "fedora-35", + "fedora-36", + "fedora-37", + "fedora-38", + "gentoo", + "gentoo-systemd", + "opensuse-15", + "opensuse-tumbleweed", +] + +BLACKLIST_GIT_3006 = [ + "almalinux-9", + "amazon-2", + "arch", + "centos-stream9", + "debian-10", + "debian-11", + "fedora-35", + "fedora-36", + "fedora-37", + "fedora-38", + "gentoo", + "gentoo-systemd", + "opensuse-15", + "opensuse-tumbleweed", + "rockylinux-9", + "ubuntu-2004", + "ubuntu-2110", + "ubuntu-2204", +] + SALT_VERSIONS = [ "3003", "3004", "3005", + "3006", "master", "latest", "nightly", - "3006-0rc1", - "3006-0rc2", ] ONEDIR_SALT_VERSIONS = [ "3005", + "3006", "latest", - "nightly", ] -ONEDIR_RC_SALT_VERSIONS = [ - "3006-0rc1", - "3006-0rc2", -] +ONEDIR_RC_SALT_VERSIONS = [] VERSION_DISPLAY_NAMES = { "3003": "v3003", "3004": "v3004", "3005": "v3005", + "3006": "v3006", "master": "Master", "latest": "Latest", "nightly": "Nightly", @@ -245,14 +274,10 @@ STABLE_VERSION_BLACKLIST = [ "master", "nightly", - "3006-0rc1", - "3006-0rc2", ] GIT_VERSION_BLACKLIST = [ "nightly", - "3006-0rc1", - "3006-0rc2", ] # TODO: Revert the commit relating to this section, once the Git-based builds @@ -555,12 +580,14 @@ def generate_test_jobs(): "3003": BLACKLIST_3003, "3004": BLACKLIST_3004, "3005": BLACKLIST_3005, + "3006": BLACKLIST_3006, } if bootstrap_type == "git": BLACKLIST = { "3003": BLACKLIST_GIT_3003, "3004": BLACKLIST_GIT_3004, "3005": BLACKLIST_GIT_3005, + "3006": BLACKLIST_GIT_3006, } # .0 versions are a virtual version for pinning to the first @@ -570,7 +597,7 @@ def generate_test_jobs(): continue if ( - salt_version in ("3003", "3004", "3005") + salt_version in ("3003", "3004", "3005", "3006") and distro in BLACKLIST[salt_version] ): continue diff --git a/.gitignore b/.gitignore index 45c7a12e6..9b7ebc0b0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,9 @@ /.project *.sw? +*.un~ .vagrant +tools +venv # Pycharm .idea diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b4d63be12..655876b51 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,10 +16,70 @@ repos: hooks: - id: mdformat + - repo: https://github.com/s0undt3ch/python-tools-scripts + rev: "0.12.0" + hooks: + - id: tools + alias: actionlint + name: Lint GitHub Actions Workflows + files: "^.github/workflows/" + types: + - yaml + args: + - pre-commit + - actionlint + + - repo: https://github.com/jazzband/pip-tools + rev: 6.13.0 + hooks: + - id: pip-compile + files: ^requirements/release\.(in|txt)$ + args: + - requirements/release.in + + - repo: https://github.com/asottile/pyupgrade + rev: v2.37.3 + hooks: + - id: pyupgrade + name: Rewrite Code to be Py3.9+ + args: [--py39-plus] + + - repo: https://github.com/asottile/reorder_python_imports + rev: v3.8.2 + hooks: + - id: reorder-python-imports + args: [--py39-plus] + - repo: https://github.com/psf/black rev: 22.6.0 hooks: - id: black + args: [] + +# - repo: https://github.com/PyCQA/flake8 +# rev: 3.9.2 +# hooks: +# - id: flake8 +# language_version: python3 +# additional_dependencies: +# - flake8-mypy-fork +# - flake8-docstrings +# - flake8-typing-imports +# +# - repo: https://github.com/pre-commit/mirrors-mypy +# rev: v1.0.0 +# hooks: +# - id: mypy +# alias: mypy-tools +# name: Run mypy against tools +# files: ^tools/.*\.py$ +# #args: [--strict] +# additional_dependencies: +# - attrs +# - rich +# - types-attrs +# - types-pyyaml +# - types-requests - repo: local hooks: diff --git a/CHANGELOG.md b/CHANGELOG.md index 277008e32..63788aabe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,15 @@ +# v2023.04.21 + +## What's Changed + +- Update release process by @s0undt3ch in https://github.com/saltstack/salt-bootstrap/pull/1908 +- Update with 3006 stable installs by @garethgreenaway in https://github.com/saltstack/salt-bootstrap/pull/1906 +- Add support for 3006 on windows bootstrap script by @twangboy in https://github.com/saltstack/salt-bootstrap/pull/1918 +- Fixes for 3006 by @garethgreenaway in https://github.com/saltstack/salt-bootstrap/pull/1912 +- Don't use GITHUB_OUTPUT by @s0undt3ch in https://github.com/saltstack/salt-bootstrap/pull/1911 + +**Full Changelog**: https://github.com/saltstack/salt-bootstrap/compare/v2023.04.06...v2023.04.21 + # v2023.04.06 ## What's Changed diff --git a/README.rst b/README.rst index 65c5702e2..ba2e9ee3d 100644 --- a/README.rst +++ b/README.rst @@ -32,6 +32,7 @@ sum** of the downloaded ``bootstrap-salt.sh`` file. The SHA256 sum of the ``bootstrap-salt.sh`` file, per release, is: +- 2023.04.06: ``994bf7e8bd92fe6d70d291c7562aff299f5651046b4e76dfa506cee0d9bb0843`` - 2022.10.04: ``d0686c2daeed18bb726e58eef75a69afe9ee56a1a23b2c32cd4e87d6005638e2`` - 2022.08.13: ``af922699c1a2bb3b89b6dac04397389999df1b3416b8d0b5c93766412f14c95c`` - 2022.08.12: ``b46f018bbf02f45c6096ab96e9261a9adb3a78ff65092c3976f32ffde909afcb`` diff --git a/bootstrap-salt.ps1 b/bootstrap-salt.ps1 index 4943e8f4d..74c7b202e 100644 --- a/bootstrap-salt.ps1 +++ b/bootstrap-salt.ps1 @@ -37,13 +37,13 @@ Specifies all the optional parameters in no particular order. .PARAMETER version - Default version defined in this script. + The version of the Salt minion to install. Default is "latest" which will + install the latest version of Salt minion available. .PARAMETER pythonVersion The version of Python the installer should use. Specify either "2" or "3". Beginning with Salt 2017.7.0, Salt will run on either Python 2 or Python 3. The default is Python 2 if not specified. This parameter only works for Salt - versions >= 2017.7.0. .PARAMETER runservice Boolean flag to start or stop the minion service. True will start the minion @@ -81,32 +81,47 @@ param( # Supports new version and latest # Option 1 means case insensitive [ValidatePattern('^(\d{4}(\.\d{1,2}){0,2}(\-\d{1})?)|(latest)$', Options=1)] - [string]$Version = '', + [Alias("v")] + [String]$Version = "latest", [Parameter(Mandatory=$false, ValueFromPipeline=$True)] - # Doesn't support Python versions prior to "2017.7.0" + # Python 3 support was added in 2017. Python 2 support was dropped in + # version 3001. This parameter is ignored for all versions before 2017 and + # after 3000. [ValidateSet("2","3")] - [string]$PythonVersion = "3", + [Alias("p")] + [String]$PythonVersion = "3", [Parameter(Mandatory=$false, ValueFromPipeline=$True)] [ValidateSet("true","false")] - [string]$RunService = "true", + [Alias("s")] + [String]$RunService = "true", [Parameter(Mandatory=$false, ValueFromPipeline=$True)] - [string]$Minion = "not-specified", + [Alias("m")] + [String]$Minion = "not-specified", [Parameter(Mandatory=$false, ValueFromPipeline=$True)] - [string]$Master = "not-specified", + [Alias("a")] + [String]$Master = "not-specified", [Parameter(Mandatory=$false, ValueFromPipeline=$True)] - [string]$RepoUrl= "https://repo.saltproject.io/windows", + [Alias("r")] + [String]$RepoUrl = "https://repo.saltproject.io/salt/py3/windows", [Parameter(Mandatory=$false, ValueFromPipeline=$True)] - [switch]$ConfigureOnly + [Alias("c")] + [Switch]$ConfigureOnly ) + +#=============================================================================== +# Script Preferences +#=============================================================================== # Powershell supports only TLS 1.0 by default. Add support for TLS 1.2 [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]'Tls12' +$ErrorActionPreference = "Stop" +$ProgressPreference = "SilentlyContinue" #=============================================================================== # Script Functions @@ -123,6 +138,129 @@ function Get-IsUacEnabled (Get-ItemProperty HKLM:\Software\Microsoft\Windows\CurrentVersion\Policies\System).EnableLua -ne 0 } +function Get-MajorVersion { + # Parses a version string and returns the major version + # + # Args: + # Version (string): The Version to parse + [CmdletBinding()] + param( + [Parameter(Mandatory=$true, Position=0)] + [String] $Version + ) + return ( $Version -split "\." )[0] +} + +function Convert-PSObjectToHashtable { + param ( + [Parameter(ValueFromPipeline)] + $InputObject + ) + if ($null -eq $InputObject) { return $null } + + $is_enum = $InputObject -is [System.Collections.IEnumerable] + $not_string = $InputObject -isnot [string] + if ($is_enum -and $not_string) { + $collection = @( + foreach ($object in $InputObject) { + Convert-PSObjectToHashtable $object + } + ) + + Write-Host -NoEnumerate $collection + } elseif ($InputObject -is [PSObject]) { + $hash = @{} + + foreach ($property in $InputObject.PSObject.Properties) { + $hash[$property.Name] = Convert-PSObjectToHashtable $property.Value + } + + $hash + } else { + $InputObject + } +} + +function Get-FileHash { + # Get-FileHash is a built-in cmdlet in powershell 5+ but we need to support + # powershell 3. This will overwrite the powershell 5 commandlet only for + # this script. But it will provide the missing cmdlet for powershell 3 + [CmdletBinding()] + param( + [Parameter(Mandatory=$true)] + [String] $Path, + + [Parameter(Mandatory=$false)] + [ValidateSet( + "SHA1", + "SHA256", + "SHA384", + "SHA512", + # https://serverfault.com/questions/820300/ + # why-isnt-mactripledes-algorithm-output-in-powershell-stable + "MACTripleDES", # don't use + "MD5", + "RIPEMD160", + IgnoreCase=$true)] + [String] $Algorithm = "SHA256" + ) + + if ( !(Test-Path $Path) ) { + Write-Verbose "Invalid path for hashing: $Path" + return @{} + } + + if ( (Get-Item -Path $Path) -isnot [System.IO.FileInfo]) { + Write-Verbose "Not a file for hashing: $Path" + return @{} + } + + $Path = Resolve-Path -Path $Path + + Switch ($Algorithm) { + SHA1 { + $hasher = [System.Security.Cryptography.SHA1CryptoServiceProvider]::Create() + } + SHA256 { + $hasher = [System.Security.Cryptography.SHA256]::Create() + } + SHA384 { + $hasher = [System.Security.Cryptography.SHA384]::Create() + } + SHA512 { + $hasher = [System.Security.Cryptography.SHA512]::Create() + } + MACTripleDES { + $hasher = [System.Security.Cryptography.MACTripleDES]::Create() + } + MD5 { + $hasher = [System.Security.Cryptography.MD5]::Create() + } + RIPEMD160 { + $hasher = [System.Security.Cryptography.RIPEMD160]::Create() + } + } + + Write-Verbose "Hashing using $Algorithm algorithm" + try { + $data = [System.IO.File]::OpenRead($Path) + $hash = $hasher.ComputeHash($data) + $hash = [System.BitConverter]::ToString($hash) -replace "-","" + return @{ + Path = $Path; + Algorithm = $Algorithm.ToUpper(); + Hash = $hash + } + } catch { + Write-Verbose "Error hashing: $Path" + return @{} + } finally { + if ($null -ne $data) { + $data.Close() + } + } +} + #=============================================================================== # Check for Elevated Privileges #=============================================================================== @@ -157,28 +295,56 @@ if (!(Get-IsAdministrator)) { } } +#=============================================================================== +# Change RepoUrl for older versions +#=============================================================================== +$defaultUrl = "https://repo.saltproject.io/salt/py3/windows" +$oldRepoUrl = "https://repo.saltproject.io/windows" +$majorVersion = Get-MajorVersion -Version $Version +$customUrl = $true +if ( $Version.ToLower() -ne "latest" ) { + # A specific version has been passed + # We only want to modify the URL if a custom URL was not passed + $uri = [Uri]($RepoUrl) + if ( $uri.AbsoluteUri -eq $defaultUrl ) { + # No customURL passed, let's check for a pre 3006 version + $customUrl = $false + if ( $majorVersion -lt "3006" ) { + # This is an older version, use the old URL + $RepoUrl = $oldRepoUrl + } else { + # This is a new URL, and a version was passed, let's look in minor + if ( $Version.ToLower() -ne $majorVersion.ToLower() ) { + $RepoUrl = "$RepoUrl/minor" + } + } + } +} else { + if ( $RepoUrl -eq $defaultUrl ) { + $customUrl = $false + } +} + #=============================================================================== # Verify Parameters #=============================================================================== Write-Verbose "Parameters passed in:" -Write-Verbose "version: $version" -Write-Verbose "runservice: $runservice" -Write-Verbose "master: $master" -Write-Verbose "minion: $minion" -Write-Verbose "repourl: $repourl" +Write-Verbose "version: $Version" +Write-Verbose "runservice: $RunService" +Write-Verbose "master: $Master" +Write-Verbose "minion: $Minion" +Write-Verbose "repourl: $RepoUrl" -if ($runservice.ToLower() -eq "true") { +if ($RunService.ToLower() -eq "true") { Write-Verbose "Windows service will be set to run" - [bool]$runservice = $True -} -elseif ($runservice.ToLower() -eq "false") { + [bool]$RunService = $True +} elseif ($RunService.ToLower() -eq "false") { Write-Verbose "Windows service will be stopped and set to manual" - [bool]$runservice = $False -} -else { + [bool]$RunService = $False +} else { # Param passed in wasn't clear so defaulting to true. Write-Verbose "Windows service defaulting to run automatically" - [bool]$runservice = $True + [bool]$RunService = $True } #=============================================================================== @@ -231,8 +397,14 @@ if (Test-Path C:\tmp\grains) { $ConfiguredAnything = $True } -if ($ConfigureOnly -and !$ConfiguredAnything) { - Write-Output "No configuration or keys were copied over. No configuration was done!" +if ( $ConfigureOnly ) { + if ( !$ConfiguredAnything ) { + Write-Host "No configuration or keys were copied over." -ForegroundColor yes + Write-Host "No configuration was done!" -ForegroundColor Yellow + } else { + Write-Host "Salt minion successfully configured" -ForegroundColor Green + } + # If we're only configuring, we want to end here exit 0 } @@ -246,97 +418,168 @@ if ([IntPtr]::Size -eq 4) { } #=============================================================================== -# Use version "Latest" if no version is passed +# Get file name to download #=============================================================================== -if ((!$version) -or ($version.ToLower() -eq 'latest')){ - $versionSection = "Latest-Py$PythonVersion" +$saltFileName = "" +$saltVersion = "" +$saltSha512= "" +$saltFileUrl = "" +if ( ($customUrl) -or ($majorVersion -lt 3006) ) { + $saltFileName = "Salt-Minion-$Version-Py3-$arch-Setup.exe" + $saltVersion = $Version + $saltFileUrl = "$RepoUrl/$saltFileName" } else { - $versionSection = $version - $year = $version.Substring(0, 4) - if ([int]$year -ge 2017) { - if ($PythonVersion -eq "3") { - $versionSection = "$version-Py3" - } else { - $versionSection = "$version-Py2" + if ( $majorVersion -ge 3006 ) { + $enc = [System.Text.Encoding]::UTF8 + try { + $response = Invoke-WebRequest -Uri "$RepoUrl/repo.json" -UseBasicParsing + if ($response.Content.GetType().Name -eq "Byte[]") { + $psobj = $enc.GetString($response.Content) | ConvertFrom-Json + } else { + $psobj = $response.Content | ConvertFrom-Json + } + $hash = Convert-PSObjectToHashtable $psobj + } catch { + Write-Verbose "repo.json not found at: $RepoUrl" + $hash = @{} + } + + $searchVersion = $Version.ToLower() + if ( $hash.Contains($searchVersion)) { + foreach ($item in $hash.($searchVersion).Keys) { + if ( $item.EndsWith(".exe") ) { + if ( $item.Contains($arch) ) { + $saltFileName = $hash.($searchVersion).($item).name + $saltVersion = $hash.($searchVersion).($item).version + $saltSha512 = $hash.($searchVersion).($item).SHA512 + } + } + } + } + if ( $saltFileName -and $saltVersion -and $saltSha512 ) { + if ( $RepoUrl.Contains("minor") ) { + $saltFileUrl = @($RepoUrl, $saltVersion, $saltFileName) -join "/" + } else { + $saltFileUrl = @($RepoUrl, "minor", $saltVersion, $saltFileName) -join "/" + } } } } -if (!$ConfigureOnly) { - #=============================================================================== - # Download minion setup file - #=============================================================================== - $saltExe = "Salt-Minion-$versionSection-$arch-Setup.exe" - Write-Output "Downloading Salt minion installer $saltExe" - $webclient = New-Object System.Net.WebClient - $url = "$repourl/$saltExe" - $file = "C:\Windows\Temp\$saltExe" - $webclient.DownloadFile($url, $file) - - #=============================================================================== - # Set the parameters for the installer - #=============================================================================== - # Unless specified, use the installer defaults - # - id: - # - master: salt - # - Start the service - $parameters = "" - if($minion -ne "not-specified") {$parameters = "/minion-name=$minion"} - if($master -ne "not-specified") {$parameters = "$parameters /master=$master"} - if($runservice -eq $false) {$parameters = "$parameters /start-service=0"} - - #=============================================================================== - # Install minion silently - #=============================================================================== - #Wait for process to exit before continuing. - Write-Output "Installing Salt minion" - Start-Process C:\Windows\Temp\$saltExe -ArgumentList "/S $parameters" -Wait -NoNewWindow -PassThru | Out-Null - - #=============================================================================== - # Configure the minion service - #=============================================================================== - # Wait for salt-minion service to be registered before trying to start it - $service = Get-Service salt-minion -ErrorAction SilentlyContinue - while (!$service) { - Start-Sleep -s 2 - $service = Get-Service salt-minion -ErrorAction SilentlyContinue +#=============================================================================== +# Download minion setup file +#=============================================================================== +Write-Host "===============================================================================" -ForegroundColor Yellow +Write-Host " Bootstrapping Salt Minion" -ForegroundColor Green +Write-Host " - version: $Version" +Write-Host " - file name: $saltFileName" +Write-Host " - file url: $saltFileUrl" +Write-Host "-------------------------------------------------------------------------------" -ForegroundColor Yellow +Write-Host "Downloading Installer: " -NoNewline +$webclient = New-Object System.Net.WebClient +$localFile = "C:\Windows\Temp\$saltFileName" +$webclient.DownloadFile($saltFileUrl, $localFile) + +if ( Test-Path -Path $localFile ) { + Write-Host "Success" -ForegroundColor Green +} else { + Write-Host "Failed" -ForegroundColor Red +} + +if ( $saltSha512 ) { + $localSha512 = (Get-FileHash -Path $localFile -Algorithm SHA512).Hash + Write-Host "Comparing Hash: " -NoNewline + if ( $localSha512 -eq $saltSha512 ) { + Write-Host "Success" -ForegroundColor Green + } else { + Write-Host "Failed" -ForegroundColor Red + exit 1 } +} - if($runservice) { - # Start service - Write-Output "Starting the Salt minion service" +#=============================================================================== +# Set the parameters for the installer +#=============================================================================== +# Unless specified, use the installer defaults +# - id: +# - master: salt +# - Start the service +$parameters = "" +if($Minion -ne "not-specified") {$parameters = "/minion-name=$Minion"} +if($Master -ne "not-specified") {$parameters = "$parameters /master=$Master"} +if($RunService -eq $false) {$parameters = "$parameters /start-service=0"} + +#=============================================================================== +# Install minion silently +#=============================================================================== +#Wait for process to exit before continuing. +Write-Host "Installing Salt Minion: " -NoNewline +Start-Process $localFile -ArgumentList "/S $parameters" -Wait -NoNewWindow -PassThru | Out-Null + +#=============================================================================== +# Configure the minion service +#=============================================================================== +# Wait for salt-minion service to be registered before trying to start it +$service = Get-Service salt-minion -ErrorAction SilentlyContinue +while (!$service) { + Start-Sleep -s 2 + $service = Get-Service salt-minion -ErrorAction SilentlyContinue +} +if ( $service ) { + Write-Host "Success" -ForegroundColor Green +} else { + Write-Host "Failed" -ForegroundColor Red + exit 1 +} + +if($RunService) { + # Start service + Write-Host "Starting Service: " -NoNewline + Start-Service -Name "salt-minion" -ErrorAction SilentlyContinue + + # Check if service is started, otherwise retry starting the + # service 4 times. + $try = 0 + while (($service.Status -ne "Running") -and ($try -ne 4)) { Start-Service -Name "salt-minion" -ErrorAction SilentlyContinue + $service = Get-Service salt-minion -ErrorAction SilentlyContinue + Start-Sleep -s 2 + $try += 1 + } - # Check if service is started, otherwise retry starting the - # service 4 times. - $try = 0 - while (($service.Status -ne "Running") -and ($try -ne 4)) { - Start-Service -Name "salt-minion" -ErrorAction SilentlyContinue - $service = Get-Service salt-minion -ErrorAction SilentlyContinue - Start-Sleep -s 2 - $try += 1 - } + # If the salt-minion service is still not running, something probably + # went wrong and user intervention is required - report failure. + if ($service.Status -eq "Running") { + Write-Host "Success" -ForegroundColor Green + } else { + Write-Host "Failed" -ForegroundColor Red + exit 1 + } - # If the salt-minion service is still not running, something probably - # went wrong and user intervention is required - report failure. - if ($service.Status -eq "Stopped") { - Write-Output -NoNewline "Failed to start salt minion" - exit 1 - } +} else { + Write-Host "Setting Service to 'Manual': " -NoNewline + Set-Service "salt-minion" -StartupType "Manual" + if ( (Get-Service "salt-minion").StartType -eq "Manual" ) { + Write-Host "Success" -ForegroundColor Green + } else { + Write-Host "Failed" -ForegroundColor Red + exit 1 } - else { - Write-Output -NoNewline "Stopping salt minion and setting it to 'Manual'" - Set-Service "salt-minion" -StartupType "Manual" - Stop-Service "salt-minion" + + Write-Host "Stopping Service: " -NoNewline + Stop-Service "salt-minion" + if ( (Get-Service "salt-minion").Status -eq "Stopped" ) { + Write-Host "Success" -ForegroundColor Green + } else { + Write-Host "Failed" -ForegroundColor Red + exit 1 } } #=============================================================================== # Script Complete #=============================================================================== -if ($ConfigureOnly) { - Write-Output "Salt minion successfully configured" -} -else { - Write-Output "Salt minion successfully installed" -} +Write-Host "-------------------------------------------------------------------------------" -ForegroundColor Yellow +Write-Host "Salt Minion Installed Successfully" -ForegroundColor Green +Write-Host "===============================================================================" -ForegroundColor Yellow +exit 0 diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 1eb391b86..a7240e5f2 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -23,7 +23,7 @@ #====================================================================================================================== set -o nounset # Treat unset variables as an error -__ScriptVersion="2023.04.06" +__ScriptVersion="2023.04.21" __ScriptName="bootstrap-salt.sh" __ScriptFullName="$0" @@ -615,12 +615,24 @@ if [ "$ITYPE" = "git" ]; then # If doing stable install, check if version specified elif [ "$ITYPE" = "stable" ]; then if [ "$#" -eq 0 ];then - STABLE_REV="latest" + ONEDIR_REV="latest" + _ONEDIR_REV="$1" + ITYPE="onedir" else - if [ "$(echo "$1" | grep -E '^(latest|1\.6|1\.7|2014\.1|2014\.7|2015\.5|2015\.8|2016\.3|2016\.11|2017\.7|2018\.3|2019\.2|3000|3001|3002|3003|3004|3005)$')" != "" ]; then + if [ "$(echo "$1" | grep -E '^(nightly|latest|3006)$')" != "" ]; then + ONEDIR_REV="$1" + _ONEDIR_REV="$1" + ITYPE="onedir" + shift + elif [ "$(echo "$1" | grep -E '^(3003|3004|3005)$')" != "" ]; then STABLE_REV="$1" shift - elif [ "$(echo "$1" | grep -E '^(2[0-9]*\.[0-9]*\.[0-9]*|[3-9][0-9]{3}(\.[0-9]*)?)$')" != "" ]; then + elif [ "$(echo "$1" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + ONEDIR_REV="minor/$1" + _ONEDIR_REV="$1" + ITYPE="onedir" + shift + elif [ "$(echo "$1" | grep -E '^([3-9][0-5]{3}(\.[0-9]*)?)$')" != "" ]; then # Handle the 3xxx.0 version as 3xxx archive (pin to minor) and strip the fake ".0" suffix STABLE_REV=$(echo "$1" | sed -E 's/^([3-9][0-9]{3})\.0$/\1/') if [ "$(uname)" != "Darwin" ]; then @@ -628,7 +640,7 @@ elif [ "$ITYPE" = "stable" ]; then fi shift else - echo "Unknown stable version: $1 (valid: 1.6, 1.7, 2014.1, 2014.7, 2015.5, 2015.8, 2016.3, 2016.11, 2017.7, 2018.3, 2019.2, 3000, 3001, 3002, 3003, 3004, 3005, latest, \$MAJOR.\$MINOR.\$PATCH until 2019.2, \$MAJOR or \$MAJOR.\$PATCH starting from 3000)" + echo "Unknown stable version: $1 (valid: 3003, 3004, 3005, 3006, latest)" exit 1 fi fi @@ -637,7 +649,7 @@ elif [ "$ITYPE" = "onedir" ]; then if [ "$#" -eq 0 ];then ONEDIR_REV="latest" else - if [ "$(echo "$1" | grep -E '^(nightly|latest|3005)$')" != "" ]; then + if [ "$(echo "$1" | grep -E '^(nightly|latest|3005|3006)$')" != "" ]; then ONEDIR_REV="$1" shift elif [ "$(echo "$1" | grep -E '^(3005(\.[0-9]*)?)')" != "" ]; then @@ -649,7 +661,7 @@ elif [ "$ITYPE" = "onedir" ]; then ONEDIR_REV="minor/$1" shift else - echo "Unknown onedir version: $1 (valid: 3005, latest, nightly.)" + echo "Unknown onedir version: $1 (valid: 3005, 3006, latest, nightly.)" exit 1 fi fi @@ -887,6 +899,18 @@ __fetch_verify() { return 1 } +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: __check_url_exists +# DESCRIPTION: Checks if a URL exists +#---------------------------------------------------------------------------------------------------------------------- +__check_url_exists() { + _URL="$1" + if curl --output /dev/null --silent --fail "${_URL}"; then + return 0 + else + return 1 + fi +} #--- FUNCTION ------------------------------------------------------------------------------------------------------- # NAME: __gather_hardware_info # DESCRIPTION: Discover hardware information @@ -1936,10 +1960,6 @@ if [ "$ITYPE" = "git" ]; then if [ "$__NEW_VS_TAG_REGEX_MATCH" = "MATCH" ]; then _POST_NEON_INSTALL=$BS_TRUE __TAG_REGEX_MATCH="${__NEW_VS_TAG_REGEX_MATCH}" - if [ "$(echo "${GIT_REV}" | cut -c -1)" != "v" ]; then - # We do this to properly clone tags - GIT_REV="v${GIT_REV}" - fi echodebug "Post Neon Tag Regex Match On: ${GIT_REV}" else __TAG_REGEX_MATCH=$(echo "${GIT_REV}" | sed -E 's/^(v?[0-9]{1,4}\.[0-9]{1,2})(\.[0-9]{1,2})?.*$/MATCH/') @@ -1951,10 +1971,6 @@ if [ "$ITYPE" = "git" ]; then if [ "$__NEW_VS_TAG_REGEX_MATCH" = "MATCH" ]; then _POST_NEON_INSTALL=$BS_TRUE __TAG_REGEX_MATCH="${__NEW_VS_TAG_REGEX_MATCH}" - if [ "$(echo "${GIT_REV}" | cut -c -1)" != "v" ]; then - # We do this to properly clone tags - GIT_REV="v${GIT_REV}" - fi echodebug "Post Neon Tag Regex Match On: ${GIT_REV}" else __TAG_REGEX_MATCH=$(echo "${GIT_REV}" | sed 's/^.*\(v\?[[:digit:]]\{1,4\}\.[[:digit:]]\{1,2\}\)\(\.[[:digit:]]\{1,2\}\)\?.*$/MATCH/') @@ -3077,8 +3093,11 @@ __install_saltstack_ubuntu_onedir_repository() { fi echo "$__REPO_ARCH_DEB $SALTSTACK_UBUNTU_URL $UBUNTU_CODENAME main" > /etc/apt/sources.list.d/salt.list - if [ "$(echo "${ONEDIR_REV}" | grep -E '(3004|3005|nightly)')" != "" ]; then + if [ "$(echo "${ONEDIR_REV}" | grep -E '(3004|3005)')" != "" ]; then __apt_key_fetch "${SALTSTACK_UBUNTU_URL}salt-archive-keyring.gpg" || return 1 + elif [ "$(echo "${ONEDIR_REV}" | grep -E '(latest|nightly)')" != "" ]; then + __apt_key_fetch "${SALTSTACK_UBUNTU_URL}salt-archive-keyring.gpg" || \ + __apt_key_fetch "${SALTSTACK_UBUNTU_URL}SALT-PROJECT-GPG-PUBKEY-2023.gpg" || return 1 else __apt_key_fetch "${SALTSTACK_UBUNTU_URL}SALT-PROJECT-GPG-PUBKEY-2023.gpg" || return 1 fi @@ -3621,8 +3640,11 @@ __install_saltstack_debian_onedir_repository() { fi echo "$__REPO_ARCH_DEB $SALTSTACK_DEBIAN_URL $DEBIAN_CODENAME main" > "/etc/apt/sources.list.d/salt.list" - if [ "$(echo "${ONEDIR_REV}" | grep -E '(3004|3005|nightly)')" != "" ]; then + if [ "$(echo "${ONEDIR_REV}" | grep -E '(3004|3005)')" != "" ]; then __apt_key_fetch "${SALTSTACK_DEBIAN_URL}salt-archive-keyring.gpg" || return 1 + elif [ "$(echo "${ONEDIR_REV}" | grep -E '(latest|nightly)')" != "" ]; then + __apt_key_fetch "${SALTSTACK_DEBIAN_URL}salt-archive-keyring.gpg" || \ + __apt_key_fetch "${SALTSTACK_DEBIAN_URL}SALT-PROJECT-GPG-PUBKEY-2023.gpg" || return 1 else __apt_key_fetch "${SALTSTACK_DEBIAN_URL}SALT-PROJECT-GPG-PUBKEY-2023.gpg" || return 1 fi @@ -4426,6 +4448,19 @@ install_fedora_check_services() { return 0 } +install_fedora_onedir() { + STABLE_REV=$ONEDIR_REV + install_fedora_stable || return 1 + + return 0 +} + +install_fedora_onedir_post() { + STABLE_REV=$ONEDIR_REV + install_fedora_stable_post || return 1 + + return 0 +} # # Ended Fedora Install Functions # @@ -4509,7 +4544,7 @@ __install_saltstack_rhel_onedir_repository() { if [ "${ONEDIR_REV}" = "nightly" ] ; then base_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/redhat/${DISTRO_MAJOR_VERSION}/\$basearch/" fi - if [ "$(echo "${ONEDIR_REV}" | grep -E '(3004|3005|nightly)')" != "" ]; then + if [ "$(echo "${ONEDIR_REV}" | grep -E '(3004|3005)')" != "" ]; then if [ "${DISTRO_MAJOR_VERSION}" -eq 9 ]; then gpg_key="SALTSTACK-GPG-KEY2.pub" else @@ -4909,6 +4944,8 @@ install_centos_onedir_deps() { __PACKAGES="yum-utils chkconfig" fi + __PACKAGES="${__PACKAGES} procps" + # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -6302,13 +6339,13 @@ install_amazon_linux_ami_2_onedir_deps() { base_url="$HTTP_VAL://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/amazon/2/\$basearch/" fi - if [ "$(echo "${ONEDIR_REV}" | grep -E '(3004|3005|nightly)')" != "" ]; then + if [ "$(echo "${ONEDIR_REV}" | grep -E '(3004|3005)')" != "" ]; then gpg_key="${base_url}SALTSTACK-GPG-KEY.pub,${base_url}base/RPM-GPG-KEY-CentOS-7" if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then - gpg_key="${base_url}SALTSTACK-GPG-KEY.pub" + gpg_key="${base_url}SALTSTACK-GPG-KEY.pub" fi else - gpg_key="${base_url}SALT-PROJECT-GPG-PUBKEY-2023.pub" + gpg_key="${base_url}SALT-PROJECT-GPG-PUBKEY-2023.pub" fi # This should prob be refactored to use __install_saltstack_rhel_repository() @@ -6527,6 +6564,10 @@ install_arch_linux_git_deps() { return 0 } +install_arch_linux_onedir_deps() { + install_arch_linux_stable_deps || return 1 +} + install_arch_linux_stable() { # Pacman does not resolve dependencies on outdated versions # They always need to be updated @@ -6679,6 +6720,18 @@ install_arch_check_services() { return 0 } + +install_arch_linux_onedir() { + install_arch_linux_stable || return 1 + + return 0 +} + +install_arch_linux_onedir_post() { + install_arch_linux_post || return 1 + + return 0 +} # # Ended Arch Install Functions # @@ -6857,6 +6910,15 @@ install_freebsd_restart_daemons() { service salt_$fname start done } + +install_freebsd_onedir() { +# +# call install_freebsd_stable +# + install_freebsd_stable || return 1 + + return 0 +} # # Ended FreeBSD Install Functions # @@ -6975,6 +7037,14 @@ install_openbsd_restart_daemons() { return 0 } +install_openbsd_onedir() { +# +# Call install_openbsd_stable +# + install_openbsd_stable || return 1 + + return 0 +} # # Ended OpenBSD Install Functions # @@ -7175,6 +7245,14 @@ install_smartos_restart_daemons() { return 0 } +install_smartos_onedir() { +# +# call install_smartos_stable +# + install_smartos_stable || return 1 + + return 0 +} # # Ended SmartOS Install Functions # @@ -7359,6 +7437,10 @@ install_opensuse_git_deps() { return 0 } +install_opensuse_onedir_deps() { + install_opensuse_stable_deps || return 1 +} + install_opensuse_stable() { __PACKAGES="" @@ -7391,6 +7473,10 @@ install_opensuse_git() { return 0 } +install_opensuse_onedir() { + install_opensuse_stable || return 1 +} + install_opensuse_stable_post() { for fname in api master minion syndic; do # Skip salt-api since the service should be opt-in and not necessarily started on boot @@ -7460,6 +7546,10 @@ install_opensuse_git_post() { return 0 } +install_opensuse_onedir_post() { + install_opensuse_stable_post || return 1 +} + install_opensuse_restart_daemons() { [ $_START_DAEMONS -eq $BS_FALSE ] && return @@ -7965,11 +8055,6 @@ __gentoo_pre_dep() { mkdir /etc/portage fi - # Enable Python 3.6 target for pre Neon Salt release - if echo "${STABLE_REV}" | grep -q "2019" || [ "${ITYPE}" = "git" ] && [ "${_POST_NEON_INSTALL}" -eq $BS_FALSE ]; then - EXTRA_PYTHON_TARGET=python3_6 - fi - # Enable Python 3.7 target for Salt Neon using GIT if [ "${ITYPE}" = "git" ] && [ "${GIT_REV}" = "v3000" ]; then EXTRA_PYTHON_TARGET=python3_7 @@ -8115,6 +8200,11 @@ install_gentoo_git() { return 0 } +install_gentoo_onedir() { + STABLE_REV=${ONEDIR_REV} + install_gentoo_stable || return 1 +} + install_gentoo_post() { for fname in api master minion syndic; do # Skip salt-api since the service should be opt-in and not necessarily started on boot @@ -8204,6 +8294,10 @@ _eof return 0 } +install_gentoo_onedir_post() { + install_gentoo_post || return 1 +} + install_gentoo_restart_daemons() { [ $_START_DAEMONS -eq $BS_FALSE ] && return @@ -8358,17 +8452,66 @@ __macosx_get_packagesite() { SALTPKGCONFURL="https://${_REPO_URL}/osx/${PKG}" } +__parse_repo_json_python() { + + # Using latest, grab the right + # version from the repo.json + _JSON_VERSION=$(python - <<-EOF +import json, urllib.request +url = "https://repo.saltproject.io/salt/py3/macos/repo.json" +response = urllib.request.urlopen(url) +data = json.loads(response.read()) +version = data["${_ONEDIR_REV}"][list(data["${_ONEDIR_REV}"])[0]]['version'] +print(version) +EOF +) +echo "${_JSON_VERSION}" +} + +__macosx_get_packagesite_onedir() { + DARWIN_ARCH="x86_64" + + __PY_VERSION_REPO="py2" + if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 3 ]; then + __PY_VERSION_REPO="py3" + fi + + if [ "$(echo "$_ONEDIR_REV" | grep -E '^(latest)$')" != "" ]; then + _PKG_VERSION=$(__parse_repo_json_python) + elif [ "$(echo "$_ONEDIR_REV" | grep -E '^([3-9][0-9]{3}(\.[0-9]*))')" != "" ]; then + _PKG_VERSION=$_ONEDIR_REV + else + _PKG_VERSION=$(__parse_repo_json_python) + fi + if [ "$(echo "$_ONEDIR_REV" | grep -E '^(3005)')" != "" ]; then + PKG="salt-${_PKG_VERSION}-macos-${DARWIN_ARCH}.pkg" + else + PKG="salt-${_PKG_VERSION}-${__PY_VERSION_REPO}-${DARWIN_ARCH}.pkg" + fi + SALTPKGCONFURL="https://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/macos/${ONEDIR_REV}/${PKG}" +} + # Using a separate conf step to head for idempotent install... __configure_macosx_pkg_details() { __macosx_get_packagesite || return 1 return 0 } +__configure_macosx_pkg_details_onedir() { + __macosx_get_packagesite_onedir || return 1 + return 0 +} + install_macosx_stable_deps() { __configure_macosx_pkg_details || return 1 return 0 } +install_macosx_onedir_deps() { + __configure_macosx_pkg_details_onedir || return 1 + return 0 +} + install_macosx_git_deps() { install_macosx_stable_deps || return 1 @@ -8415,6 +8558,16 @@ install_macosx_stable() { return 0 } +install_macosx_onedir() { + install_macosx_onedir_deps || return 1 + + __fetch_url "/tmp/${PKG}" "${SALTPKGCONFURL}" || return 1 + + /usr/sbin/installer -pkg "/tmp/${PKG}" -target / || return 1 + + return 0 +} + install_macosx_git() { if [ -n "$_PY_EXE" ]; then @@ -8452,6 +8605,11 @@ install_macosx_stable_post() { return 0 } +install_macosx_onedir_post() { + install_macosx_stable_post || return 1 + return 0 +} + install_macosx_git_post() { install_macosx_stable_post || return 1 return 0 @@ -8460,8 +8618,15 @@ install_macosx_git_post() { install_macosx_restart_daemons() { [ $_START_DAEMONS -eq $BS_FALSE ] && return - /bin/launchctl unload -w /Library/LaunchDaemons/com.saltstack.salt.minion.plist || return 1 - /bin/launchctl load -w /Library/LaunchDaemons/com.saltstack.salt.minion.plist || return 1 + if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then + /bin/launchctl unload -w /Library/LaunchDaemons/com.saltstack.salt.minion.plist || return 1 + /bin/launchctl load -w /Library/LaunchDaemons/com.saltstack.salt.minion.plist || return 1 + fi + + if [ "$_INSTALL_MASTER" -eq $BS_TRUE ]; then + /bin/launchctl unload -w /Library/LaunchDaemons/com.saltstack.salt.master.plist || return 1 + /bin/launchctl load -w /Library/LaunchDaemons/com.saltstack.salt.master.plist || return 1 + fi return 0 } diff --git a/kitchen.macos.yml b/kitchen.macos.yml index 3504f2e13..f6cd2dced 100644 --- a/kitchen.macos.yml +++ b/kitchen.macos.yml @@ -5,7 +5,6 @@ driver: provisioner: sudo: true salt_bootstrap_options: -MP stable %s - salt_call_command: /opt/salt/bin/salt-call init_environment: | echo 'auto_accept: true' > /tmp/auto-accept-keys.conf sudo mkdir -p /etc/salt/master.d @@ -22,15 +21,23 @@ suites: - name: stable-3003 provisioner: salt_version: 3003.4 + salt_call_command: /opt/salt/bin/salt-call - name: stable-3004 provisioner: salt_version: 3004.1 + salt_call_command: /opt/salt/bin/salt-call - name: stable-3005 provisioner: salt_version: 3005.1 + salt_call_command: /opt/salt/bin/salt-call + - name: stable-3006 + provisioner: + salt_version: 3006 + salt_call_command: /opt/salt/salt-call - name: latest provisioner: salt_version: latest + salt_call_command: /opt/salt/salt-call verifier: command: pytest --cache-clear -v -s -ra --log-cli-level=debug -k "not test_ping" tests/integration/ diff --git a/kitchen.windows.yml b/kitchen.windows.yml index f68f4b12f..f3e16b390 100644 --- a/kitchen.windows.yml +++ b/kitchen.windows.yml @@ -20,16 +20,22 @@ suites: - name: stable-3003 provisioner: salt_version: 3003.4-1 + salt_call_command: c:\salt\salt-call.bat - name: stable-3004 provisioner: salt_version: 3004.1-1 + salt_call_command: c:\Program Files\Salt Project\Salt\salt-call.bat - name: stable-3005 provisioner: salt_version: 3005.1-1 + - name: stable-3006 + provisioner: + salt_version: 3006.0 + salt_call_command: c:\Program Files\Salt Project\Salt\salt-call.exe - name: latest provisioner: salt_version: latest - salt_call_command: c:\Program Files\Salt Project\Salt\salt-call.bat + salt_call_command: c:\Program Files\Salt Project\Salt\salt-call.exe verifier: command: pytest --cache-clear -v -s -ra --log-cli-level=debug -k "not test_ping" tests/integration/ diff --git a/kitchen.yml b/kitchen.yml index 8ff0c11e5..615a8b267 100644 --- a/kitchen.yml +++ b/kitchen.yml @@ -150,11 +150,11 @@ platforms: suites: - name: git-3003 provisioner: - salt_version: 3003 + salt_version: v3003 salt_bootstrap_options: -x python3 -MPfq git %s - name: git-3004 provisioner: - salt_version: 3004 + salt_version: v3004 salt_bootstrap_options: -x python3 -MPfq git %s excludes: - opensuse-15 @@ -168,7 +168,21 @@ suites: - openbsd-6 - name: git-3005 provisioner: - salt_version: 3005 + salt_version: v3005 + salt_bootstrap_options: -x python3 -MPfq git %s + excludes: + - opensuse-15 + - opensuse-tumbleweed + - debian-11 + - arch + - gentoo + - gentoo-systemd + - freebsd-131 + - freebsd-123 + - openbsd-6 + - name: git-3006 + provisioner: + salt_version: v3006 salt_bootstrap_options: -x python3 -MPfq git %s excludes: - opensuse-15 @@ -246,6 +260,17 @@ suites: - freebsd-131 - freebsd-123 - openbsd-6 + - name: stable-3006 + provisioner: + salt_version: 3006 + salt_bootstrap_options: -x python3 -MP stable %s + excludes: + - opensuse-15 + - opensuse-tumbleweed + - arch + - freebsd-131 + - freebsd-123 + - openbsd-6 - name: git-master provisioner: salt_version: master @@ -271,6 +296,11 @@ suites: salt_version: 3005 salt_bootstrap_options: -MP onedir %s + - name: onedir-3006 + provisioner: + salt_version: 3006 + salt_bootstrap_options: -MP onedir %s + - name: onedir-rc-3006-0rc2 provisioner: salt_version: 3006.0rc2 diff --git a/requirements/release.in b/requirements/release.in new file mode 100644 index 000000000..d0d1a7378 --- /dev/null +++ b/requirements/release.in @@ -0,0 +1,3 @@ +pre-commit +python-tools-scripts >= 0.12.0 +boto3 diff --git a/requirements/release.txt b/requirements/release.txt new file mode 100644 index 000000000..b4d5a12dd --- /dev/null +++ b/requirements/release.txt @@ -0,0 +1,69 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile requirements/release.in +# +attrs==22.2.0 + # via python-tools-scripts +boto3==1.26.110 + # via -r requirements/release.in +botocore==1.29.110 + # via + # boto3 + # s3transfer +certifi==2022.12.7 + # via requests +cfgv==3.3.1 + # via pre-commit +charset-normalizer==3.1.0 + # via requests +distlib==0.3.6 + # via virtualenv +filelock==3.11.0 + # via virtualenv +identify==2.5.22 + # via pre-commit +idna==3.4 + # via requests +jmespath==1.0.1 + # via + # boto3 + # botocore +markdown-it-py==2.2.0 + # via rich +mdurl==0.1.2 + # via markdown-it-py +nodeenv==1.7.0 + # via pre-commit +platformdirs==3.2.0 + # via virtualenv +pre-commit==3.2.2 + # via -r requirements/release.in +pygments==2.15.0 + # via rich +python-dateutil==2.8.2 + # via botocore +python-tools-scripts==0.12.0 + # via -r requirements/release.in +pyyaml==6.0 + # via pre-commit +requests==2.28.2 + # via python-tools-scripts +rich==13.3.3 + # via python-tools-scripts +s3transfer==0.6.0 + # via boto3 +six==1.16.0 + # via python-dateutil +typing-extensions==4.5.0 + # via python-tools-scripts +urllib3==1.26.15 + # via + # botocore + # requests +virtualenv==20.21.0 + # via pre-commit + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/tests/conftest.py b/tests/conftest.py index 9e26a11b1..b24d90c8a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,8 +1,9 @@ +import logging import os import pprint + import pytest import testinfra -import logging log = logging.getLogger(__name__) @@ -47,6 +48,8 @@ def target_python_version(): @pytest.fixture(scope="session") def target_salt_version(): target_salt = os.environ["KITCHEN_SUITE"].split("-", 2)[-1].replace("-", ".") + if target_salt.startswith("v"): + target_salt = target_salt[1:] if target_salt in ("latest", "master", "nightly"): pytest.skip("Don't have a specific salt version to test against") return target_salt diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index aacc7de6c..9ee18d49c 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -1,6 +1,5 @@ -# -*- coding: utf-8 -*- -import os import logging +import os from contextlib import nullcontext log = logging.getLogger(__name__) @@ -26,7 +25,7 @@ def test_target_python_version(host, target_python_version): def test_target_salt_version(host, target_salt_version): with selected_context_manager(host): ret = host.salt("grains.item", "saltversion", "--timeout=120") - if target_salt_version.endswith(".0"): + if target_salt_version.endswith(".0") or target_salt_version.endswith(".x"): assert ret["saltversion"] == ".".join(target_salt_version.split(".")[:-1]) else: assert ret["saltversion"].startswith(target_salt_version) diff --git a/tools/__init__.py b/tools/__init__.py new file mode 100644 index 000000000..4a820ca8f --- /dev/null +++ b/tools/__init__.py @@ -0,0 +1,9 @@ +import logging + +import ptscripts + +ptscripts.register_tools_module("tools.pre_commit") +ptscripts.register_tools_module("tools.release") + +for name in ("boto3", "botocore", "urllib3"): + logging.getLogger(name).setLevel(logging.INFO) diff --git a/tools/pre_commit.py b/tools/pre_commit.py new file mode 100644 index 000000000..da24bf820 --- /dev/null +++ b/tools/pre_commit.py @@ -0,0 +1,51 @@ +""" +These commands are used by pre-commit. +""" +# pylint: disable=resource-leakage,broad-except,3rd-party-module-not-gated +from __future__ import annotations + +import logging +import shutil + +from ptscripts import Context, command_group + + +log = logging.getLogger(__name__) + +# Define the command group +cgroup = command_group( + name="pre-commit", help="Pre-Commit Related Commands", description=__doc__ +) + + +@cgroup.command( + name="actionlint", + arguments={ + "files": { + "help": "Files to run actionlint against", + "nargs": "*", + }, + "no_color": { + "help": "Disable colors in output", + }, + }, +) +def actionlint(ctx: Context, files: list[str], no_color: bool = False): + """ + Run `actionlint` + """ + actionlint = shutil.which("actionlint") + if not actionlint: + ctx.warn("Could not find the 'actionlint' binary") + ctx.exit(0) + cmdline = [actionlint] + if no_color is False: + cmdline.append("-color") + shellcheck = shutil.which("shellcheck") + if shellcheck: + cmdline.append(f"-shellcheck={shellcheck}") + pyflakes = shutil.which("pyflakes") + if pyflakes: + cmdline.append(f"-pyflakes={pyflakes}") + ret = ctx.run(*cmdline, *files, check=False) + ctx.exit(ret.returncode) diff --git a/tools/release.py b/tools/release.py new file mode 100644 index 000000000..fa703c4c6 --- /dev/null +++ b/tools/release.py @@ -0,0 +1,147 @@ +""" +These commands are used to release Salt Bootstrap. +""" +# pylint: disable=resource-leakage,broad-except,3rd-party-module-not-gated +from __future__ import annotations + +import logging +import os +import pathlib +import sys +from typing import TYPE_CHECKING + +from ptscripts import command_group +from ptscripts import Context + +import tools.utils + +try: + import boto3 +except ImportError: + print( + "\nPlease run 'python -m pip install -r requirements/release.txt'\n", + file=sys.stderr, + flush=True, + ) + raise + +log = logging.getLogger(__name__) + +# Define the command group +release = command_group( + name="release", + help="Release Related Commands", + description=__doc__, +) + + +@release.command( + name="s3-publish", + arguments={ + "branch": { + "help": "The kind of publish to do.", + "choices": ("stable", "develop"), + }, + "key_id": { + "help": "The GnuPG key ID used to sign.", + "required": True, + }, + }, +) +def s3_publish(ctx: Context, branch: str, key_id: str = None): + """ + Publish scripts to S3. + """ + if TYPE_CHECKING: + assert key_id + + ctx.info("Preparing upload ...") + s3 = boto3.client("s3") + + ctx.info( + f"Uploading release artifacts to {tools.utils.RELEASE_BUCKET_NAME!r} bucket ..." + ) + upload_files = { + "stable": { + f"{tools.utils.GPG_KEY_FILENAME}.gpg": [ + f"bootstrap/stable/{tools.utils.GPG_KEY_FILENAME}.gpg", + ], + f"{tools.utils.GPG_KEY_FILENAME}.pub": [ + f"bootstrap/stable/{tools.utils.GPG_KEY_FILENAME}.pub", + ], + "bootstrap-salt.sh": [ + "bootstrap/stable/bootstrap-salt.sh", + ], + "bootstrap-salt.sh.sha256": [ + "bootstrap/stable/bootstrap-salt.sh.sha256", + "bootstrap/stable/bootstrap/sha256", + ], + "bootstrap-salt.ps1": [ + "bootstrap/stable/bootstrap-salt.ps1", + ], + "bootstrap-salt.ps1.sha256": [ + "bootstrap/stable/bootstrap-salt.ps1.sha256", + "bootstrap/stable/winbootstrap/sha256", + ], + }, + "develop": { + f"{tools.utils.GPG_KEY_FILENAME}.gpg": [ + f"bootstrap/develop/{tools.utils.GPG_KEY_FILENAME}.gpg", + ], + f"{tools.utils.GPG_KEY_FILENAME}.pub": [ + f"bootstrap/develop/{tools.utils.GPG_KEY_FILENAME}.pub", + ], + "bootstrap-salt.sh": [ + "bootstrap/develop/bootstrap-salt.sh", + "bootstrap/develop/bootstrap/develop", + ], + "bootstrap-salt.sh.sha256": [ + "bootstrap/develop/bootstrap-salt.sh.sha256", + ], + "bootstrap-salt.ps1": [ + "bootstrap/develop/bootstrap-salt.ps1", + "bootstrap/develop/winbootstrap/develop", + ], + "bootstrap-salt.ps1.sha256": [ + "bootstrap/develop/bootstrap-salt.ps1.sha256", + ], + }, + } + + files_to_upload: list[tuple[str, str]] = [] + + try: + # Export the GPG key in use + tools.utils.export_gpg_key(ctx, key_id, tools.utils.REPO_ROOT) + for lpath, rpaths in upload_files[branch].items(): + ctx.info(f"Processing {lpath} ...") + if lpath.endswith(".sha256") and not os.path.exists(lpath): + ret = ctx.run( + "sha256sum", + lpath.replace(".sha256", ""), + capture=True, + check=False, + ) + if ret.returncode: + ctx.error(f"Failed to get the sha256sum of {lpath}") + ctx.exit(1) + pathlib.Path(lpath).write_bytes(ret.stdout) + for rpath in rpaths: + files_to_upload.append((lpath, rpath)) + if not lpath.endswith((".gpg", ".pub")): + tools.utils.gpg_sign(ctx, key_id, pathlib.Path(lpath)) + files_to_upload.append((f"{lpath}.asc", f"{rpaths[0]}.asc")) + + for lpath, rpath in sorted(files_to_upload): + size = pathlib.Path(lpath).stat().st_size + ctx.info(f" Uploading {lpath} -> {rpath}") + with tools.utils.create_progress_bar(file_progress=True) as progress: + task = progress.add_task(description="Uploading...", total=size) + s3.upload_file( + lpath, + tools.utils.RELEASE_BUCKET_NAME, + rpath, + Callback=tools.utils.UpdateProgress(progress, task), + ) + except KeyboardInterrupt: + pass diff --git a/tools/utils.py b/tools/utils.py new file mode 100644 index 000000000..b07beb9bc --- /dev/null +++ b/tools/utils.py @@ -0,0 +1,80 @@ +# pylint: disable=resource-leakage,broad-except,3rd-party-module-not-gated +from __future__ import annotations + +import os +import pathlib + +from ptscripts import Context +from rich.progress import BarColumn +from rich.progress import Column +from rich.progress import DownloadColumn +from rich.progress import Progress +from rich.progress import TextColumn +from rich.progress import TimeRemainingColumn +from rich.progress import TransferSpeedColumn + +REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent +GPG_KEY_FILENAME = "SALT-PROJECT-GPG-PUBKEY-2023" +SPB_ENVIRONMENT = os.environ.get("SPB_ENVIRONMENT") or "prod" +RELEASE_BUCKET_NAME = f"salt-project-{SPB_ENVIRONMENT}-salt-artifacts-release" + + +class UpdateProgress: + def __init__(self, progress, task): + self.progress = progress + self.task = task + + def __call__(self, chunk_size): + self.progress.update(self.task, advance=chunk_size) + + +def create_progress_bar(file_progress: bool = False, **kwargs): + if file_progress: + return Progress( + TextColumn("[progress.description]{task.description}"), + BarColumn(), + DownloadColumn(), + TransferSpeedColumn(), + TextColumn("eta"), + TimeRemainingColumn(), + **kwargs, + ) + return Progress( + TextColumn( + "[progress.description]{task.description}", table_column=Column(ratio=3) + ), + BarColumn(), + expand=True, + **kwargs, + ) + + +def export_gpg_key(ctx: Context, key_id: str, export_path: pathlib.Path): + keyfile_gpg = export_path.joinpath(GPG_KEY_FILENAME).with_suffix(".gpg") + if keyfile_gpg.exists(): + keyfile_gpg.unlink() + ctx.info(f"Exporting GnuPG Key '{key_id}' to {keyfile_gpg} ...") + ctx.run("gpg", "--output", str(keyfile_gpg), "--export", key_id) + keyfile_pub = export_path.joinpath(GPG_KEY_FILENAME).with_suffix(".pub") + if keyfile_pub.exists(): + keyfile_pub.unlink() + ctx.info(f"Exporting GnuPG Key '{key_id}' to {keyfile_pub} ...") + ctx.run("gpg", "--armor", "--output", str(keyfile_pub), "--export", key_id) + + +def gpg_sign(ctx: Context, key_id: str, path: pathlib.Path): + ctx.info(f"GPG Signing '{path}' ...") + signature_fpath = path.parent / f"{path.name}.asc" + if signature_fpath.exists(): + signature_fpath.unlink() + ctx.run( + "gpg", + "--local-user", + key_id, + "--output", + str(signature_fpath), + "--armor", + "--detach-sign", + "--sign", + str(path), + )