From 57b4abe8fd21bfd03d1010f82ff20d33aea353c4 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 27 May 2022 18:04:15 -0400 Subject: [PATCH 001/326] Update actions/setup-go from v2 to v3 Additionally as of v3.1.0 of actions/setup-go there is a go-version output value to retrieve the version of Go installed by the Action. This allows us to remove the step to manually retrieve this information from the Go executable. --- .github/workflows/build.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 27b2dcf..3216eec 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -26,14 +26,10 @@ jobs: python-version: "3.10" # We need the Go version and Go cache location for the actions/cache step, # so the Go installation must happen before that. - - uses: actions/setup-go@v2 + - id: setup-go + uses: actions/setup-go@v3 with: go-version: "1.16" - - name: Store installed Go version - id: go-version - run: | - echo "::set-output name=version::"\ - "$(go version | sed 's/^go version go\([0-9.]\+\) .*/\1/')" - name: Lookup Go cache directory id: go-cache run: | @@ -42,7 +38,7 @@ jobs: env: BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ py${{ steps.setup-python.outputs.python-version }}-\ - go${{ steps.go-version.outputs.version }}-\ + go${{ steps.setup-go.outputs.go-version }}-\ packer${{ steps.setup-env.outputs.packer-version }}-\ tf${{ steps.setup-env.outputs.terraform-version }}-" with: From 8f6849857bc2a66254cafa94db31526692fab209 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 27 May 2022 18:07:10 -0400 Subject: [PATCH 002/326] Update the version of Go used from 1.16 to 1.18 Go 1.16 is no longer supported as of the release of 1.18 so it makes sense to update to the latest version available. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3216eec..d4c50c5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,7 +29,7 @@ jobs: - id: setup-go uses: actions/setup-go@v3 with: - go-version: "1.16" + go-version: "1.18" - name: Lookup Go cache directory id: go-cache run: | From fc55e84feba28e9652e2a21b976e3f8225b708af Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 8 Nov 2022 13:27:47 -0500 Subject: [PATCH 003/326] Update the version of Go used from 1.18 to 1.19 Go 1.19 was released while this branch was in the wings and it makes sense to bump to the latest Go release. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d4c50c5..66e0c95 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,7 +29,7 @@ jobs: - id: setup-go uses: actions/setup-go@v3 with: - go-version: "1.18" + go-version: "1.19" - name: Lookup Go cache directory id: go-cache run: | From 2775dfa470b433cb9692b5b8c604ad55aa24ea2a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Nov 2022 20:00:23 +0000 Subject: [PATCH 004/326] Bump hashicorp/setup-terraform from 1 to 2 Bumps [hashicorp/setup-terraform](https://github.com/hashicorp/setup-terraform) from 1 to 2. - [Release notes](https://github.com/hashicorp/setup-terraform/releases) - [Changelog](https://github.com/hashicorp/setup-terraform/blob/main/CHANGELOG.md) - [Commits](https://github.com/hashicorp/setup-terraform/compare/v1...v2) --- updated-dependencies: - dependency-name: hashicorp/setup-terraform dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 66e0c95..37df1a3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -74,7 +74,7 @@ jobs: ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" sudo mv /usr/local/bin/packer /usr/local/bin/packer-default sudo ln -s /opt/packer/packer /usr/local/bin/packer - - uses: hashicorp/setup-terraform@v1 + - uses: hashicorp/setup-terraform@v2 with: terraform_version: ${{ steps.setup-env.outputs.terraform-version }} - name: Install shfmt From 1f7c62e5f21f1ee0a034122b1e96e2e035ae4a5b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Nov 2022 20:06:28 +0000 Subject: [PATCH 005/326] Bump actions/setup-python from 3 to 4 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 3 to 4. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 37df1a3..a624fe2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -21,7 +21,7 @@ jobs: uses: cisagov/setup-env-github-action@develop - uses: actions/checkout@v3 - id: setup-python - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: "3.10" # We need the Go version and Go cache location for the actions/cache step, From 6a9c630f869dbd96001d9a00ea300e3195f2c05d Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 27 May 2022 17:30:33 -0400 Subject: [PATCH 006/326] Add an ownership comment to the Dependabot configuration Add a comment that states that the commented out ignore directives are managed by cisagov/skeleton-generic. --- .github/dependabot.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 208659c..38ffdc0 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -12,6 +12,7 @@ updates: schedule: interval: "weekly" # ignore: + # # Managed by cisagov/skeleton-generic # - dependency-name: actions/cache # - dependency-name: actions/checkout # - dependency-name: actions/setup-python From dfd54e9ad294cedce783c61071f5dab151cf79e5 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 27 May 2022 17:38:06 -0400 Subject: [PATCH 007/326] Add additional Dependabot ignore directives This adds the other versioned Actions that should be managed by cisagov/skeleton-generic to the list of commented out dependencies to ignore. --- .github/dependabot.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 38ffdc0..fef57f5 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -15,7 +15,10 @@ updates: # # Managed by cisagov/skeleton-generic # - dependency-name: actions/cache # - dependency-name: actions/checkout + # - dependency-name: actions/setup-go # - dependency-name: actions/setup-python + # - dependency-name: hashicorp/setup-terraform + # - dependency-name: mxschmitt/action-tmate - package-ecosystem: "pip" directory: "/" From 71a88a5694347f17115f90f69e8ab09108aae1ed Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 11 Oct 2022 10:57:16 -0400 Subject: [PATCH 008/326] Remove usage of set-output from our GitHub Actions workflow GitHub has deprecated the set-output command per: https://github.blog/changelog/2022-10-11-github-actions-deprecating-save-state-and-set-output-commands/ This updates the GitHub Actions workflow to use the newly preferred method to set the output for a job's step. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a624fe2..8d17421 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -33,7 +33,7 @@ jobs: - name: Lookup Go cache directory id: go-cache run: | - echo "::set-output name=dir::$(go env GOCACHE)" + echo "dir=$(go env GOCACHE)" >> $GITHUB_OUTPUT - uses: actions/cache@v3 env: BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ From 21a68f461cf04ee5009c22e6311afdfdeff6f0fe Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 7 Sep 2022 13:35:44 -0400 Subject: [PATCH 009/326] Enable the allow-non-breakable-inline-mappings rule for yamllint --- .yamllint | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.yamllint b/.yamllint index 76a1cce..b5b302f 100644 --- a/.yamllint +++ b/.yamllint @@ -8,6 +8,14 @@ rules: # this behavior. comments-indentation: disable + # yamllint does not allow inline mappings that exceed the line length by + # default. There are many scenarios where the inline mapping may be a key, + # hash, or other long value that would exceed the line length but cannot + # reasonably be broken across lines. + line-length: + # This rule implies the allow-non-breakable-words rule + allow-non-breakable-inline-mappings: true + # yamllint doesn't like when we use yes and no for true and false, # but that's pretty standard in Ansible. truthy: disable From 3547c4f28d98a0e08582f5cf02e53d102a9e5dac Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 7 Sep 2022 13:37:21 -0400 Subject: [PATCH 010/326] Increase the maximum line length in the yamllint configuration Increase the line length maximum from the default of 80 to 88. This 10% increase mirrors the line length allowed in other linter configurations such as black. --- .yamllint | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.yamllint b/.yamllint index b5b302f..8f4e6bf 100644 --- a/.yamllint +++ b/.yamllint @@ -15,7 +15,8 @@ rules: line-length: # This rule implies the allow-non-breakable-words rule allow-non-breakable-inline-mappings: true - + # Allows a 10% overage from the default limit of 80 + max: 88 # yamllint doesn't like when we use yes and no for true and false, # but that's pretty standard in Ansible. truthy: disable From b1ea3d4239179f9dc15a3509593f921c77dedc6b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 14 Oct 2022 09:48:04 -0400 Subject: [PATCH 011/326] Fix the repo of the flake8 pre-commit hook Per PyCQA/flake8#1290 this hook moved from GitLab to GitHub. The version we use is bumped to the latest tag on GitHub as well. --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c5c2e10..804fa6c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -91,8 +91,8 @@ repos: rev: 22.3.0 hooks: - id: black - - repo: https://gitlab.com/pycqa/flake8 - rev: 3.9.2 + - repo: https://github.com/PyCQA/flake8 + rev: 5.0.4 hooks: - id: flake8 additional_dependencies: From fb8d13f5baff79b0f0e8a231cdf2c68817695d4e Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 27 May 2022 18:45:45 -0400 Subject: [PATCH 012/326] Update pre-commit hooks Update pre-commit hooks using `pre-commit autoupdate`. The `ansible-lint` hook is intentionally held back due to issues with upgrading to v6. --- .pre-commit-config.yaml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 804fa6c..25f8d1d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ default_language_version: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.1.0 + rev: v4.3.0 hooks: - id: check-case-conflict - id: check-executables-have-shebangs @@ -31,17 +31,17 @@ repos: # Text file hooks - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.31.1 + rev: v0.32.2 hooks: - id: markdownlint args: - --config=.mdl_config.yaml - repo: https://github.com/pre-commit/mirrors-prettier - rev: v2.6.1 + rev: v3.0.0-alpha.4 hooks: - id: prettier - repo: https://github.com/adrienverge/yamllint - rev: v1.26.3 + rev: v1.28.0 hooks: - id: yamllint args: @@ -49,14 +49,14 @@ repos: # GitHub Actions hooks - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.14.2 + rev: 0.18.4 hooks: - id: check-github-actions - id: check-github-workflows # pre-commit hooks - repo: https://github.com/pre-commit/pre-commit - rev: v2.17.0 + rev: v2.20.0 hooks: - id: validate_manifest @@ -88,7 +88,7 @@ repos: args: - --config=.bandit.yml - repo: https://github.com/psf/black - rev: 22.3.0 + rev: 22.10.0 hooks: - id: black - repo: https://github.com/PyCQA/flake8 @@ -102,11 +102,11 @@ repos: hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.942 + rev: v0.990 hooks: - id: mypy - repo: https://github.com/asottile/pyupgrade - rev: v2.31.1 + rev: v3.2.0 hooks: - id: pyupgrade @@ -119,14 +119,14 @@ repos: # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.64.0 + rev: v1.76.0 hooks: - id: terraform_fmt - id: terraform_validate # Docker hooks - repo: https://github.com/IamTheFij/docker-pre-commit - rev: v2.1.0 + rev: v2.1.1 hooks: - id: docker-compose-check From e5fbd70b2035b6de74c5bcc804af929a19d9cb17 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 5 Oct 2022 04:44:13 -0400 Subject: [PATCH 013/326] Add a label configuration file This configuration file stores information about the labels expected in this repository. --- .github/labels.yml | 67 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 .github/labels.yml diff --git a/.github/labels.yml b/.github/labels.yml new file mode 100644 index 0000000..7d7f7f3 --- /dev/null +++ b/.github/labels.yml @@ -0,0 +1,67 @@ +--- +# Rather than breaking up descriptions into multiline strings we disable that +# specific rule in yamllint for this file. +# yamllint disable rule:line-length +- color: "#eb6420" + description: This issue or pull request is awaiting the outcome of another issue or pull request + name: blocked +- color: "#000000" + description: This issue or pull request involves changes to existing functionality + name: breaking change +- color: "#d73a4a" + description: This issue or pull request addresses broken functionality + name: bug +- color: "#07648d" + description: This issue will be advertised on code.gov's Open Tasks page (https://code.gov/open-tasks) + name: code.gov +- color: "#0366d6" + description: Pull requests that update a dependency file + name: dependencies +- color: "#5319e7" + description: This issue or pull request improves or adds to documentation + name: documentation +- color: "#5319e7" + description: This issue or pull request improves or adds to documentation + name: documentation +- color: "#cfd3d7" + description: This issue or pull request already exists or is covered in another issue or pull request + name: duplicate +- color: "#b005bc" + description: A high-level objective issue encompassing multiple issues instead of a specific unit of work + name: epic +- color: "#000000" + description: Pull requests that update GitHub Actions code + name: github-actions +- color: "#0e8a16" + description: This issue or pull request is well-defined and good for newcomers + name: good first issue +- color: "#ff7518" + description: Pull request that should count toward Hacktoberfest participation + name: hacktoberfest-accepted +- color: "#a2eeef" + description: This issue or pull request will add or improve functionality, maintainability, or ease of use + name: improvement +- color: "#fef2c0" + description: This issue or pull request is not applicable, incorrect, or obsolete + name: invalid +- color: "#ce099a" + description: This pull request is ready to merge during the next Lineage Kraken release + name: kraken 🐙 +- color: "#a4fc5d" + description: This issue or pull request requires further information + name: need info +- color: "#fcdb45" + description: This pull request is awaiting an action or decision to move forward + name: on hold +- color: "#ef476c" + description: This issue is a request for information or needs discussion + name: question +- color: "#1d76db" + description: This issue or pull request pulls in upstream updates + name: upstream update +- color: "#d4c5f9" + description: This issue or pull request increments the version number + name: version bump +- color: "#ffffff" + description: This issue will not be incorporated + name: wontfix From b9792a8d18d3690b8497e0cfe31ce855c85a7c2b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 5 Oct 2022 04:57:03 -0400 Subject: [PATCH 014/326] Add a workflow to synchronize repository labels This adds a workflow to ensure that the repository labels are updated to reflect changes to the label configuration file .github/labels.yml. --- .github/workflows/sync-labels.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .github/workflows/sync-labels.yml diff --git a/.github/workflows/sync-labels.yml b/.github/workflows/sync-labels.yml new file mode 100644 index 0000000..ae7c091 --- /dev/null +++ b/.github/workflows/sync-labels.yml @@ -0,0 +1,29 @@ +--- +name: sync-labels + +on: + push: + paths: + - '.github/labels.yml' + - '.github/workflows/sync-labels.yml' + +permissions: + contents: read + +jobs: + labeler: + permissions: + # actions/checkout needs this to fetch code + contents: read + # crazy-max/ghaction-github-labeler needs this to manage repository labels + issues: write + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Sync repository labels + if: success() + uses: crazy-max/ghaction-github-labeler@v4 + with: + # This is a hideous ternary equivalent so we only do a dry run unless + # this workflow is triggered by the develop branch. + dry-run: ${{ github.ref_name == 'develop' && 'false' || 'true' }} From 161c4aebba002d4073a634749f3ae67f649a8a2a Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 21 Oct 2022 15:56:30 -0400 Subject: [PATCH 015/326] Remove duplicate label --- .github/labels.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/labels.yml b/.github/labels.yml index 7d7f7f3..24bd511 100644 --- a/.github/labels.yml +++ b/.github/labels.yml @@ -20,9 +20,6 @@ - color: "#5319e7" description: This issue or pull request improves or adds to documentation name: documentation -- color: "#5319e7" - description: This issue or pull request improves or adds to documentation - name: documentation - color: "#cfd3d7" description: This issue or pull request already exists or is covered in another issue or pull request name: duplicate From 5993231c3a136c86fe2dad178ac47495771cccce Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 21 Oct 2022 15:56:49 -0400 Subject: [PATCH 016/326] Add a label for issues or PRs that involve test code --- .github/labels.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/labels.yml b/.github/labels.yml index 24bd511..b108748 100644 --- a/.github/labels.yml +++ b/.github/labels.yml @@ -53,6 +53,9 @@ - color: "#ef476c" description: This issue is a request for information or needs discussion name: question +- color: "#00008b" + description: This issue or pull request adds or otherwise modifies test code + name: test - color: "#1d76db" description: This issue or pull request pulls in upstream updates name: upstream update From 6cc53dc1f0bf5123465082100c57bb6a325024cf Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 9 Nov 2022 11:23:04 -0500 Subject: [PATCH 017/326] Fix GitHub label color values Update the configuration for repository labels to remove the leading `#` from color values. With a `#` leading the values they are seen as invalid by the GitHub API. --- .github/labels.yml | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/labels.yml b/.github/labels.yml index b108748..500b000 100644 --- a/.github/labels.yml +++ b/.github/labels.yml @@ -2,66 +2,66 @@ # Rather than breaking up descriptions into multiline strings we disable that # specific rule in yamllint for this file. # yamllint disable rule:line-length -- color: "#eb6420" +- color: "eb6420" description: This issue or pull request is awaiting the outcome of another issue or pull request name: blocked -- color: "#000000" +- color: "000000" description: This issue or pull request involves changes to existing functionality name: breaking change -- color: "#d73a4a" +- color: "d73a4a" description: This issue or pull request addresses broken functionality name: bug -- color: "#07648d" +- color: "07648d" description: This issue will be advertised on code.gov's Open Tasks page (https://code.gov/open-tasks) name: code.gov -- color: "#0366d6" +- color: "0366d6" description: Pull requests that update a dependency file name: dependencies -- color: "#5319e7" +- color: "5319e7" description: This issue or pull request improves or adds to documentation name: documentation -- color: "#cfd3d7" +- color: "cfd3d7" description: This issue or pull request already exists or is covered in another issue or pull request name: duplicate -- color: "#b005bc" +- color: "b005bc" description: A high-level objective issue encompassing multiple issues instead of a specific unit of work name: epic -- color: "#000000" +- color: "000000" description: Pull requests that update GitHub Actions code name: github-actions -- color: "#0e8a16" +- color: "0e8a16" description: This issue or pull request is well-defined and good for newcomers name: good first issue -- color: "#ff7518" +- color: "ff7518" description: Pull request that should count toward Hacktoberfest participation name: hacktoberfest-accepted -- color: "#a2eeef" +- color: "a2eeef" description: This issue or pull request will add or improve functionality, maintainability, or ease of use name: improvement -- color: "#fef2c0" +- color: "fef2c0" description: This issue or pull request is not applicable, incorrect, or obsolete name: invalid -- color: "#ce099a" +- color: "ce099a" description: This pull request is ready to merge during the next Lineage Kraken release name: kraken 🐙 -- color: "#a4fc5d" +- color: "a4fc5d" description: This issue or pull request requires further information name: need info -- color: "#fcdb45" +- color: "fcdb45" description: This pull request is awaiting an action or decision to move forward name: on hold -- color: "#ef476c" +- color: "ef476c" description: This issue is a request for information or needs discussion name: question -- color: "#00008b" +- color: "00008b" description: This issue or pull request adds or otherwise modifies test code name: test -- color: "#1d76db" +- color: "1d76db" description: This issue or pull request pulls in upstream updates name: upstream update -- color: "#d4c5f9" +- color: "d4c5f9" description: This issue or pull request increments the version number name: version bump -- color: "#ffffff" +- color: "ffffff" description: This issue will not be incorporated name: wontfix From 8f1263549992de1edd4d4e30afae3fd7c8a8c546 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 9 Nov 2022 11:24:25 -0500 Subject: [PATCH 018/326] Correct spacing in yamllint rules There was a missing empty line in the `.yamllint` file between two rule definitions. --- .yamllint | 1 + 1 file changed, 1 insertion(+) diff --git a/.yamllint b/.yamllint index 8f4e6bf..2a119a6 100644 --- a/.yamllint +++ b/.yamllint @@ -17,6 +17,7 @@ rules: allow-non-breakable-inline-mappings: true # Allows a 10% overage from the default limit of 80 max: 88 + # yamllint doesn't like when we use yes and no for true and false, # but that's pretty standard in Ansible. truthy: disable From 663dd877b8b2849462deac74365b674d7cb858f0 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 14 Nov 2022 17:38:17 -0500 Subject: [PATCH 019/326] Add repository specific label --- .github/labels.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/labels.yml b/.github/labels.yml index 500b000..e8813ce 100644 --- a/.github/labels.yml +++ b/.github/labels.yml @@ -17,6 +17,9 @@ - color: "0366d6" description: Pull requests that update a dependency file name: dependencies +- color: "2497ed" + description: Pull requests that update Docker code + name: docker - color: "5319e7" description: This issue or pull request improves or adds to documentation name: documentation From 3aa675e339d684172e6525cabeb82786787af549 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 14 Nov 2022 17:42:20 -0500 Subject: [PATCH 020/326] Update Python setup in the test job for GHA Bump actions/setup-python from v3 to v4 and use Python 3.10 to mirror the `lint` job. --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 35e2ee4..87b5d2b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -301,9 +301,9 @@ jobs: steps: - uses: actions/checkout@v3 - id: setup-python - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: - python-version: 3.9 + python-version: "3.10" - name: Cache testing environments uses: actions/cache@v3 env: From 827ff50eff69295d4e1a0361b371116abbdaac97 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 14 Nov 2022 23:38:07 -0500 Subject: [PATCH 021/326] Remove uses of the `set-output` command Remove the remaining uses of the `set-output` command since it has been deprecated per: https://github.blog/changelog/2022-10-11-github-actions-deprecating-save-state-and-set-output-commands/ --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 87b5d2b..da15890 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -215,9 +215,9 @@ jobs: do TAGS="${TAGS},ghcr.io/${i}" done - echo ::set-output name=created::$(date -u +'%Y-%m-%dT%H:%M:%SZ') - echo ::set-output name=source_version::$(./bump_version.sh show) - echo ::set-output name=tags::${TAGS} + echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT + echo "source_version=$(./bump_version.sh show)" >> $GITHUB_OUTPUT + echo "tags=${TAGS}" >> $GITHUB_OUTPUT echo tags=${TAGS} - name: Setup tmate debug session uses: mxschmitt/action-tmate@v3 From a856426b6c321ffbef11d1b1a2503ae21193ebea Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 21 Nov 2022 13:51:53 -0500 Subject: [PATCH 022/326] Update the version of the second bandit hook The `bandit (everything else)` hook was not updated in sync with the `bandit (tests tree)` hook and is using an older version of bandit. --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e1ec95b..86c3d22 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -92,7 +92,7 @@ repos: - --config=.bandit.yml # Run bandit everything but tests directory - repo: https://github.com/PyCQA/bandit - rev: 1.7.0 + rev: 1.7.4 hooks: - id: bandit name: bandit (everything else) From 0ad31cc94043023ae72e92032362008623c13e98 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 21 Nov 2022 13:53:21 -0500 Subject: [PATCH 023/326] Correct poor English in comments for the bandit pre-commit hooks Co-authored-by: dav3r --- .pre-commit-config.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 86c3d22..a119f7c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -82,6 +82,7 @@ repos: - id: shell-lint # Python hooks + # Run bandit on the "tests" tree with a configuration - repo: https://github.com/PyCQA/bandit rev: 1.7.4 hooks: @@ -90,7 +91,7 @@ repos: files: tests args: - --config=.bandit.yml - # Run bandit everything but tests directory + # Run bandit on everything except the "tests" tree - repo: https://github.com/PyCQA/bandit rev: 1.7.4 hooks: From 0d24f9c08a6844a08d34adb616a7d49616ef1f6b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 14 Nov 2022 17:57:51 -0500 Subject: [PATCH 024/326] Update the CodeQL configuration Pull in the CodeQL workflow from cisagov/skeleton-python-library to update the CodeQL configuration for this repository. We also add github/codeql-action to the dependencies managed by this repository in the dependabot configuration. --- .github/dependabot.yml | 1 + .github/workflows/codeql-analysis.yml | 42 +++++++++++++-------------- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 9f71e38..f6eb2af 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -32,6 +32,7 @@ updates: # - dependency-name: docker/login-action # - dependency-name: docker/setup-buildx-action # - dependency-name: docker/setup-qemu-action + # - dependency-name: github/codeql-action - package-ecosystem: "pip" directory: "/" diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 33d1999..bf0d148 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -1,5 +1,4 @@ --- - # For most projects, this workflow file will not need changing; you simply need # to commit it to your repository. # @@ -11,10 +10,12 @@ on: push: # Dependabot triggered push events have read-only access, but uploading code # scanning requires write access. - branches-ignore: [dependabot/**] + branches-ignore: + - dependabot/** pull_request: # The branches below must be a subset of the branches above - branches: [develop] + branches: + - develop schedule: - cron: '0 21 * * 6' @@ -22,48 +23,45 @@ jobs: analyze: name: Analyze runs-on: ubuntu-latest - + permissions: + # required for all workflows + security-events: write strategy: fail-fast: false matrix: # Override automatic language detection by changing the below list - # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', - # 'python'] - language: ['python'] + # Supported options are go, javascript, csharp, python, cpp, and java + language: + - python # Learn more... # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v1 + uses: github/codeql-action/init@v2 with: languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a - # config file. By default, queries listed here will override any - # specified in a config file. Prefix the list here with "+" to use - # these queries and those in the config file. queries: - # ./path/to/local/query, your-org/your-repo/queries@main - # Autobuild attempts to build any compiled languages (C/C++, C#, or + # Autobuild attempts to build any compiled languages (C/C++, C#, or # Java). If this step fails, then you should remove it and run the build - # manually (see below) + # manually (see below). - name: Autobuild - uses: github/codeql-action/autobuild@v1 + uses: github/codeql-action/autobuild@v2 # ℹī¸ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following - # three lines and modify them (or add more) to build your code if your - # project uses a compiled language + # three lines and modify them (or add more) to build your code if your + # project uses a compiled language # - run: | - # make bootstrap - # make release + # make bootstrap + # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + uses: github/codeql-action/analyze@v2 From 87d0f97e200fe18d899e1d339fd0db6e36359535 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 30 Jun 2022 14:29:15 -0400 Subject: [PATCH 025/326] Remove dangling argument in the pre-commit configuration This argument references a file that is no longer found in the repository. The configuration also no longer uses any such files so it is safe to remove this as opposed to updating it instead. --- .pre-commit-config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a119f7c..b5dce48 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,7 +18,6 @@ repos: args: - --allow-missing-credentials - id: detect-private-key - exclude: src/secrets/privkey.pem - id: end-of-file-fixer exclude: files/(issue|motd) - id: mixed-line-ending From eacc4763ab864fdc35ab78d8e07dc83cca525561 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 30 Jun 2022 14:33:47 -0400 Subject: [PATCH 026/326] Update files used for Python requirements by LGTM Add all requirements*.txt files in the configuration to this file. This ensures that all requirements in use are being used for scanning. --- .lgtm.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.lgtm.yml b/.lgtm.yml index 8950263..70385b2 100644 --- a/.lgtm.yml +++ b/.lgtm.yml @@ -4,5 +4,7 @@ extraction: python_setup: version: 3 requirements_files: + - requirements-dev.txt - requirements-test.txt + - requirements.txt setup_py: false From 42f3468e16963694c8e7d3663c80a3faf9d7b1c0 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 30 Jun 2022 15:11:06 -0400 Subject: [PATCH 027/326] Restore the alerts badging for LGTM Until we completely remove LGTM from our repositories we should keep at least the minimum badging. This project does not directly use any Python, JavaScript, etc. so I am enabling the "Total alerts" badge. Other possibilities are commented out to make it easier for downstream repositories to enable what suits their configurations. --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 57f8c30..a04d771 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,11 @@ [![GitHub Build Status](https://github.com/cisagov/skeleton-docker/workflows/build/badge.svg)](https://github.com/cisagov/skeleton-docker/actions/workflows/build.yml) [![CodeQL](https://github.com/cisagov/skeleton-docker/workflows/CodeQL/badge.svg)](https://github.com/cisagov/skeleton-docker/actions/workflows/codeql-analysis.yml) [![Known Vulnerabilities](https://snyk.io/test/github/cisagov/skeleton-docker/badge.svg)](https://snyk.io/test/github/cisagov/skeleton-docker) +[![Total alerts](https://img.shields.io/lgtm/alerts/g/cisagov/skeleton-docker.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/cisagov/skeleton-docker/alerts/) + ## Docker Image ## From 5589f23d20ab012728eca11faab053d0e043189f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Nov 2022 22:59:02 +0000 Subject: [PATCH 028/326] Bump actions/github-script from 5 to 6 Bumps [actions/github-script](https://github.com/actions/github-script) from 5 to 6. - [Release notes](https://github.com/actions/github-script/releases) - [Commits](https://github.com/actions/github-script/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/github-script dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index da15890..1f3a541 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -174,7 +174,7 @@ jobs: - uses: actions/checkout@v3 - name: Gather repository metadata id: repo - uses: actions/github-script@v5 + uses: actions/github-script@v6 with: script: | const repo = await github.rest.repos.get(context.repo) From 1a8ab89ff211813ff2f7e4c40d5aabdce2968b99 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 13 Sep 2022 09:59:08 -0400 Subject: [PATCH 029/326] Fix overly match-happy sed commands We saw in cisagov/postfix-docker#47 that the sed commands in the bump_version.sh script could inadvertently match the CC0 version in the README.md file. This change escapes the periods in the version before passing it on to sed so that they only match periods and not just any character. --- bump_version.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/bump_version.sh b/bump_version.sh index a6c8ed9..963389f 100755 --- a/bump_version.sh +++ b/bump_version.sh @@ -12,6 +12,9 @@ README_FILE=README.md HELP_INFORMATION="bump_version.sh (show|major|minor|patch|prerelease|build|finalize)" old_version=$(sed -n "s/^__version__ = \"\(.*\)\"$/\1/p" $VERSION_FILE) +# Comment out periods so they are interpreted as periods and don't +# just match any character +old_version_regex=${old_version//\./\\\.} if [ $# -ne 1 ]; then echo "$HELP_INFORMATION" @@ -21,9 +24,9 @@ else new_version=$(python -c "import semver; print(semver.bump_$1('$old_version'))") echo Changing version from "$old_version" to "$new_version" tmp_file=/tmp/version.$$ - sed "s/$old_version/$new_version/" $VERSION_FILE > $tmp_file + sed "s/$old_version_regex/$new_version/" $VERSION_FILE > $tmp_file mv $tmp_file $VERSION_FILE - sed "s/$old_version/$new_version/" $README_FILE > $tmp_file + sed "s/$old_version_regex/$new_version/" $README_FILE > $tmp_file mv $tmp_file $README_FILE git add $VERSION_FILE $README_FILE git commit -m"Bump version from $old_version to $new_version" @@ -33,12 +36,12 @@ else new_version=$(python -c "import semver; print(semver.finalize_version('$old_version'))") echo Changing version from "$old_version" to "$new_version" tmp_file=/tmp/version.$$ - sed "s/$old_version/$new_version/" $VERSION_FILE > $tmp_file + sed "s/$old_version_regex/$new_version/" $VERSION_FILE > $tmp_file mv $tmp_file $VERSION_FILE - sed "s/$old_version/$new_version/" $README_FILE > $tmp_file + sed "s/$old_version_regex/$new_version/" $README_FILE > $tmp_file mv $tmp_file $README_FILE git add $VERSION_FILE $README_FILE - git commit -m"Bump version from $old_version to $new_version" + git commit -m"Finalize version from $old_version to $new_version" git push ;; show) From ba23619f6ab8a3d1a7ea65a43609024a4f9d75ca Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 4 Nov 2022 00:01:01 -0400 Subject: [PATCH 030/326] Prettify a comment --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index 248d6cf..6c50a0f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,9 +4,12 @@ FROM python:3.10.1-alpine ARG VERSION +### # For a list of pre-defined annotation keys and value types see: # https://github.com/opencontainers/image-spec/blob/master/annotations.md +# # Note: Additional labels are added by the build workflow. +### LABEL org.opencontainers.image.authors="mark.feldhousen@cisa.dhs.gov" LABEL org.opencontainers.image.vendor="Cybersecurity and Infrastructure Security Agency" From 343c0cb236fbf288f2574e6b5544000200d97e0a Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 4 Nov 2022 00:03:46 -0400 Subject: [PATCH 031/326] Rework creation of unprivileged user and installation of dependencies Also add a section to update pip and setuptools via pip. --- Dockerfile | 62 ++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 55 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6c50a0f..54bd465 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,17 +13,65 @@ ARG VERSION LABEL org.opencontainers.image.authors="mark.feldhousen@cisa.dhs.gov" LABEL org.opencontainers.image.vendor="Cybersecurity and Infrastructure Security Agency" -ARG CISA_UID=421 +### +# Unprivileged user setup variables +### +ARG CISA_GID=421 +ARG CISA_UID=${CISA_GID} +ENV CISA_USER="cisa" +ENV CISA_GROUP=${CISA_USER} ENV CISA_HOME="/home/cisa" ENV ECHO_MESSAGE="Hello World from Dockerfile" -RUN addgroup --system --gid ${CISA_UID} cisa \ - && adduser --system --uid ${CISA_UID} --ingroup cisa cisa +### +# Unprivileged user setup dependencies +# +# Install shadow, so we have adduser and addgroup. +# +# Note that we use apk --no-cache to avoid writing to a local cache. +# This results in a smaller final image, at the cost of slightly +# longer install times. +# +# Setup user dependencies are only needed for setting up the user and +# will be removed at the end of that process. +### +ENV SETUP_USER_DEPS \ + shadow +RUN apk --update --no-cache --quiet upgrade +RUN apk --no-cache --quiet add ${SETUP_USER_DEPS} + +### +# Create unprivileged user +### +RUN addgroup --system --gid ${CISA_UID} ${CISA_GROUP} \ + && adduser --system --uid ${CISA_UID} --ingroup ${CISA_GROUP} ${CISA_USER} + +### +# Remove build dependencies for unprivileged user +### +RUN apk --no-cache --quiet del ${SETUP_USER_DEPS} + +### +# Dependencies +# +# Note that we use apk --no-cache to avoid writing to a local cache. +# This results in a smaller final image, at the cost of slightly +# longer install times. +### +ENV DEPS \ + ca-certificates \ + openssl \ + py-pip +RUN apk --no-cache --quiet add ${DEPS} -RUN apk --update --no-cache add \ -ca-certificates \ -openssl \ -py-pip +### +# Make sure pip and setuptools are the latest versions +# +# Note that we use pip --no-cache-dir to avoid writing to a local +# cache. This results in a smaller final image, at the cost of +# slightly longer install times. +### +RUN pip install --no-cache-dir --upgrade pip setuptools WORKDIR ${CISA_HOME} From 717334c62cb1c972a16795d0e2bebd0ab724327f Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 4 Nov 2022 00:05:06 -0400 Subject: [PATCH 032/326] Add comments and prefer long CLI options --- Dockerfile | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 54bd465..de48f7c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -75,14 +75,24 @@ RUN pip install --no-cache-dir --upgrade pip setuptools WORKDIR ${CISA_HOME} -RUN wget -O sourcecode.tgz https://github.com/cisagov/skeleton-python-library/archive/v${VERSION}.tar.gz && \ - tar xzf sourcecode.tgz --strip-components=1 && \ - pip install --requirement requirements.txt && \ - ln -snf /run/secrets/quote.txt src/example/data/secret.txt && \ - rm sourcecode.tgz +### +# Install Python dependencies +# +# Note that we use pip --no-cache-dir to avoid writing to a local +# cache. This results in a smaller final image, at the cost of +# slightly longer install times. +### +RUN wget --output-document sourcecode.tgz \ + https://github.com/cisagov/skeleton-python-library/archive/v${VERSION}.tar.gz && \ + tar --extract --gzip --file sourcecode.tgz --strip-components=1 && \ + pip install --no-cache-dir --requirement requirements.txt && \ + ln -snf /run/secrets/quote.txt src/example/data/secret.txt && \ + rm sourcecode.tgz +### +# Prepare to run +### USER cisa - EXPOSE 8080/TCP VOLUME ["/var/log"] ENTRYPOINT ["example"] From 2a72bf612a152a0a4b32a481363cf394f5fbbf3b Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 4 Nov 2022 00:16:05 -0400 Subject: [PATCH 033/326] Move ECHO_MESSAGE declaration to the "Prepare to run" section --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index de48f7c..f4dddd9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,6 @@ ARG CISA_UID=${CISA_GID} ENV CISA_USER="cisa" ENV CISA_GROUP=${CISA_USER} ENV CISA_HOME="/home/cisa" -ENV ECHO_MESSAGE="Hello World from Dockerfile" ### # Unprivileged user setup dependencies @@ -92,6 +91,7 @@ RUN wget --output-document sourcecode.tgz \ ### # Prepare to run ### +ENV ECHO_MESSAGE="Hello World from Dockerfile" USER cisa EXPOSE 8080/TCP VOLUME ["/var/log"] From 8cf8d234f67bde8648ca60e6caa4886fdf851872 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 21 Nov 2022 21:47:09 -0500 Subject: [PATCH 034/326] Remove LGTM from the repository LGTM.com is shutting down 2022-12-16 so we should remove all LGTM-related items to prepare for disabling the LGTM.com integration in the organization. --- .lgtm.yml | 10 ---------- README.md | 5 ----- 2 files changed, 15 deletions(-) delete mode 100644 .lgtm.yml diff --git a/.lgtm.yml b/.lgtm.yml deleted file mode 100644 index 70385b2..0000000 --- a/.lgtm.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -extraction: - python: - python_setup: - version: 3 - requirements_files: - - requirements-dev.txt - - requirements-test.txt - - requirements.txt - setup_py: false diff --git a/README.md b/README.md index a04d771..57f8c30 100644 --- a/README.md +++ b/README.md @@ -3,11 +3,6 @@ [![GitHub Build Status](https://github.com/cisagov/skeleton-docker/workflows/build/badge.svg)](https://github.com/cisagov/skeleton-docker/actions/workflows/build.yml) [![CodeQL](https://github.com/cisagov/skeleton-docker/workflows/CodeQL/badge.svg)](https://github.com/cisagov/skeleton-docker/actions/workflows/codeql-analysis.yml) [![Known Vulnerabilities](https://snyk.io/test/github/cisagov/skeleton-docker/badge.svg)](https://snyk.io/test/github/cisagov/skeleton-docker) -[![Total alerts](https://img.shields.io/lgtm/alerts/g/cisagov/skeleton-docker.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/cisagov/skeleton-docker/alerts/) - ## Docker Image ## From 8c73bf429d5b7e7b0403abdc789436fadb6840e1 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 22 Nov 2022 10:25:48 -0500 Subject: [PATCH 035/326] Restore Dockerfile changes from review I accidentally clobbered these changes when I was updating cisagov/skeleton-docker#135 for merge. This restores the changes made during review as well as fixing a missed reference to the CISA_USER argument. Co-authored-by: Shane Frasier --- Dockerfile | 45 +++++++++++++++++++-------------------------- 1 file changed, 19 insertions(+), 26 deletions(-) diff --git a/Dockerfile b/Dockerfile index f4dddd9..e3cf8aa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,46 +10,39 @@ ARG VERSION # # Note: Additional labels are added by the build workflow. ### -LABEL org.opencontainers.image.authors="mark.feldhousen@cisa.dhs.gov" +# github@cisa.dhs.gov is a very generic email distribution, and it is +# unlikely that anyone on that distribution is familiar with the +# particulars of your repository. It is therefore *strongly* +# suggested that you use an email address here that is specific to the +# person or group that maintains this repository; for example: +# LABEL org.opencontainers.image.authors="vm-fusion-dev-group@trio.dhs.gov" +LABEL org.opencontainers.image.authors="github@cisa.dhs.gov" LABEL org.opencontainers.image.vendor="Cybersecurity and Infrastructure Security Agency" ### # Unprivileged user setup variables ### -ARG CISA_GID=421 -ARG CISA_UID=${CISA_GID} -ENV CISA_USER="cisa" +ARG CISA_UID=421 +ARG CISA_GID=${CISA_UID} +ARG CISA_USER="cisa" ENV CISA_GROUP=${CISA_USER} -ENV CISA_HOME="/home/cisa" +ENV CISA_HOME="/home/${CISA_USER}" ### -# Unprivileged user setup dependencies -# -# Install shadow, so we have adduser and addgroup. +# Upgrade the system # # Note that we use apk --no-cache to avoid writing to a local cache. # This results in a smaller final image, at the cost of slightly # longer install times. -# -# Setup user dependencies are only needed for setting up the user and -# will be removed at the end of that process. ### -ENV SETUP_USER_DEPS \ - shadow RUN apk --update --no-cache --quiet upgrade -RUN apk --no-cache --quiet add ${SETUP_USER_DEPS} ### # Create unprivileged user ### -RUN addgroup --system --gid ${CISA_UID} ${CISA_GROUP} \ +RUN addgroup --system --gid ${CISA_GID} ${CISA_GROUP} \ && adduser --system --uid ${CISA_UID} --ingroup ${CISA_GROUP} ${CISA_USER} -### -# Remove build dependencies for unprivileged user -### -RUN apk --no-cache --quiet del ${SETUP_USER_DEPS} - ### # Dependencies # @@ -82,17 +75,17 @@ WORKDIR ${CISA_HOME} # slightly longer install times. ### RUN wget --output-document sourcecode.tgz \ - https://github.com/cisagov/skeleton-python-library/archive/v${VERSION}.tar.gz && \ - tar --extract --gzip --file sourcecode.tgz --strip-components=1 && \ - pip install --no-cache-dir --requirement requirements.txt && \ - ln -snf /run/secrets/quote.txt src/example/data/secret.txt && \ - rm sourcecode.tgz + https://github.com/cisagov/skeleton-python-library/archive/v${VERSION}.tar.gz \ + && tar --extract --gzip --file sourcecode.tgz --strip-components=1 \ + && pip install --no-cache-dir --requirement requirements.txt \ + && ln -snf /run/secrets/quote.txt src/example/data/secret.txt \ + && rm sourcecode.tgz ### # Prepare to run ### ENV ECHO_MESSAGE="Hello World from Dockerfile" -USER cisa +USER ${CISA_USER} EXPOSE 8080/TCP VOLUME ["/var/log"] ENTRYPOINT ["example"] From 0bb91c5654db04ad15182dc8c0e491ae0664394f Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 31 Jan 2023 13:06:42 -0500 Subject: [PATCH 036/326] Update pre-commit hooks Update pre-commit hooks using `pre-commit autoupdate`. The `ansible-lint` hook is intentionally held back to be updated independently to v6. --- .pre-commit-config.yaml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 25f8d1d..2764bb4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ default_language_version: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.3.0 + rev: v4.4.0 hooks: - id: check-case-conflict - id: check-executables-have-shebangs @@ -31,7 +31,7 @@ repos: # Text file hooks - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.32.2 + rev: v0.33.0 hooks: - id: markdownlint args: @@ -41,7 +41,7 @@ repos: hooks: - id: prettier - repo: https://github.com/adrienverge/yamllint - rev: v1.28.0 + rev: v1.29.0 hooks: - id: yamllint args: @@ -49,14 +49,14 @@ repos: # GitHub Actions hooks - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.18.4 + rev: 0.21.0 hooks: - id: check-github-actions - id: check-github-workflows # pre-commit hooks - repo: https://github.com/pre-commit/pre-commit - rev: v2.20.0 + rev: v3.0.2 hooks: - id: validate_manifest @@ -88,25 +88,25 @@ repos: args: - --config=.bandit.yml - repo: https://github.com/psf/black - rev: 22.10.0 + rev: 22.12.0 hooks: - id: black - repo: https://github.com/PyCQA/flake8 - rev: 5.0.4 + rev: 6.0.0 hooks: - id: flake8 additional_dependencies: - flake8-docstrings - repo: https://github.com/PyCQA/isort - rev: 5.10.1 + rev: 5.12.0 hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.990 + rev: v0.991 hooks: - id: mypy - repo: https://github.com/asottile/pyupgrade - rev: v3.2.0 + rev: v3.3.1 hooks: - id: pyupgrade @@ -119,7 +119,7 @@ repos: # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.76.0 + rev: v1.77.0 hooks: - id: terraform_fmt - id: terraform_validate From 313e79263e8b6a1fbd311e5b9543a2b9a99ab93b Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Thu, 2 Feb 2023 13:56:59 -0500 Subject: [PATCH 037/326] Add a security label --- .github/labels.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/labels.yml b/.github/labels.yml index 500b000..5b16492 100644 --- a/.github/labels.yml +++ b/.github/labels.yml @@ -53,6 +53,9 @@ - color: "ef476c" description: This issue is a request for information or needs discussion name: question +- color: "d73a4a" + description: This issue or pull request addresses a security issue + name: security - color: "00008b" description: This issue or pull request adds or otherwise modifies test code name: test From da15398ef27c2022f4fd1e88215cc0a72c9751b7 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 6 Dec 2022 11:24:07 -0500 Subject: [PATCH 038/326] Add package write permission to workflow When Dependabot creates a PR it requires this permission in order to push Docker images to ghcr.io. --- .github/workflows/build.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1f3a541..af07e26 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -345,6 +345,10 @@ jobs: runs-on: ubuntu-latest needs: [lint, prepare, test] if: github.event_name != 'pull_request' + # When Dependabot creates a PR it requires this permission in + # order to push Docker images to ghcr.io. + permissions: + packages: write steps: - name: Login to Docker Hub uses: docker/login-action@v2 From 4e5287fe05d6ba47aedce58cc851a69fe495e769 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Sat, 3 Dec 2022 15:05:50 -0500 Subject: [PATCH 039/326] Update wheel along with pip and setuptools Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- Dockerfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index e3cf8aa..3635be0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -57,13 +57,16 @@ ENV DEPS \ RUN apk --no-cache --quiet add ${DEPS} ### -# Make sure pip and setuptools are the latest versions +# Make sure pip, setuptools, and wheel are the latest versions # # Note that we use pip --no-cache-dir to avoid writing to a local # cache. This results in a smaller final image, at the cost of # slightly longer install times. ### -RUN pip install --no-cache-dir --upgrade pip setuptools +RUN pip install --no-cache-dir --upgrade \ + pip \ + setuptools \ + wheel WORKDIR ${CISA_HOME} From 83beae5295821391ffc88f7190514b0881520b95 Mon Sep 17 00:00:00 2001 From: Jeremy Morrow Date: Thu, 9 Feb 2023 21:24:50 -0500 Subject: [PATCH 040/326] Specify pip3 In this case it doesn't matter because we are starting from a Python3-specific base container, but other projects that use this skeleton may not be. Specifying pip3 ensures that the Python 2 version of pip is not called by mistake. --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3635be0..9f4e9c2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -59,7 +59,7 @@ RUN apk --no-cache --quiet add ${DEPS} ### # Make sure pip, setuptools, and wheel are the latest versions # -# Note that we use pip --no-cache-dir to avoid writing to a local +# Note that we use pip3 --no-cache-dir to avoid writing to a local # cache. This results in a smaller final image, at the cost of # slightly longer install times. ### @@ -73,14 +73,14 @@ WORKDIR ${CISA_HOME} ### # Install Python dependencies # -# Note that we use pip --no-cache-dir to avoid writing to a local +# Note that we use pip3 --no-cache-dir to avoid writing to a local # cache. This results in a smaller final image, at the cost of # slightly longer install times. ### RUN wget --output-document sourcecode.tgz \ https://github.com/cisagov/skeleton-python-library/archive/v${VERSION}.tar.gz \ && tar --extract --gzip --file sourcecode.tgz --strip-components=1 \ - && pip install --no-cache-dir --requirement requirements.txt \ + && pip3 install --no-cache-dir --requirement requirements.txt \ && ln -snf /run/secrets/quote.txt src/example/data/secret.txt \ && rm sourcecode.tgz From d6bf3c0e641a60def66cefea4d3185b540a7eae2 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 22 Nov 2022 12:13:43 -0500 Subject: [PATCH 041/326] Specify the default group to run under in the Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9f4e9c2..b32ee0f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -88,7 +88,7 @@ RUN wget --output-document sourcecode.tgz \ # Prepare to run ### ENV ECHO_MESSAGE="Hello World from Dockerfile" -USER ${CISA_USER} +USER ${CISA_USER}:${CISA_GROUP} EXPOSE 8080/TCP VOLUME ["/var/log"] ENTRYPOINT ["example"] From 73a526f57f0756ac2d622d0e714a76515344f5f0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Feb 2023 04:55:35 +0000 Subject: [PATCH 042/326] Bump docker/build-push-action from 3 to 4 Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 3 to 4. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v3...v4) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index af07e26..08d67c7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -248,7 +248,7 @@ jobs: run: mkdir -p dist - name: Build image id: docker_build - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v4 with: build-args: | VERSION=${{ needs.prepare.outputs.source_version }} @@ -380,7 +380,7 @@ jobs: run: ./buildx-dockerfile.sh - name: Build and push platform images to registries id: docker_build - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v4 with: build-args: | VERSION=${{ needs.prepare.outputs.source_version }} From 7aab0747e550ee1b81c016f171f2fcb5f26f42e0 Mon Sep 17 00:00:00 2001 From: Jeremy Morrow Date: Fri, 10 Feb 2023 10:26:10 -0500 Subject: [PATCH 043/326] Specify pip3 in the file where it got blown away by a rebase --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b32ee0f..1516787 100644 --- a/Dockerfile +++ b/Dockerfile @@ -63,7 +63,7 @@ RUN apk --no-cache --quiet add ${DEPS} # cache. This results in a smaller final image, at the cost of # slightly longer install times. ### -RUN pip install --no-cache-dir --upgrade \ +RUN pip3 install --no-cache-dir --upgrade \ pip \ setuptools \ wheel From 744f07eb7497619ef4124bfe20ef8a6d2d071fb5 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 24 Mar 2023 11:06:27 -0400 Subject: [PATCH 044/326] Add @jasonodoom as a default codeowner --- .github/CODEOWNERS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 371258c..8f5c8c5 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,8 +3,8 @@ # These owners will be the default owners for everything in the # repo. Unless a later match takes precedence, these owners will be # requested for review when someone opens a pull request. -* @dav3r @felddy @jsf9k @mcdonnnj +* @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj # These folks own any files in the .github directory at the root of # the repository and any of its subdirectories. -/.github/ @dav3r @felddy @jsf9k @mcdonnnj +/.github/ @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj From 8d39c8b402fc24084d57a27f33dada98832cced6 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 19 Apr 2023 11:56:30 -0400 Subject: [PATCH 045/326] Use Python 3.11 for the `lint` job in the `build` workflow This is the latest minor release of Python so it makes sense to use it as the default for this job. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8d17421..dc14a7d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,7 +23,7 @@ jobs: - id: setup-python uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.11" # We need the Go version and Go cache location for the actions/cache step, # so the Go installation must happen before that. - id: setup-go From db5e7ffebaa44d240b1fb333bef5e34512115aa8 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 27 Mar 2023 08:37:24 -0400 Subject: [PATCH 046/326] Update pre-commit hook versions This is done automatically with the `pre-commit autoupdate` command. However the `ansible-lint` hook is manually kept back as we have not tested functionality to confirm that our roles will generally pass with the new version. --- .pre-commit-config.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2764bb4..0dabeb6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,11 +37,11 @@ repos: args: - --config=.mdl_config.yaml - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.0.0-alpha.4 + rev: v3.0.0-alpha.6 hooks: - id: prettier - repo: https://github.com/adrienverge/yamllint - rev: v1.29.0 + rev: v1.30.0 hooks: - id: yamllint args: @@ -49,14 +49,14 @@ repos: # GitHub Actions hooks - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.21.0 + rev: 0.22.0 hooks: - id: check-github-actions - id: check-github-workflows # pre-commit hooks - repo: https://github.com/pre-commit/pre-commit - rev: v3.0.2 + rev: v3.2.1 hooks: - id: validate_manifest @@ -82,13 +82,13 @@ repos: # Python hooks - repo: https://github.com/PyCQA/bandit - rev: 1.7.4 + rev: 1.7.5 hooks: - id: bandit args: - --config=.bandit.yml - repo: https://github.com/psf/black - rev: 22.12.0 + rev: 23.1.0 hooks: - id: black - repo: https://github.com/PyCQA/flake8 @@ -102,7 +102,7 @@ repos: hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.991 + rev: v1.1.1 hooks: - id: mypy - repo: https://github.com/asottile/pyupgrade @@ -119,7 +119,7 @@ repos: # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.77.0 + rev: v1.77.1 hooks: - id: terraform_fmt - id: terraform_validate From 4c595e68e41d663fe934124fcbeb289747cb1d54 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Mar 2023 15:54:24 +0000 Subject: [PATCH 047/326] Bump actions/setup-go from 3 to 4 Bumps [actions/setup-go](https://github.com/actions/setup-go) from 3 to 4. - [Release notes](https://github.com/actions/setup-go/releases) - [Commits](https://github.com/actions/setup-go/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/setup-go dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index dc14a7d..269122b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,7 +27,7 @@ jobs: # We need the Go version and Go cache location for the actions/cache step, # so the Go installation must happen before that. - id: setup-go - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: go-version: "1.19" - name: Lookup Go cache directory From 9984792a22f733d33421b7b6cc77f0fb44ca8d7c Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 24 Feb 2023 11:17:03 -0500 Subject: [PATCH 048/326] Install/upgrade setuptools and wheel when upgrading pip When wheel gets installed alongside other packages, it may not get used when those other packages are installed. When that happens I see warnings like this: DEPRECATION: ansible-core is being installed using the legacy 'setup.py install' method, because it does not have a 'pyproject.toml' and the 'wheel' package is not installed. pip 23.1 will enforce this behaviour change. A possible replacement is to enable the '--use-pep517' option. Discussion can be found at https://github.com/pypa/pip/issues/8559 This change should get rid of these warnings. Nota bene: This is the practice we follow in the Dockerfile in cisagov/skeleton-docker, but for some reason we never started using it in our workflows. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 269122b..bdd8c98 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -89,7 +89,7 @@ jobs: run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - name: Install dependencies run: | - python -m pip install --upgrade pip + python -m pip install --upgrade pip setuptools wheel pip install --upgrade --requirement requirements-test.txt - name: Set up pre-commit hook environments run: pre-commit install-hooks From a0bc20bb08292f0d532f52378b8af4f585c94427 Mon Sep 17 00:00:00 2001 From: Jason Odoom Date: Fri, 5 May 2023 14:00:52 -0400 Subject: [PATCH 049/326] Add Go hooks to support go code for our projects --- .pre-commit-config.yaml | 56 +++++++++++++++++++++++++++++++++-------- 1 file changed, 45 insertions(+), 11 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0dabeb6..65c676b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,17 +31,17 @@ repos: # Text file hooks - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.33.0 + rev: v0.34.0 hooks: - id: markdownlint args: - --config=.mdl_config.yaml - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.0.0-alpha.6 + rev: v3.0.0-alpha.9-for-vscode hooks: - id: prettier - repo: https://github.com/adrienverge/yamllint - rev: v1.30.0 + rev: v1.31.0 hooks: - id: yamllint args: @@ -49,17 +49,51 @@ repos: # GitHub Actions hooks - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.22.0 + rev: 0.23.0 hooks: - id: check-github-actions - id: check-github-workflows # pre-commit hooks - repo: https://github.com/pre-commit/pre-commit - rev: v3.2.1 + rev: v3.3.1 hooks: - id: validate_manifest + # Go hooks + - repo: https://github.com/TekWizely/pre-commit-golang + rev: v1.0.0-rc.1 + hooks: + # Style Checkers + - id: go-lint + - id: go-critic + # Go Build + - id: go-build-mod + - id: go-build-pkg + - id: go-build-repo-mod + - id: go-build-repo-pkg + + # Go Mod Tidy + - id: go-mod-tidy + - id: go-mod-tidy-repo + + # Go Test + - id: go-test-mod + - id: go-test-pkg + - id: go-test-repo-mod + - id: go-test-repo-pkg + + # Go Vet + - id: go-vet + - id: go-vet-mod + - id: go-vet-pkg + - id: go-vet-repo-mod + - id: go-vet-repo-pkg + # GoSec + - id: go-sec-mod + - id: go-sec-pkg + - id: go-sec-repo-mod + - id: go-sec-repo-pkg # Shell script hooks - repo: https://github.com/cisagov/pre-commit-shfmt rev: v0.0.2 @@ -88,7 +122,7 @@ repos: args: - --config=.bandit.yml - repo: https://github.com/psf/black - rev: 23.1.0 + rev: 23.3.0 hooks: - id: black - repo: https://github.com/PyCQA/flake8 @@ -102,31 +136,31 @@ repos: hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.1.1 + rev: v1.2.0 hooks: - id: mypy - repo: https://github.com/asottile/pyupgrade - rev: v3.3.1 + rev: v3.3.2 hooks: - id: pyupgrade # Ansible hooks - repo: https://github.com/ansible-community/ansible-lint - rev: v5.4.0 + rev: v6.15.0 hooks: - id: ansible-lint # files: molecule/default/playbook.yml # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.77.1 + rev: v1.78.0 hooks: - id: terraform_fmt - id: terraform_validate # Docker hooks - repo: https://github.com/IamTheFij/docker-pre-commit - rev: v2.1.1 + rev: v3.0.1 hooks: - id: docker-compose-check From 3628e5fa20d334723ca4733a873110b52b07b406 Mon Sep 17 00:00:00 2001 From: Jason Odoom Date: Fri, 5 May 2023 14:08:25 -0400 Subject: [PATCH 050/326] Add Staticcheck --- .pre-commit-config.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 65c676b..b04a130 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -67,6 +67,11 @@ repos: # Style Checkers - id: go-lint - id: go-critic + # StaticCheck + - id: go-staticcheck-mod + - id: go-staticcheck-pkg + - id: go-staticcheck-repo-mod + - id: go-staticcheck-repo-pkg # Go Build - id: go-build-mod - id: go-build-pkg From 668724f0a5fc2742a5fd9ce7de16753664876d02 Mon Sep 17 00:00:00 2001 From: Jason Odoom Date: Fri, 5 May 2023 16:07:08 -0400 Subject: [PATCH 051/326] Fix indentation Co-authored-by: Shane Frasier --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b04a130..10d005e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -64,7 +64,7 @@ repos: - repo: https://github.com/TekWizely/pre-commit-golang rev: v1.0.0-rc.1 hooks: - # Style Checkers + # Style Checkers - id: go-lint - id: go-critic # StaticCheck From 3d422b8493f09867ce17b2254ba25678a9759d03 Mon Sep 17 00:00:00 2001 From: Jason Odoom Date: Fri, 5 May 2023 16:07:17 -0400 Subject: [PATCH 052/326] Fix indentation Co-authored-by: Shane Frasier --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 10d005e..20e57fa 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -67,7 +67,7 @@ repos: # Style Checkers - id: go-lint - id: go-critic - # StaticCheck + # StaticCheck - id: go-staticcheck-mod - id: go-staticcheck-pkg - id: go-staticcheck-repo-mod From 5571f4756c1499d29da617b6f53afe8bfaebcf8d Mon Sep 17 00:00:00 2001 From: Jason Odoom Date: Fri, 5 May 2023 16:07:25 -0400 Subject: [PATCH 053/326] Fix indentation Co-authored-by: Shane Frasier --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 20e57fa..9d698a8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -72,7 +72,7 @@ repos: - id: go-staticcheck-pkg - id: go-staticcheck-repo-mod - id: go-staticcheck-repo-pkg - # Go Build + # Go Build - id: go-build-mod - id: go-build-pkg - id: go-build-repo-mod From 4fd9501f753e1789261307b9e3829dd19dd3d42f Mon Sep 17 00:00:00 2001 From: Jason Odoom Date: Fri, 5 May 2023 16:07:44 -0400 Subject: [PATCH 054/326] Fix formatting Co-authored-by: Shane Frasier --- .pre-commit-config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9d698a8..4946d17 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -77,7 +77,6 @@ repos: - id: go-build-pkg - id: go-build-repo-mod - id: go-build-repo-pkg - # Go Mod Tidy - id: go-mod-tidy - id: go-mod-tidy-repo From 412101aff6363f06a9155fa21679637d5d5ca7dd Mon Sep 17 00:00:00 2001 From: Jason Odoom Date: Fri, 5 May 2023 16:08:16 -0400 Subject: [PATCH 055/326] Fix formatting Co-authored-by: Shane Frasier --- .pre-commit-config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4946d17..837d28b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -86,7 +86,6 @@ repos: - id: go-test-pkg - id: go-test-repo-mod - id: go-test-repo-pkg - # Go Vet - id: go-vet - id: go-vet-mod From a346fb5a789040104624eb0b193460463a8d811a Mon Sep 17 00:00:00 2001 From: Jason Odoom Date: Fri, 5 May 2023 16:10:19 -0400 Subject: [PATCH 056/326] Fix formatting Co-authored-by: Shane Frasier --- .pre-commit-config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 837d28b..9ac7fd1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -80,7 +80,6 @@ repos: # Go Mod Tidy - id: go-mod-tidy - id: go-mod-tidy-repo - # Go Test - id: go-test-mod - id: go-test-pkg From c47038b51d4b537b9bd64b0299c1b99ca0b6e455 Mon Sep 17 00:00:00 2001 From: Jason Odoom Date: Fri, 5 May 2023 16:19:50 -0400 Subject: [PATCH 057/326] Revert updated versions for existing hooks --- .pre-commit-config.yaml | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9ac7fd1..d14f75a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,17 +31,17 @@ repos: # Text file hooks - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.34.0 + rev: v0.33.0 hooks: - id: markdownlint args: - --config=.mdl_config.yaml - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.0.0-alpha.9-for-vscode + rev: v3.0.0-alpha.6 hooks: - id: prettier - repo: https://github.com/adrienverge/yamllint - rev: v1.31.0 + rev: v1.30.0 hooks: - id: yamllint args: @@ -49,14 +49,14 @@ repos: # GitHub Actions hooks - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.23.0 + rev: 0.22.0 hooks: - id: check-github-actions - id: check-github-workflows # pre-commit hooks - repo: https://github.com/pre-commit/pre-commit - rev: v3.3.1 + rev: v3.2.1 hooks: - id: validate_manifest @@ -96,6 +96,7 @@ repos: - id: go-sec-pkg - id: go-sec-repo-mod - id: go-sec-repo-pkg + # Shell script hooks - repo: https://github.com/cisagov/pre-commit-shfmt rev: v0.0.2 @@ -124,7 +125,7 @@ repos: args: - --config=.bandit.yml - repo: https://github.com/psf/black - rev: 23.3.0 + rev: 23.1.0 hooks: - id: black - repo: https://github.com/PyCQA/flake8 @@ -138,31 +139,31 @@ repos: hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.2.0 + rev: v1.1.1 hooks: - id: mypy - repo: https://github.com/asottile/pyupgrade - rev: v3.3.2 + rev: v3.3.1 hooks: - id: pyupgrade # Ansible hooks - repo: https://github.com/ansible-community/ansible-lint - rev: v6.15.0 + rev: v5.4.0 hooks: - id: ansible-lint # files: molecule/default/playbook.yml # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.78.0 + rev: v1.77.1 hooks: - id: terraform_fmt - id: terraform_validate # Docker hooks - repo: https://github.com/IamTheFij/docker-pre-commit - rev: v3.0.1 + rev: v2.1.1 hooks: - id: docker-compose-check @@ -171,4 +172,4 @@ repos: rev: v0.0.2 hooks: - id: packer_validate - - id: packer_fmt + - id: packer_fmt \ No newline at end of file From eb52678f773e6f05857b6880b34b9f8afdb71dc5 Mon Sep 17 00:00:00 2001 From: Jason Odoom Date: Fri, 5 May 2023 16:30:11 -0400 Subject: [PATCH 058/326] Fix formatting --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d14f75a..6e8143b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -172,4 +172,4 @@ repos: rev: v0.0.2 hooks: - id: packer_validate - - id: packer_fmt \ No newline at end of file + - id: packer_fmt From f8179f1aa31be65b2c363fd86badd6bf7883a033 Mon Sep 17 00:00:00 2001 From: Jason Odoom Date: Fri, 5 May 2023 17:16:19 -0400 Subject: [PATCH 059/326] Remove extra hook ids --- .pre-commit-config.yaml | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6e8143b..0810309 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -69,33 +69,16 @@ repos: - id: go-critic # StaticCheck - id: go-staticcheck-mod - - id: go-staticcheck-pkg - - id: go-staticcheck-repo-mod - - id: go-staticcheck-repo-pkg # Go Build - id: go-build-mod - - id: go-build-pkg - - id: go-build-repo-mod - - id: go-build-repo-pkg # Go Mod Tidy - id: go-mod-tidy - - id: go-mod-tidy-repo # Go Test - id: go-test-mod - - id: go-test-pkg - - id: go-test-repo-mod - - id: go-test-repo-pkg # Go Vet - id: go-vet - - id: go-vet-mod - - id: go-vet-pkg - - id: go-vet-repo-mod - - id: go-vet-repo-pkg # GoSec - id: go-sec-mod - - id: go-sec-pkg - - id: go-sec-repo-mod - - id: go-sec-repo-pkg # Shell script hooks - repo: https://github.com/cisagov/pre-commit-shfmt From 42820e2e7d1fb492be77faa313de934f1c472cd2 Mon Sep 17 00:00:00 2001 From: Jason Odoom Date: Mon, 8 May 2023 09:41:46 -0400 Subject: [PATCH 060/326] Replace hooks with repo based option when available --- .pre-commit-config.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0810309..3a798f0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -68,17 +68,17 @@ repos: - id: go-lint - id: go-critic # StaticCheck - - id: go-staticcheck-mod + - id: go-staticcheck-repo-mod # Go Build - - id: go-build-mod + - id: go-build-repo-mod # Go Mod Tidy - - id: go-mod-tidy + - id: go-mod-tidy-repo # Go Test - - id: go-test-mod + - id: go-test-repo-mod # Go Vet - - id: go-vet + - id: go-vet-repo-mod # GoSec - - id: go-sec-mod + - id: go-sec-repo-mod # Shell script hooks - repo: https://github.com/cisagov/pre-commit-shfmt From 7f372e63ac7875b3fb8d25dd094f7b9183e6e42e Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 7 Jun 2023 11:11:13 -0400 Subject: [PATCH 061/326] Remove the go-lint hook from the pre-commit configuration The golang/lint tool was archived on May 9th, 2021 and based on golang/go#38968 no future work is planned. Coupled with the fact that it is not available from brew we are removing this hook as local development may be hindered by trying to satisfy running this hook. Co-authored-by: Shane Frasier Co-authored-by: dav3r --- .pre-commit-config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3a798f0..4f826b1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -65,7 +65,6 @@ repos: rev: v1.0.0-rc.1 hooks: # Style Checkers - - id: go-lint - id: go-critic # StaticCheck - id: go-staticcheck-repo-mod From b8231113bac671a5d9f026fc519e9d3088eda935 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 31 May 2023 12:45:10 -0400 Subject: [PATCH 062/326] Update pre-commit hook versions This is done automatically with the `pre-commit autoupdate` command. --- .pre-commit-config.yaml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0dabeb6..724b3e6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,17 +31,17 @@ repos: # Text file hooks - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.33.0 + rev: v0.34.0 hooks: - id: markdownlint args: - --config=.mdl_config.yaml - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.0.0-alpha.6 + rev: v3.0.0-alpha.9-for-vscode hooks: - id: prettier - repo: https://github.com/adrienverge/yamllint - rev: v1.30.0 + rev: v1.32.0 hooks: - id: yamllint args: @@ -49,14 +49,14 @@ repos: # GitHub Actions hooks - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.22.0 + rev: 0.23.1 hooks: - id: check-github-actions - id: check-github-workflows # pre-commit hooks - repo: https://github.com/pre-commit/pre-commit - rev: v3.2.1 + rev: v3.3.2 hooks: - id: validate_manifest @@ -88,7 +88,7 @@ repos: args: - --config=.bandit.yml - repo: https://github.com/psf/black - rev: 23.1.0 + rev: 23.3.0 hooks: - id: black - repo: https://github.com/PyCQA/flake8 @@ -102,31 +102,31 @@ repos: hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.1.1 + rev: v1.3.0 hooks: - id: mypy - repo: https://github.com/asottile/pyupgrade - rev: v3.3.1 + rev: v3.4.0 hooks: - id: pyupgrade # Ansible hooks - repo: https://github.com/ansible-community/ansible-lint - rev: v5.4.0 + rev: v6.17.0 hooks: - id: ansible-lint # files: molecule/default/playbook.yml # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.77.1 + rev: v1.80.0 hooks: - id: terraform_fmt - id: terraform_validate # Docker hooks - repo: https://github.com/IamTheFij/docker-pre-commit - rev: v2.1.1 + rev: v3.0.1 hooks: - id: docker-compose-check From b770fc687a7a954d5e0cc12ae264fcda62a3355a Mon Sep 17 00:00:00 2001 From: Jason Odoom Date: Wed, 31 May 2023 12:29:10 -0400 Subject: [PATCH 063/326] Add apostrophe to Qapla --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index f526cdb..436d163 100755 --- a/setup-env +++ b/setup-env @@ -186,5 +186,5 @@ else: END_OF_LINE )" -# Qapla +# Qapla' echo "Success!" From 24a0008be136a946c2d742542ddb78dceb128073 Mon Sep 17 00:00:00 2001 From: arcsector <26469747+arcsector@users.noreply.github.com> Date: Wed, 17 May 2023 16:01:54 -0700 Subject: [PATCH 064/326] detecting virtualenv installation successfully --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 436d163..129102f 100755 --- a/setup-env +++ b/setup-env @@ -65,7 +65,7 @@ done eval set -- "$PARAMS" # Check to see if pyenv is installed -if [ -z "$(command -v pyenv)" ] || [ -z "$(command -v pyenv-virtualenv)" ]; then +if [ -z "$(command -v pyenv)" ] || { [ -z "$(command -v pyenv-virtualenv)" ] && [ ! -f "$(pyenv root)/plugins/pyenv-virtualenv/bin/pyenv-virtualenv" ] ; }; then echo "pyenv and pyenv-virtualenv are required." if [[ "$OSTYPE" == "darwin"* ]]; then cat << 'END_OF_LINE' From f01965a1e56c2a6743eec95e9daa724037e7699d Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 19 May 2023 11:46:28 -0400 Subject: [PATCH 065/326] Whitespace change to make shfmt linter happy --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 129102f..77926bf 100755 --- a/setup-env +++ b/setup-env @@ -65,7 +65,7 @@ done eval set -- "$PARAMS" # Check to see if pyenv is installed -if [ -z "$(command -v pyenv)" ] || { [ -z "$(command -v pyenv-virtualenv)" ] && [ ! -f "$(pyenv root)/plugins/pyenv-virtualenv/bin/pyenv-virtualenv" ] ; }; then +if [ -z "$(command -v pyenv)" ] || { [ -z "$(command -v pyenv-virtualenv)" ] && [ ! -f "$(pyenv root)/plugins/pyenv-virtualenv/bin/pyenv-virtualenv" ]; }; then echo "pyenv and pyenv-virtualenv are required." if [[ "$OSTYPE" == "darwin"* ]]; then cat << 'END_OF_LINE' From 43d588fe56c9522535bc341a4664569be798c6c4 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 25 Apr 2023 09:17:14 -0400 Subject: [PATCH 066/326] Fix two broken links Resolves cisagov/skeleton-generic#131. --- .ansible-lint | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.ansible-lint b/.ansible-lint index 0e80b05..4ffc0ef 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -1,10 +1,9 @@ --- -# See https://ansible-lint.readthedocs.io/en/latest/configuring.html -# for a list of the configuration elements that can exist in this -# file. +# See https://ansible-lint.readthedocs.io/configuring/ for a list of +# the configuration elements that can exist in this file. enable_list: # Useful checks that one must opt-into. See here for more details: - # https://ansible-lint.readthedocs.io/en/latest/rules.html + # https://ansible-lint.readthedocs.io/rules/ - fcqn-builtins - no-log-password - no-same-owner From e0982552895542bd267696744548aeba18fee2d5 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 31 May 2023 13:39:14 -0400 Subject: [PATCH 067/326] Disable caching for the setup-go Action The cache key used relies on the existence of a go.sum file. Since we have no expectation for Go source code, including that file, and since we already include the Go cache in our job caching, we can safely disable caching in the Action. --- .github/workflows/build.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bdd8c98..4351bd2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,6 +29,9 @@ jobs: - id: setup-go uses: actions/setup-go@v4 with: + # There is no expectation for actual Go code so we disable caching as + # it relies on the existence of a go.sum file. + cache: false go-version: "1.19" - name: Lookup Go cache directory id: go-cache From edf133832040ef12c407b0772c40f18f64076eb8 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 31 May 2023 13:41:03 -0400 Subject: [PATCH 068/326] Bump Go from 1.19 to 1.20 Bump the version of Go used in our GitHub Actions configuration to the latest stable Go release. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4351bd2..c992e8e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,7 +32,7 @@ jobs: # There is no expectation for actual Go code so we disable caching as # it relies on the existence of a go.sum file. cache: false - go-version: "1.19" + go-version: "1.20" - name: Lookup Go cache directory id: go-cache run: | From 4cb9a4b5863abc317de4daa8fa72487ac930c3d0 Mon Sep 17 00:00:00 2001 From: Jason Odoom Date: Mon, 10 Jul 2023 12:52:36 -0400 Subject: [PATCH 069/326] Add steps to setup go packages for pre-commit --- .github/workflows/build.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c992e8e..f07c640 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -33,6 +33,22 @@ jobs: # it relies on the existence of a go.sum file. cache: false go-version: "1.20" + - id: setup-go-critic + run: go install -v github.com/go-critic/go-critic/cmd/gocritic@latest + - id: setup staticheck + uses: actions/checkout@v2 + with: + fetch-depth: 1 + - name: Run staticcheck + uses: dominikh/staticcheck-action@v1.3.0 + with: + version: "2022.1.3" + - id: setup-go-sec + uses: actions/checkout@v3 + env: + GO111MODULE: on + - name: Run Gosec Security Scanner + uses: securego/gosec@master - name: Lookup Go cache directory id: go-cache run: | From 53f56b01103783ba2d71c7ad64dc5c4c390c8388 Mon Sep 17 00:00:00 2001 From: Jason Odoom Date: Mon, 10 Jul 2023 18:01:24 -0400 Subject: [PATCH 070/326] Update action/checkout to use same version --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f07c640..4731169 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -36,7 +36,7 @@ jobs: - id: setup-go-critic run: go install -v github.com/go-critic/go-critic/cmd/gocritic@latest - id: setup staticheck - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 1 - name: Run staticcheck From f2610994eacc942a8b4a3771c0c0d8d69fb286c1 Mon Sep 17 00:00:00 2001 From: Jason Odoom Date: Mon, 10 Jul 2023 18:17:46 -0400 Subject: [PATCH 071/326] Set specific gocritic version --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4731169..cfbc847 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,7 +34,7 @@ jobs: cache: false go-version: "1.20" - id: setup-go-critic - run: go install -v github.com/go-critic/go-critic/cmd/gocritic@latest + run: go install -v github.com/go-critic/go-critic/cmd/gocritic@0.8.1 - id: setup staticheck uses: actions/checkout@v3 with: From 097b573a1eba564d2aeb260f676f01776ff10aba Mon Sep 17 00:00:00 2001 From: Jason Odoom Date: Mon, 10 Jul 2023 19:34:48 -0400 Subject: [PATCH 072/326] Update build workflow with package installtion --- .github/workflows/build.yml | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cfbc847..3cab3fb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -33,22 +33,6 @@ jobs: # it relies on the existence of a go.sum file. cache: false go-version: "1.20" - - id: setup-go-critic - run: go install -v github.com/go-critic/go-critic/cmd/gocritic@0.8.1 - - id: setup staticheck - uses: actions/checkout@v3 - with: - fetch-depth: 1 - - name: Run staticcheck - uses: dominikh/staticcheck-action@v1.3.0 - with: - version: "2022.1.3" - - id: setup-go-sec - uses: actions/checkout@v3 - env: - GO111MODULE: on - - name: Run Gosec Security Scanner - uses: securego/gosec@master - name: Lookup Go cache directory id: go-cache run: | @@ -80,6 +64,16 @@ jobs: ${{ env.BASE_CACHE_KEY }} - name: Setup curl cache run: mkdir -p ${{ env.CURL_CACHE_DIR }} + - name: Install GoCritic + env: + PACKAGE_URL: github.com/go-critic/go-critic/cmd/gocritic + PACKAGE_VERSION: ${{ steps.setup-env.outputs.go-critic-version }} + run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} + - name: Install GoSec + env: + PACKAGE_URL: github.com/securego/gosec/v2/cmd/gosec + PACKAGE_VERSION: ${{ steps.setup-env.outputs.gosec-version }} + run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - name: Install Packer env: PACKER_VERSION: ${{ steps.setup-env.outputs.packer-version }} @@ -101,6 +95,11 @@ jobs: PACKAGE_URL: mvdan.cc/sh/v3/cmd/shfmt PACKAGE_VERSION: ${{ steps.setup-env.outputs.shfmt-version }} run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} + - name: Install Staticcheck + env: + PACKAGE_URL: dominikh/staticcheck-action + PACKAGE_VERSION: ${{ steps.setup-env.outputs.go-staticcheck-version }} + run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - name: Install Terraform-docs env: PACKAGE_URL: github.com/terraform-docs/terraform-docs From 254282a5f9c5c55c2e95459f929e41a1d85e9ae2 Mon Sep 17 00:00:00 2001 From: Jason Odoom Date: Tue, 11 Jul 2023 15:06:07 -0400 Subject: [PATCH 073/326] Group go tools together with comment --- .github/workflows/build.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3cab3fb..a3c90ea 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -64,6 +64,7 @@ jobs: ${{ env.BASE_CACHE_KEY }} - name: Setup curl cache run: mkdir -p ${{ env.CURL_CACHE_DIR }} + # Install GoCrtitic, GoSec and Staticcheck Go tools - name: Install GoCritic env: PACKAGE_URL: github.com/go-critic/go-critic/cmd/gocritic @@ -74,6 +75,11 @@ jobs: PACKAGE_URL: github.com/securego/gosec/v2/cmd/gosec PACKAGE_VERSION: ${{ steps.setup-env.outputs.gosec-version }} run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} + - name: Install Staticcheck + env: + PACKAGE_URL: dominikh/staticcheck-action + PACKAGE_VERSION: ${{ steps.setup-env.outputs.go-staticcheck-version }} + run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - name: Install Packer env: PACKER_VERSION: ${{ steps.setup-env.outputs.packer-version }} @@ -95,11 +101,6 @@ jobs: PACKAGE_URL: mvdan.cc/sh/v3/cmd/shfmt PACKAGE_VERSION: ${{ steps.setup-env.outputs.shfmt-version }} run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - - name: Install Staticcheck - env: - PACKAGE_URL: dominikh/staticcheck-action - PACKAGE_VERSION: ${{ steps.setup-env.outputs.go-staticcheck-version }} - run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - name: Install Terraform-docs env: PACKAGE_URL: github.com/terraform-docs/terraform-docs From ecf329956ac7e2428e2226e2ac0c9c3c0cea94bf Mon Sep 17 00:00:00 2001 From: Jason Odoom Date: Tue, 11 Jul 2023 16:38:13 -0400 Subject: [PATCH 074/326] Update staticcheck id Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a3c90ea..f54946c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -78,7 +78,7 @@ jobs: - name: Install Staticcheck env: PACKAGE_URL: dominikh/staticcheck-action - PACKAGE_VERSION: ${{ steps.setup-env.outputs.go-staticcheck-version }} + PACKAGE_VERSION: ${{ steps.setup-env.outputs.staticcheck-version }} run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - name: Install Packer env: From 1eaedf57d23227522f0f302e6055ded6eb888a7a Mon Sep 17 00:00:00 2001 From: Jason Odoom Date: Tue, 11 Jul 2023 16:40:06 -0400 Subject: [PATCH 075/326] Update tool names to match author stylization Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f54946c..db696ad 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -64,8 +64,8 @@ jobs: ${{ env.BASE_CACHE_KEY }} - name: Setup curl cache run: mkdir -p ${{ env.CURL_CACHE_DIR }} - # Install GoCrtitic, GoSec and Staticcheck Go tools - - name: Install GoCritic + # Install go-critic, GoSec and staticcheck Go tools + - name: Install go-critic env: PACKAGE_URL: github.com/go-critic/go-critic/cmd/gocritic PACKAGE_VERSION: ${{ steps.setup-env.outputs.go-critic-version }} From fd04757b3339ed91a085e3028b2b88f9bf60e080 Mon Sep 17 00:00:00 2001 From: Jason Odoom Date: Tue, 11 Jul 2023 16:42:09 -0400 Subject: [PATCH 076/326] Update statcicheck stylization Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index db696ad..fee1ebe 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -75,7 +75,7 @@ jobs: PACKAGE_URL: github.com/securego/gosec/v2/cmd/gosec PACKAGE_VERSION: ${{ steps.setup-env.outputs.gosec-version }} run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - - name: Install Staticcheck + - name: Install staticcheck env: PACKAGE_URL: dominikh/staticcheck-action PACKAGE_VERSION: ${{ steps.setup-env.outputs.staticcheck-version }} From ddbf6f7f6c7cb928ce9bc22f1ae3b1796f0602e7 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 12 Jul 2023 09:09:32 -0400 Subject: [PATCH 077/326] Temporarily use a different branch of cisagov/setup-env-github-action This is being done for testing purposes, and this commit can be reverted (or removed) once cisagov/setup-env-github-action#65 is merged. --- .github/workflows/build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fee1ebe..b7c0910 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest steps: - id: setup-env - uses: cisagov/setup-env-github-action@develop + uses: cisagov/setup-env-github-action@add-go-actions-for-go-skeleton - uses: actions/checkout@v3 - id: setup-python uses: actions/setup-python@v4 @@ -64,18 +64,18 @@ jobs: ${{ env.BASE_CACHE_KEY }} - name: Setup curl cache run: mkdir -p ${{ env.CURL_CACHE_DIR }} - # Install go-critic, GoSec and staticcheck Go tools + # Install go-critic, GoSec and staticcheck Go tools - name: Install go-critic env: PACKAGE_URL: github.com/go-critic/go-critic/cmd/gocritic PACKAGE_VERSION: ${{ steps.setup-env.outputs.go-critic-version }} run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - - name: Install GoSec + - name: Install GoSec env: PACKAGE_URL: github.com/securego/gosec/v2/cmd/gosec PACKAGE_VERSION: ${{ steps.setup-env.outputs.gosec-version }} run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - - name: Install staticcheck + - name: Install staticcheck env: PACKAGE_URL: dominikh/staticcheck-action PACKAGE_VERSION: ${{ steps.setup-env.outputs.staticcheck-version }} From a8af336b975def35f947694d61828ab633d8bc6d Mon Sep 17 00:00:00 2001 From: Jason Odoom Date: Wed, 12 Jul 2023 10:11:28 -0400 Subject: [PATCH 078/326] Move go packages install to separate section --- .github/workflows/build.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b7c0910..4f0b8cd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -64,22 +64,6 @@ jobs: ${{ env.BASE_CACHE_KEY }} - name: Setup curl cache run: mkdir -p ${{ env.CURL_CACHE_DIR }} - # Install go-critic, GoSec and staticcheck Go tools - - name: Install go-critic - env: - PACKAGE_URL: github.com/go-critic/go-critic/cmd/gocritic - PACKAGE_VERSION: ${{ steps.setup-env.outputs.go-critic-version }} - run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - - name: Install GoSec - env: - PACKAGE_URL: github.com/securego/gosec/v2/cmd/gosec - PACKAGE_VERSION: ${{ steps.setup-env.outputs.gosec-version }} - run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - - name: Install staticcheck - env: - PACKAGE_URL: dominikh/staticcheck-action - PACKAGE_VERSION: ${{ steps.setup-env.outputs.staticcheck-version }} - run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - name: Install Packer env: PACKER_VERSION: ${{ steps.setup-env.outputs.packer-version }} @@ -96,6 +80,22 @@ jobs: - uses: hashicorp/setup-terraform@v2 with: terraform_version: ${{ steps.setup-env.outputs.terraform-version }} + # Begin Go packages install below + - name: Install GoCritic + env: + PACKAGE_URL: github.com/go-critic/go-critic/cmd/gocritic + PACKAGE_VERSION: ${{ steps.setup-env.outputs.go-critic-version }} + run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} + - name: Install GoSec + env: + PACKAGE_URL: github.com/securego/gosec/v2/cmd/gosec + PACKAGE_VERSION: ${{ steps.setup-env.outputs.gosec-version }} + run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} + - name: Install Staticcheck + env: + PACKAGE_URL: dominikh/staticcheck-action + PACKAGE_VERSION: ${{ steps.setup-env.outputs.go-staticcheck-version }} + run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - name: Install shfmt env: PACKAGE_URL: mvdan.cc/sh/v3/cmd/shfmt From 1675b127e5ba645e36726d3f1858a974e7612998 Mon Sep 17 00:00:00 2001 From: Jason Odoom Date: Wed, 12 Jul 2023 16:52:32 -0400 Subject: [PATCH 079/326] Update go-critic name Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4f0b8cd..faafd2f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -81,7 +81,7 @@ jobs: with: terraform_version: ${{ steps.setup-env.outputs.terraform-version }} # Begin Go packages install below - - name: Install GoCritic + - name: Install go-critic env: PACKAGE_URL: github.com/go-critic/go-critic/cmd/gocritic PACKAGE_VERSION: ${{ steps.setup-env.outputs.go-critic-version }} From c6ab22ea25d5608ab794542c96139bd5d65dac47 Mon Sep 17 00:00:00 2001 From: Jason Odoom Date: Wed, 12 Jul 2023 16:52:50 -0400 Subject: [PATCH 080/326] Update staticcheck name Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index faafd2f..a799374 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -91,7 +91,7 @@ jobs: PACKAGE_URL: github.com/securego/gosec/v2/cmd/gosec PACKAGE_VERSION: ${{ steps.setup-env.outputs.gosec-version }} run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - - name: Install Staticcheck + - name: Install staticcheck env: PACKAGE_URL: dominikh/staticcheck-action PACKAGE_VERSION: ${{ steps.setup-env.outputs.go-staticcheck-version }} From 29de03457f4ec4092d0dd98465d18a83a2193719 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Thu, 13 Jul 2023 10:05:20 -0400 Subject: [PATCH 081/326] Use the correct repo name for the ansible-lint pre-commit hook The repo name we were using redirects to the correct place, but we may as well cut out the middle man. --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6b87032..9dc41ff 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -130,7 +130,7 @@ repos: - id: pyupgrade # Ansible hooks - - repo: https://github.com/ansible-community/ansible-lint + - repo: https://github.com/ansible/ansible-lint rev: v6.17.0 hooks: - id: ansible-lint From 1e7cb4f721542b260680ee51d4cd75126da0a979 Mon Sep 17 00:00:00 2001 From: Jason Odoom Date: Wed, 12 Jul 2023 16:53:34 -0400 Subject: [PATCH 082/326] Correct staticcheck reference to setup-env, sort alphabetically and correct staticcheck package URL Co-Authored By: @mcdonnnj --- .github/workflows/build.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a799374..7435881 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -91,16 +91,16 @@ jobs: PACKAGE_URL: github.com/securego/gosec/v2/cmd/gosec PACKAGE_VERSION: ${{ steps.setup-env.outputs.gosec-version }} run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - - name: Install staticcheck - env: - PACKAGE_URL: dominikh/staticcheck-action - PACKAGE_VERSION: ${{ steps.setup-env.outputs.go-staticcheck-version }} - run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - name: Install shfmt env: PACKAGE_URL: mvdan.cc/sh/v3/cmd/shfmt PACKAGE_VERSION: ${{ steps.setup-env.outputs.shfmt-version }} run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} + - name: Install staticcheck + env: + PACKAGE_URL: honnef.co/go/tools/cmd/staticcheck + PACKAGE_VERSION: ${{ steps.setup-env.outputs.staticcheck-version }} + run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - name: Install Terraform-docs env: PACKAGE_URL: github.com/terraform-docs/terraform-docs From 5f3bc132490dcdfcc7431133ed38c293eeeffaf5 Mon Sep 17 00:00:00 2001 From: Jason Odoom Date: Thu, 13 Jul 2023 11:03:03 -0400 Subject: [PATCH 083/326] Remove Go section coment Co-authored-by: Shane Frasier --- .github/workflows/build.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7435881..00c5cdf 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -80,7 +80,6 @@ jobs: - uses: hashicorp/setup-terraform@v2 with: terraform_version: ${{ steps.setup-env.outputs.terraform-version }} - # Begin Go packages install below - name: Install go-critic env: PACKAGE_URL: github.com/go-critic/go-critic/cmd/gocritic From d31182550a11db54cff212171bb9cc0dac8bb636 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 13 Jul 2023 22:30:33 -0400 Subject: [PATCH 084/326] Remove unnecessary quotes in the dependabot configuration We generally only use quotes when they are strictly necessary to ensure data is interpreted as a string value. --- .github/dependabot.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index fef57f5..60f04c0 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -7,10 +7,10 @@ version: 2 updates: - - package-ecosystem: "github-actions" - directory: "/" + - package-ecosystem: github-actions + directory: / schedule: - interval: "weekly" + interval: weekly # ignore: # # Managed by cisagov/skeleton-generic # - dependency-name: actions/cache @@ -20,12 +20,12 @@ updates: # - dependency-name: hashicorp/setup-terraform # - dependency-name: mxschmitt/action-tmate - - package-ecosystem: "pip" - directory: "/" + - package-ecosystem: pip + directory: / schedule: - interval: "weekly" + interval: weekly - - package-ecosystem: "terraform" - directory: "/" + - package-ecosystem: terraform + directory: / schedule: - interval: "weekly" + interval: weekly From 2294d4970596551c7c2c794b869d186dcda49921 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 13 Jul 2023 22:41:38 -0400 Subject: [PATCH 085/326] Sort the keys in the Dependabot configuration Our standard practice for YAML files is to sort keys alphabetically. --- .github/dependabot.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 60f04c0..da2c02c 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,12 +5,8 @@ # these updates when the pull request(s) in the appropriate skeleton are merged # and Lineage processes these changes. -version: 2 updates: - - package-ecosystem: github-actions - directory: / - schedule: - interval: weekly + - directory: / # ignore: # # Managed by cisagov/skeleton-generic # - dependency-name: actions/cache @@ -19,13 +15,17 @@ updates: # - dependency-name: actions/setup-python # - dependency-name: hashicorp/setup-terraform # - dependency-name: mxschmitt/action-tmate + package-ecosystem: github-actions + schedule: + interval: weekly - - package-ecosystem: pip - directory: / + - directory: / + package-ecosystem: pip schedule: interval: weekly - - package-ecosystem: terraform - directory: / + - directory: / + package-ecosystem: terraform schedule: interval: weekly +version: 2 From af90040833213a2c22c23cbb1ddd8de8403c8d97 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 24 Mar 2023 11:06:27 -0400 Subject: [PATCH 086/326] Add @jasonodoom as a default codeowner --- .github/CODEOWNERS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 371258c..8f5c8c5 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,8 +3,8 @@ # These owners will be the default owners for everything in the # repo. Unless a later match takes precedence, these owners will be # requested for review when someone opens a pull request. -* @dav3r @felddy @jsf9k @mcdonnnj +* @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj # These folks own any files in the .github directory at the root of # the repository and any of its subdirectories. -/.github/ @dav3r @felddy @jsf9k @mcdonnnj +/.github/ @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj From f037c16e433d0b46201d83c69482fd567d2a460b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 19 Apr 2023 11:56:30 -0400 Subject: [PATCH 087/326] Use Python 3.11 for the `lint` job in the `build` workflow This is the latest minor release of Python so it makes sense to use it as the default for this job. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 08d67c7..8d0b3eb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -47,7 +47,7 @@ jobs: - id: setup-python uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.11" # We need the Go version and Go cache location for the actions/cache step, # so the Go installation must happen before that. - id: setup-go From 92d7a39cbf71d16e5cd8808f04548a2e963912f6 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 27 Mar 2023 08:37:24 -0400 Subject: [PATCH 088/326] Update pre-commit hook versions This is done automatically with the `pre-commit autoupdate` command. However the `ansible-lint` hook is manually kept back as we have not tested functionality to confirm that our roles will generally pass with the new version. --- .pre-commit-config.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b4e2017..0092977 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,11 +37,11 @@ repos: args: - --config=.mdl_config.yaml - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.0.0-alpha.4 + rev: v3.0.0-alpha.6 hooks: - id: prettier - repo: https://github.com/adrienverge/yamllint - rev: v1.29.0 + rev: v1.30.0 hooks: - id: yamllint args: @@ -49,14 +49,14 @@ repos: # GitHub Actions hooks - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.21.0 + rev: 0.22.0 hooks: - id: check-github-actions - id: check-github-workflows # pre-commit hooks - repo: https://github.com/pre-commit/pre-commit - rev: v3.0.2 + rev: v3.2.1 hooks: - id: validate_manifest @@ -83,7 +83,7 @@ repos: # Python hooks # Run bandit on the "tests" tree with a configuration - repo: https://github.com/PyCQA/bandit - rev: 1.7.4 + rev: 1.7.5 hooks: - id: bandit name: bandit (tests tree) @@ -98,7 +98,7 @@ repos: name: bandit (everything else) exclude: tests - repo: https://github.com/psf/black - rev: 22.12.0 + rev: 23.1.0 hooks: - id: black - repo: https://github.com/PyCQA/flake8 @@ -112,7 +112,7 @@ repos: hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.991 + rev: v1.1.1 hooks: - id: mypy - repo: https://github.com/asottile/pyupgrade @@ -129,7 +129,7 @@ repos: # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.77.0 + rev: v1.77.1 hooks: - id: terraform_fmt - id: terraform_validate From e30ba2e7ea4bab7ea1ce5b4814fd36165f50855c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Mar 2023 15:54:24 +0000 Subject: [PATCH 089/326] Bump actions/setup-go from 3 to 4 Bumps [actions/setup-go](https://github.com/actions/setup-go) from 3 to 4. - [Release notes](https://github.com/actions/setup-go/releases) - [Commits](https://github.com/actions/setup-go/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/setup-go dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8d0b3eb..1084232 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -51,7 +51,7 @@ jobs: # We need the Go version and Go cache location for the actions/cache step, # so the Go installation must happen before that. - id: setup-go - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: go-version: "1.19" - name: Lookup Go cache directory From f3da594b5ee3a7de75f8003a1a7ba49632c103ad Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 24 Feb 2023 11:17:03 -0500 Subject: [PATCH 090/326] Install/upgrade setuptools and wheel when upgrading pip When wheel gets installed alongside other packages, it may not get used when those other packages are installed. When that happens I see warnings like this: DEPRECATION: ansible-core is being installed using the legacy 'setup.py install' method, because it does not have a 'pyproject.toml' and the 'wheel' package is not installed. pip 23.1 will enforce this behaviour change. A possible replacement is to enable the '--use-pep517' option. Discussion can be found at https://github.com/pypa/pip/issues/8559 This change should get rid of these warnings. Nota bene: This is the practice we follow in the Dockerfile in cisagov/skeleton-docker, but for some reason we never started using it in our workflows. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1084232..b39d97f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -113,7 +113,7 @@ jobs: run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - name: Install dependencies run: | - python -m pip install --upgrade pip + python -m pip install --upgrade pip setuptools wheel pip install --upgrade --requirement requirements-test.txt - name: Set up pre-commit hook environments run: pre-commit install-hooks From 46896a8ce25afd460aa6530c1013dc952c01b785 Mon Sep 17 00:00:00 2001 From: Jason Odoom Date: Fri, 5 May 2023 14:00:52 -0400 Subject: [PATCH 091/326] Add Go hooks to support go code for our projects --- .pre-commit-config.yaml | 56 +++++++++++++++++++++++++++++++++-------- 1 file changed, 45 insertions(+), 11 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0092977..56652e0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,17 +31,17 @@ repos: # Text file hooks - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.33.0 + rev: v0.34.0 hooks: - id: markdownlint args: - --config=.mdl_config.yaml - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.0.0-alpha.6 + rev: v3.0.0-alpha.9-for-vscode hooks: - id: prettier - repo: https://github.com/adrienverge/yamllint - rev: v1.30.0 + rev: v1.31.0 hooks: - id: yamllint args: @@ -49,17 +49,51 @@ repos: # GitHub Actions hooks - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.22.0 + rev: 0.23.0 hooks: - id: check-github-actions - id: check-github-workflows # pre-commit hooks - repo: https://github.com/pre-commit/pre-commit - rev: v3.2.1 + rev: v3.3.1 hooks: - id: validate_manifest + # Go hooks + - repo: https://github.com/TekWizely/pre-commit-golang + rev: v1.0.0-rc.1 + hooks: + # Style Checkers + - id: go-lint + - id: go-critic + # Go Build + - id: go-build-mod + - id: go-build-pkg + - id: go-build-repo-mod + - id: go-build-repo-pkg + + # Go Mod Tidy + - id: go-mod-tidy + - id: go-mod-tidy-repo + + # Go Test + - id: go-test-mod + - id: go-test-pkg + - id: go-test-repo-mod + - id: go-test-repo-pkg + + # Go Vet + - id: go-vet + - id: go-vet-mod + - id: go-vet-pkg + - id: go-vet-repo-mod + - id: go-vet-repo-pkg + # GoSec + - id: go-sec-mod + - id: go-sec-pkg + - id: go-sec-repo-mod + - id: go-sec-repo-pkg # Shell script hooks - repo: https://github.com/cisagov/pre-commit-shfmt rev: v0.0.2 @@ -98,7 +132,7 @@ repos: name: bandit (everything else) exclude: tests - repo: https://github.com/psf/black - rev: 23.1.0 + rev: 23.3.0 hooks: - id: black - repo: https://github.com/PyCQA/flake8 @@ -112,31 +146,31 @@ repos: hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.1.1 + rev: v1.2.0 hooks: - id: mypy - repo: https://github.com/asottile/pyupgrade - rev: v3.3.1 + rev: v3.3.2 hooks: - id: pyupgrade # Ansible hooks - repo: https://github.com/ansible-community/ansible-lint - rev: v5.4.0 + rev: v6.15.0 hooks: - id: ansible-lint # files: molecule/default/playbook.yml # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.77.1 + rev: v1.78.0 hooks: - id: terraform_fmt - id: terraform_validate # Docker hooks - repo: https://github.com/IamTheFij/docker-pre-commit - rev: v2.1.1 + rev: v3.0.1 hooks: - id: docker-compose-check From ff0ebdfe1eafad4408b810c862c0f2a24cac879e Mon Sep 17 00:00:00 2001 From: Jason Odoom Date: Fri, 5 May 2023 14:08:25 -0400 Subject: [PATCH 092/326] Add Staticcheck --- .pre-commit-config.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 56652e0..5d50867 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -67,6 +67,11 @@ repos: # Style Checkers - id: go-lint - id: go-critic + # StaticCheck + - id: go-staticcheck-mod + - id: go-staticcheck-pkg + - id: go-staticcheck-repo-mod + - id: go-staticcheck-repo-pkg # Go Build - id: go-build-mod - id: go-build-pkg From f953e8e863a3f40725aba5d5ee0f7996d5b2659e Mon Sep 17 00:00:00 2001 From: Jason Odoom Date: Fri, 5 May 2023 16:07:08 -0400 Subject: [PATCH 093/326] Fix indentation Co-authored-by: Shane Frasier --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5d50867..6176b9a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -64,7 +64,7 @@ repos: - repo: https://github.com/TekWizely/pre-commit-golang rev: v1.0.0-rc.1 hooks: - # Style Checkers + # Style Checkers - id: go-lint - id: go-critic # StaticCheck From 932b62e635e5b10f9b8bdee0ef199919f42bbb61 Mon Sep 17 00:00:00 2001 From: Jason Odoom Date: Fri, 5 May 2023 16:07:17 -0400 Subject: [PATCH 094/326] Fix indentation Co-authored-by: Shane Frasier --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6176b9a..80c5508 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -67,7 +67,7 @@ repos: # Style Checkers - id: go-lint - id: go-critic - # StaticCheck + # StaticCheck - id: go-staticcheck-mod - id: go-staticcheck-pkg - id: go-staticcheck-repo-mod From 2eca0cdbdce085aefc0bbec358405f86e4c164b6 Mon Sep 17 00:00:00 2001 From: Jason Odoom Date: Fri, 5 May 2023 16:07:25 -0400 Subject: [PATCH 095/326] Fix indentation Co-authored-by: Shane Frasier --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 80c5508..e053114 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -72,7 +72,7 @@ repos: - id: go-staticcheck-pkg - id: go-staticcheck-repo-mod - id: go-staticcheck-repo-pkg - # Go Build + # Go Build - id: go-build-mod - id: go-build-pkg - id: go-build-repo-mod From fab2344cb2525a4d3090c806380aedb4a190860a Mon Sep 17 00:00:00 2001 From: Jason Odoom Date: Fri, 5 May 2023 16:07:44 -0400 Subject: [PATCH 096/326] Fix formatting Co-authored-by: Shane Frasier --- .pre-commit-config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e053114..68c799b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -77,7 +77,6 @@ repos: - id: go-build-pkg - id: go-build-repo-mod - id: go-build-repo-pkg - # Go Mod Tidy - id: go-mod-tidy - id: go-mod-tidy-repo From 892673b1f262bb9aa84edb551ca0979f1ca14b08 Mon Sep 17 00:00:00 2001 From: Jason Odoom Date: Fri, 5 May 2023 16:08:16 -0400 Subject: [PATCH 097/326] Fix formatting Co-authored-by: Shane Frasier --- .pre-commit-config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 68c799b..627a9cc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -86,7 +86,6 @@ repos: - id: go-test-pkg - id: go-test-repo-mod - id: go-test-repo-pkg - # Go Vet - id: go-vet - id: go-vet-mod From 0d77207d3e0b64fa3983c98c6975d93fd9df624a Mon Sep 17 00:00:00 2001 From: Jason Odoom Date: Fri, 5 May 2023 16:10:19 -0400 Subject: [PATCH 098/326] Fix formatting Co-authored-by: Shane Frasier --- .pre-commit-config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 627a9cc..8c4eb25 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -80,7 +80,6 @@ repos: # Go Mod Tidy - id: go-mod-tidy - id: go-mod-tidy-repo - # Go Test - id: go-test-mod - id: go-test-pkg From b661316605fc9ec531a6fe1b61b50f80e1eb729b Mon Sep 17 00:00:00 2001 From: Jason Odoom Date: Fri, 5 May 2023 16:19:50 -0400 Subject: [PATCH 099/326] Revert updated versions for existing hooks --- .pre-commit-config.yaml | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8c4eb25..81c3a6e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,17 +31,17 @@ repos: # Text file hooks - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.34.0 + rev: v0.33.0 hooks: - id: markdownlint args: - --config=.mdl_config.yaml - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.0.0-alpha.9-for-vscode + rev: v3.0.0-alpha.6 hooks: - id: prettier - repo: https://github.com/adrienverge/yamllint - rev: v1.31.0 + rev: v1.30.0 hooks: - id: yamllint args: @@ -49,14 +49,14 @@ repos: # GitHub Actions hooks - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.23.0 + rev: 0.22.0 hooks: - id: check-github-actions - id: check-github-workflows # pre-commit hooks - repo: https://github.com/pre-commit/pre-commit - rev: v3.3.1 + rev: v3.2.1 hooks: - id: validate_manifest @@ -96,6 +96,7 @@ repos: - id: go-sec-pkg - id: go-sec-repo-mod - id: go-sec-repo-pkg + # Shell script hooks - repo: https://github.com/cisagov/pre-commit-shfmt rev: v0.0.2 @@ -134,7 +135,7 @@ repos: name: bandit (everything else) exclude: tests - repo: https://github.com/psf/black - rev: 23.3.0 + rev: 23.1.0 hooks: - id: black - repo: https://github.com/PyCQA/flake8 @@ -148,31 +149,31 @@ repos: hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.2.0 + rev: v1.1.1 hooks: - id: mypy - repo: https://github.com/asottile/pyupgrade - rev: v3.3.2 + rev: v3.3.1 hooks: - id: pyupgrade # Ansible hooks - repo: https://github.com/ansible-community/ansible-lint - rev: v6.15.0 + rev: v5.4.0 hooks: - id: ansible-lint # files: molecule/default/playbook.yml # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.78.0 + rev: v1.77.1 hooks: - id: terraform_fmt - id: terraform_validate # Docker hooks - repo: https://github.com/IamTheFij/docker-pre-commit - rev: v3.0.1 + rev: v2.1.1 hooks: - id: docker-compose-check @@ -181,4 +182,4 @@ repos: rev: v0.0.2 hooks: - id: packer_validate - - id: packer_fmt + - id: packer_fmt \ No newline at end of file From 6a40279cb4f5624cd182dc126957dd08d83ddc25 Mon Sep 17 00:00:00 2001 From: Jason Odoom Date: Fri, 5 May 2023 16:30:11 -0400 Subject: [PATCH 100/326] Fix formatting --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 81c3a6e..6421594 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -182,4 +182,4 @@ repos: rev: v0.0.2 hooks: - id: packer_validate - - id: packer_fmt \ No newline at end of file + - id: packer_fmt From b0f2bf332f504635e7b15bcaca896e72f6f9a764 Mon Sep 17 00:00:00 2001 From: Jason Odoom Date: Fri, 5 May 2023 17:16:19 -0400 Subject: [PATCH 101/326] Remove extra hook ids --- .pre-commit-config.yaml | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6421594..ea1df4d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -69,33 +69,16 @@ repos: - id: go-critic # StaticCheck - id: go-staticcheck-mod - - id: go-staticcheck-pkg - - id: go-staticcheck-repo-mod - - id: go-staticcheck-repo-pkg # Go Build - id: go-build-mod - - id: go-build-pkg - - id: go-build-repo-mod - - id: go-build-repo-pkg # Go Mod Tidy - id: go-mod-tidy - - id: go-mod-tidy-repo # Go Test - id: go-test-mod - - id: go-test-pkg - - id: go-test-repo-mod - - id: go-test-repo-pkg # Go Vet - id: go-vet - - id: go-vet-mod - - id: go-vet-pkg - - id: go-vet-repo-mod - - id: go-vet-repo-pkg # GoSec - id: go-sec-mod - - id: go-sec-pkg - - id: go-sec-repo-mod - - id: go-sec-repo-pkg # Shell script hooks - repo: https://github.com/cisagov/pre-commit-shfmt From a93f6d403414c4d4b24717c98785d871f8503ad3 Mon Sep 17 00:00:00 2001 From: Jason Odoom Date: Mon, 8 May 2023 09:41:46 -0400 Subject: [PATCH 102/326] Replace hooks with repo based option when available --- .pre-commit-config.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ea1df4d..1b2aefe 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -68,17 +68,17 @@ repos: - id: go-lint - id: go-critic # StaticCheck - - id: go-staticcheck-mod + - id: go-staticcheck-repo-mod # Go Build - - id: go-build-mod + - id: go-build-repo-mod # Go Mod Tidy - - id: go-mod-tidy + - id: go-mod-tidy-repo # Go Test - - id: go-test-mod + - id: go-test-repo-mod # Go Vet - - id: go-vet + - id: go-vet-repo-mod # GoSec - - id: go-sec-mod + - id: go-sec-repo-mod # Shell script hooks - repo: https://github.com/cisagov/pre-commit-shfmt From ec058ef833e05852b7276929361e8eea4c4e0f14 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 7 Jun 2023 11:11:13 -0400 Subject: [PATCH 103/326] Remove the go-lint hook from the pre-commit configuration The golang/lint tool was archived on May 9th, 2021 and based on golang/go#38968 no future work is planned. Coupled with the fact that it is not available from brew we are removing this hook as local development may be hindered by trying to satisfy running this hook. Co-authored-by: Shane Frasier Co-authored-by: dav3r --- .pre-commit-config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1b2aefe..b2a197d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -65,7 +65,6 @@ repos: rev: v1.0.0-rc.1 hooks: # Style Checkers - - id: go-lint - id: go-critic # StaticCheck - id: go-staticcheck-repo-mod From 033e19451da3688619435721f6f8d7303d717aac Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 31 May 2023 12:45:10 -0400 Subject: [PATCH 104/326] Update pre-commit hook versions This is done automatically with the `pre-commit autoupdate` command. --- .pre-commit-config.yaml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b2a197d..5278abd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,17 +31,17 @@ repos: # Text file hooks - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.33.0 + rev: v0.34.0 hooks: - id: markdownlint args: - --config=.mdl_config.yaml - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.0.0-alpha.6 + rev: v3.0.0-alpha.9-for-vscode hooks: - id: prettier - repo: https://github.com/adrienverge/yamllint - rev: v1.30.0 + rev: v1.32.0 hooks: - id: yamllint args: @@ -49,14 +49,14 @@ repos: # GitHub Actions hooks - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.22.0 + rev: 0.23.1 hooks: - id: check-github-actions - id: check-github-workflows # pre-commit hooks - repo: https://github.com/pre-commit/pre-commit - rev: v3.2.1 + rev: v3.3.2 hooks: - id: validate_manifest @@ -117,7 +117,7 @@ repos: name: bandit (everything else) exclude: tests - repo: https://github.com/psf/black - rev: 23.1.0 + rev: 23.3.0 hooks: - id: black - repo: https://github.com/PyCQA/flake8 @@ -131,31 +131,31 @@ repos: hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.1.1 + rev: v1.3.0 hooks: - id: mypy - repo: https://github.com/asottile/pyupgrade - rev: v3.3.1 + rev: v3.4.0 hooks: - id: pyupgrade # Ansible hooks - repo: https://github.com/ansible-community/ansible-lint - rev: v5.4.0 + rev: v6.17.0 hooks: - id: ansible-lint # files: molecule/default/playbook.yml # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.77.1 + rev: v1.80.0 hooks: - id: terraform_fmt - id: terraform_validate # Docker hooks - repo: https://github.com/IamTheFij/docker-pre-commit - rev: v2.1.1 + rev: v3.0.1 hooks: - id: docker-compose-check From 43a4bd1150438f87cd378d20b0f066995460b519 Mon Sep 17 00:00:00 2001 From: Jason Odoom Date: Wed, 31 May 2023 12:29:10 -0400 Subject: [PATCH 105/326] Add apostrophe to Qapla --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index f526cdb..436d163 100755 --- a/setup-env +++ b/setup-env @@ -186,5 +186,5 @@ else: END_OF_LINE )" -# Qapla +# Qapla' echo "Success!" From 7bfd06ca5fb1272a96ec220a3e66d9f708daac17 Mon Sep 17 00:00:00 2001 From: arcsector <26469747+arcsector@users.noreply.github.com> Date: Wed, 17 May 2023 16:01:54 -0700 Subject: [PATCH 106/326] detecting virtualenv installation successfully --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 436d163..129102f 100755 --- a/setup-env +++ b/setup-env @@ -65,7 +65,7 @@ done eval set -- "$PARAMS" # Check to see if pyenv is installed -if [ -z "$(command -v pyenv)" ] || [ -z "$(command -v pyenv-virtualenv)" ]; then +if [ -z "$(command -v pyenv)" ] || { [ -z "$(command -v pyenv-virtualenv)" ] && [ ! -f "$(pyenv root)/plugins/pyenv-virtualenv/bin/pyenv-virtualenv" ] ; }; then echo "pyenv and pyenv-virtualenv are required." if [[ "$OSTYPE" == "darwin"* ]]; then cat << 'END_OF_LINE' From 6030c9a2deb6ce5e0f0264e8593d7f1fbda30f17 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 19 May 2023 11:46:28 -0400 Subject: [PATCH 107/326] Whitespace change to make shfmt linter happy --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 129102f..77926bf 100755 --- a/setup-env +++ b/setup-env @@ -65,7 +65,7 @@ done eval set -- "$PARAMS" # Check to see if pyenv is installed -if [ -z "$(command -v pyenv)" ] || { [ -z "$(command -v pyenv-virtualenv)" ] && [ ! -f "$(pyenv root)/plugins/pyenv-virtualenv/bin/pyenv-virtualenv" ] ; }; then +if [ -z "$(command -v pyenv)" ] || { [ -z "$(command -v pyenv-virtualenv)" ] && [ ! -f "$(pyenv root)/plugins/pyenv-virtualenv/bin/pyenv-virtualenv" ]; }; then echo "pyenv and pyenv-virtualenv are required." if [[ "$OSTYPE" == "darwin"* ]]; then cat << 'END_OF_LINE' From 2122a3945fc181c4f6cf30a7a5983af475deadfa Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 25 Apr 2023 09:17:14 -0400 Subject: [PATCH 108/326] Fix two broken links Resolves cisagov/skeleton-generic#131. --- .ansible-lint | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.ansible-lint b/.ansible-lint index 0e80b05..4ffc0ef 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -1,10 +1,9 @@ --- -# See https://ansible-lint.readthedocs.io/en/latest/configuring.html -# for a list of the configuration elements that can exist in this -# file. +# See https://ansible-lint.readthedocs.io/configuring/ for a list of +# the configuration elements that can exist in this file. enable_list: # Useful checks that one must opt-into. See here for more details: - # https://ansible-lint.readthedocs.io/en/latest/rules.html + # https://ansible-lint.readthedocs.io/rules/ - fcqn-builtins - no-log-password - no-same-owner From 92e0e6aa762979f52471c92931ffd77bb800f392 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 31 May 2023 13:39:14 -0400 Subject: [PATCH 109/326] Disable caching for the setup-go Action The cache key used relies on the existence of a go.sum file. Since we have no expectation for Go source code, including that file, and since we already include the Go cache in our job caching, we can safely disable caching in the Action. --- .github/workflows/build.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b39d97f..34aaceb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -53,6 +53,9 @@ jobs: - id: setup-go uses: actions/setup-go@v4 with: + # There is no expectation for actual Go code so we disable caching as + # it relies on the existence of a go.sum file. + cache: false go-version: "1.19" - name: Lookup Go cache directory id: go-cache From 2e746e2cd99228977d3177d4ddea6762f791744f Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 31 May 2023 13:41:03 -0400 Subject: [PATCH 110/326] Bump Go from 1.19 to 1.20 Bump the version of Go used in our GitHub Actions configuration to the latest stable Go release. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 34aaceb..8e3c2f3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -56,7 +56,7 @@ jobs: # There is no expectation for actual Go code so we disable caching as # it relies on the existence of a go.sum file. cache: false - go-version: "1.19" + go-version: "1.20" - name: Lookup Go cache directory id: go-cache run: | From c07be40b2ddf2cc3ff17d5286e469918b5ae5fea Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 14 Jul 2023 11:30:56 -0400 Subject: [PATCH 111/326] Update the version of Python used in the GHA `test` job The version of Python used in the `lint` job of the build workflow for GitHub Actions was updated to 3.11 so we should update the `test` job to match. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8e3c2f3..2796874 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -306,7 +306,7 @@ jobs: - id: setup-python uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.11" - name: Cache testing environments uses: actions/cache@v3 env: From 2a163268d67444967761adf1d4eccabbde102bc4 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 14 Jul 2023 11:45:51 -0400 Subject: [PATCH 112/326] Install two additional Python packages in the GHA `test` job The `setuptools` and `wheel` packages were added to the `lint` job in GitHub Actions and it should be duplicated for the `test` job. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2796874..d8f6d6d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -321,7 +321,7 @@ jobs: ${{ env.BASE_CACHE_KEY }} - name: Install dependencies run: | - python -m pip install --upgrade pip + python -m pip install --upgrade pip setuptools wheel pip install --upgrade --requirement requirements-test.txt - name: Download docker image artifact uses: actions/download-artifact@v3 From 8237d2703dc248b80cbac9ac6a805a08e43db3c2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 14 Jul 2023 19:09:03 +0000 Subject: [PATCH 113/326] Bump python from 3.10.1-alpine to 3.11.4-alpine Bumps python from 3.10.1-alpine to 3.11.4-alpine. --- updated-dependencies: - dependency-name: python dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1516787..4aea26a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ ARG VERSION=unspecified -FROM python:3.10.1-alpine +FROM python:3.11.4-alpine ARG VERSION From e678502adb77650dbd722134c469f20b5b46d32d Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 18 Jul 2023 15:06:13 -0400 Subject: [PATCH 114/326] Delete duplicate word "are" Co-authored-by: David Harris --- .bandit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bandit.yml b/.bandit.yml index 8ba42d1..ab3cb21 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -3,7 +3,7 @@ # https://bandit.readthedocs.io/en/latest/config.html # Tests are first included by `tests`, and then excluded by `skips`. -# If `tests` is empty, all tests are are considered included. +# If `tests` is empty, all tests are considered included. tests: # - B101 From c02e483e8616e4f9d9c88f07e2b86ee1d5877b22 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 20 Jul 2023 01:17:34 -0400 Subject: [PATCH 115/326] Migrate testing from pytest-dockerc to using python-on-whales The pytest-dockerc plug is unmaintained and there is now a dependency issues with PyYAML because of the release of Cython v3 (which is itself a build dependency for PyYAML). After some research this seemed like the most similar package in terms of functionality to the package we are replacing. Although it is not a pytest plugin it still provides similar access and uses the Docker composition defined in the repository. --- requirements-test.txt | 2 +- tests/conftest.py | 13 +++++++++++-- tests/container_test.py | 29 +++++++++++++++++------------ 3 files changed, 29 insertions(+), 15 deletions(-) diff --git a/requirements-test.txt b/requirements-test.txt index 5f3337c..8b41b2f 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,4 +1,4 @@ --requirement requirements.txt pre-commit pytest -pytest-dockerc +python-on-whales diff --git a/tests/conftest.py b/tests/conftest.py index 90938e6..054f09d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,16 +4,25 @@ """ # Third-Party Libraries import pytest +from python_on_whales import docker MAIN_SERVICE_NAME = "example" VERSION_SERVICE_NAME = f"{MAIN_SERVICE_NAME}-version" +@pytest.fixture(scope="session") +def dockerc(): + """Start up the Docker composition.""" + docker.compose.up(detach=True) + yield docker + docker.compose.down() + + @pytest.fixture(scope="session") def main_container(dockerc): """Return the main container from the Docker composition.""" # find the container by name even if it is stopped already - return dockerc.containers(service_names=[MAIN_SERVICE_NAME], stopped=True)[0] + return dockerc.compose.ps(services=[MAIN_SERVICE_NAME], all=True)[0] @pytest.fixture(scope="session") @@ -23,7 +32,7 @@ def version_container(dockerc): The version container should just output the version of its underlying contents. """ # find the container by name even if it is stopped already - return dockerc.containers(service_names=[VERSION_SERVICE_NAME], stopped=True)[0] + return dockerc.compose.ps(services=[VERSION_SERVICE_NAME], all=True)[0] def pytest_addoption(parser): diff --git a/tests/container_test.py b/tests/container_test.py index b92ff86..c6255df 100644 --- a/tests/container_test.py +++ b/tests/container_test.py @@ -22,7 +22,7 @@ def test_container_count(dockerc): """Verify the test composition and container.""" # stopped parameter allows non-running containers in results assert ( - len(dockerc.containers(stopped=True)) == 2 + len(dockerc.compose.ps(all=True)) == 2 ), "Wrong number of containers were started." @@ -30,7 +30,7 @@ def test_wait_for_ready(main_container): """Wait for container to be ready.""" TIMEOUT = 10 for i in range(TIMEOUT): - if READY_MESSAGE in main_container.logs().decode("utf-8"): + if READY_MESSAGE in main_container.logs(): break time.sleep(1) else: @@ -40,18 +40,21 @@ def test_wait_for_ready(main_container): ) -def test_wait_for_exits(main_container, version_container): +def test_wait_for_exits(dockerc, main_container, version_container): """Wait for containers to exit.""" - assert main_container.wait() == 0, "Container service (main) did not exit cleanly" assert ( - version_container.wait() == 0 + dockerc.wait(main_container.id) == 0 + ), "Container service (main) did not exit cleanly" + assert ( + dockerc.wait(version_container.id) == 0 ), "Container service (version) did not exit cleanly" -def test_output(main_container): +def test_output(dockerc, main_container): """Verify the container had the correct output.""" - main_container.wait() # make sure container exited if running test isolated - log_output = main_container.logs().decode("utf-8") + # make sure container exited if running test isolated + dockerc.wait(main_container.id) + log_output = main_container.logs() assert SECRET_QUOTE in log_output, "Secret not found in log output." @@ -69,10 +72,11 @@ def test_release_version(): ), "RELEASE_TAG does not match the project version" -def test_log_version(version_container): +def test_log_version(dockerc, version_container): """Verify the container outputs the correct version to the logs.""" - version_container.wait() # make sure container exited if running test isolated - log_output = version_container.logs().decode("utf-8").strip() + # make sure container exited if running test isolated + dockerc.wait(version_container.id) + log_output = version_container.logs().strip() pkg_vars = {} with open(VERSION_FILE) as f: exec(f.read(), pkg_vars) # nosec @@ -89,5 +93,6 @@ def test_container_version_label_matches(version_container): exec(f.read(), pkg_vars) # nosec project_version = pkg_vars["__version__"] assert ( - version_container.labels["org.opencontainers.image.version"] == project_version + version_container.config.labels["org.opencontainers.image.version"] + == project_version ), "Dockerfile version label does not match project version" From a518cfbe3045ae85f77df880a0c005e56819fda8 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 28 Jul 2023 15:33:41 -0400 Subject: [PATCH 116/326] Reference correct parameter name in comment The "stopped" parameter is no longer present now that we are using python-on-whales. Co-authored-by: dav3r --- tests/container_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/container_test.py b/tests/container_test.py index c6255df..c2e1874 100644 --- a/tests/container_test.py +++ b/tests/container_test.py @@ -20,7 +20,7 @@ def test_container_count(dockerc): """Verify the test composition and container.""" - # stopped parameter allows non-running containers in results + # all parameter allows non-running containers in results assert ( len(dockerc.compose.ps(all=True)) == 2 ), "Wrong number of containers were started." From 0db97d1a905bc591053b39894ffbe29e396c38e2 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 9 Aug 2023 14:54:38 -0400 Subject: [PATCH 117/326] Add the ability to specify a token in bump_version.sh The token is optionally used for the build and prerelease actions to override the default values used by the semver Python package. --- bump_version.sh | 52 +++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 46 insertions(+), 6 deletions(-) diff --git a/bump_version.sh b/bump_version.sh index 963389f..41fc121 100755 --- a/bump_version.sh +++ b/bump_version.sh @@ -1,6 +1,12 @@ #!/usr/bin/env bash -# bump_version.sh (show|major|minor|patch|prerelease|build) +# Usage: +# bump_version.sh (show|major|minor|patch|finalize) +# bump_version.sh (build|prerelease) [token] +# Notes: +# - If you specify a token it will only be used if the current version is +# tokenless or if the provided token matches the token used in the current +# version. set -o nounset set -o errexit @@ -9,18 +15,33 @@ set -o pipefail VERSION_FILE=src/version.txt README_FILE=README.md -HELP_INFORMATION="bump_version.sh (show|major|minor|patch|prerelease|build|finalize)" +function usage { + cat << HELP +Usage: + ${0##*/} (show|major|minor|patch|finalize) + ${0##*/} (build|prerelease) [token] + +Notes: + - If you specify a token it will only be used if the current version is + tokenless or if the provided token matches the token used in the current + version. +HELP + exit 1 +} old_version=$(sed -n "s/^__version__ = \"\(.*\)\"$/\1/p" $VERSION_FILE) # Comment out periods so they are interpreted as periods and don't # just match any character old_version_regex=${old_version//\./\\\.} -if [ $# -ne 1 ]; then - echo "$HELP_INFORMATION" +if [ $# -lt 1 ] || [ $# -gt 2 ]; then + usage else case $1 in - major | minor | patch | prerelease | build) + major | minor | patch) + if [ $# -ne 1 ]; then + usage + fi new_version=$(python -c "import semver; print(semver.bump_$1('$old_version'))") echo Changing version from "$old_version" to "$new_version" tmp_file=/tmp/version.$$ @@ -32,7 +53,26 @@ else git commit -m"Bump version from $old_version to $new_version" git push ;; + build | prerelease) + if [ $# -eq 2 ]; then + new_version=$(python -c "import semver; print(semver.bump_$1('$old_version', token='$2'))") + else + new_version=$(python -c "import semver; print(semver.bump_$1('$old_version'))") + fi + echo Changing version from "$old_version" to "$new_version" + tmp_file=/tmp/version.$$ + sed "s/$old_version_regex/$new_version/" $VERSION_FILE > $tmp_file + mv $tmp_file $VERSION_FILE + sed "s/$old_version_regex/$new_version/" $README_FILE > $tmp_file + mv $tmp_file $README_FILE + git add $VERSION_FILE $README_FILE + git commit -m"Bump version from $old_version to $new_version" + git push + ;; finalize) + if [ $# -ne 1 ]; then + usage + fi new_version=$(python -c "import semver; print(semver.finalize_version('$old_version'))") echo Changing version from "$old_version" to "$new_version" tmp_file=/tmp/version.$$ @@ -48,7 +88,7 @@ else echo "$old_version" ;; *) - echo "$HELP_INFORMATION" + usage ;; esac fi From f646a73a155b4480bcbe229ecca95c18ec88c0d2 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 9 Aug 2023 15:02:59 -0400 Subject: [PATCH 118/326] DRY out logic to update version information Since the replacement and commit of version change information is almost identical across the different logic flows it makes sense to DRY out the code. --- bump_version.sh | 49 +++++++++++++++++++------------------------------ 1 file changed, 19 insertions(+), 30 deletions(-) diff --git a/bump_version.sh b/bump_version.sh index 41fc121..91fec93 100755 --- a/bump_version.sh +++ b/bump_version.sh @@ -29,10 +29,23 @@ HELP exit 1 } +function update_version { + # Comment out periods so they are interpreted as periods and don't + # just match any character + old_version_regex=${1//\./\\\.} + + echo Changing version from "$1" to "$2" + tmp_file=/tmp/version.$$ + sed "s/$old_version_regex/$2/" $VERSION_FILE > $tmp_file + mv $tmp_file $VERSION_FILE + sed "s/$old_version_regex/$2/" $README_FILE > $tmp_file + mv $tmp_file $README_FILE + git add $VERSION_FILE $README_FILE + git commit --message "$3" + git push +} + old_version=$(sed -n "s/^__version__ = \"\(.*\)\"$/\1/p" $VERSION_FILE) -# Comment out periods so they are interpreted as periods and don't -# just match any character -old_version_regex=${old_version//\./\\\.} if [ $# -lt 1 ] || [ $# -gt 2 ]; then usage @@ -43,15 +56,7 @@ else usage fi new_version=$(python -c "import semver; print(semver.bump_$1('$old_version'))") - echo Changing version from "$old_version" to "$new_version" - tmp_file=/tmp/version.$$ - sed "s/$old_version_regex/$new_version/" $VERSION_FILE > $tmp_file - mv $tmp_file $VERSION_FILE - sed "s/$old_version_regex/$new_version/" $README_FILE > $tmp_file - mv $tmp_file $README_FILE - git add $VERSION_FILE $README_FILE - git commit -m"Bump version from $old_version to $new_version" - git push + update_version "$old_version" "$new_version" "Bump version from $old_version to $new_version" ;; build | prerelease) if [ $# -eq 2 ]; then @@ -59,30 +64,14 @@ else else new_version=$(python -c "import semver; print(semver.bump_$1('$old_version'))") fi - echo Changing version from "$old_version" to "$new_version" - tmp_file=/tmp/version.$$ - sed "s/$old_version_regex/$new_version/" $VERSION_FILE > $tmp_file - mv $tmp_file $VERSION_FILE - sed "s/$old_version_regex/$new_version/" $README_FILE > $tmp_file - mv $tmp_file $README_FILE - git add $VERSION_FILE $README_FILE - git commit -m"Bump version from $old_version to $new_version" - git push + update_version "$old_version" "$new_version" "Bump version from $old_version to $new_version" ;; finalize) if [ $# -ne 1 ]; then usage fi new_version=$(python -c "import semver; print(semver.finalize_version('$old_version'))") - echo Changing version from "$old_version" to "$new_version" - tmp_file=/tmp/version.$$ - sed "s/$old_version_regex/$new_version/" $VERSION_FILE > $tmp_file - mv $tmp_file $VERSION_FILE - sed "s/$old_version_regex/$new_version/" $README_FILE > $tmp_file - mv $tmp_file $README_FILE - git add $VERSION_FILE $README_FILE - git commit -m"Finalize version from $old_version to $new_version" - git push + update_version "$old_version" "$new_version" "Finalize version from $old_version to $new_version" ;; show) echo "$old_version" From cf5c0d742dd04e89de476f19effdac2cdb6832a5 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 9 Aug 2023 15:06:16 -0400 Subject: [PATCH 119/326] Change when existing version information is retrieved Only retrieve the existing version information if the input to the bump_version.sh script isn't immediately invalid (if the script does not get 1-2 arguments). --- bump_version.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bump_version.sh b/bump_version.sh index 91fec93..63b5d04 100755 --- a/bump_version.sh +++ b/bump_version.sh @@ -45,11 +45,10 @@ function update_version { git push } -old_version=$(sed -n "s/^__version__ = \"\(.*\)\"$/\1/p" $VERSION_FILE) - if [ $# -lt 1 ] || [ $# -gt 2 ]; then usage else + old_version=$(sed -n "s/^__version__ = \"\(.*\)\"$/\1/p" $VERSION_FILE) case $1 in major | minor | patch) if [ $# -ne 1 ]; then From d6715e70a650d317170d809722585ddb323c6382 Mon Sep 17 00:00:00 2001 From: Nick <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 15 Aug 2023 16:02:50 -0400 Subject: [PATCH 120/326] Remove git push from `bump_version.sh` script The team has come to the consensus that the `git push` should be at the discretion of the user. This both prevents needless GitHub Actions runs and ensures that the repository is updated when the user feels it is in an appropriate state. Co-authored-by: Shane Frasier --- bump_version.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/bump_version.sh b/bump_version.sh index 63b5d04..0071670 100755 --- a/bump_version.sh +++ b/bump_version.sh @@ -42,7 +42,6 @@ function update_version { mv $tmp_file $README_FILE git add $VERSION_FILE $README_FILE git commit --message "$3" - git push } if [ $# -lt 1 ] || [ $# -gt 2 ]; then From 948ebdeee11f77759311f85d256f19b5c4d7204d Mon Sep 17 00:00:00 2001 From: Jason Odoom Date: Wed, 16 Aug 2023 10:20:21 -0400 Subject: [PATCH 121/326] Fix gosec stylization Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 00c5cdf..ec15b7d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -85,7 +85,7 @@ jobs: PACKAGE_URL: github.com/go-critic/go-critic/cmd/gocritic PACKAGE_VERSION: ${{ steps.setup-env.outputs.go-critic-version }} run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - - name: Install GoSec + - name: Install gosec env: PACKAGE_URL: github.com/securego/gosec/v2/cmd/gosec PACKAGE_VERSION: ${{ steps.setup-env.outputs.gosec-version }} From 98d3d3f6914b4601cf9d194ed7d750224a4fb401 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 22 Aug 2023 10:18:25 -0400 Subject: [PATCH 122/326] Revert "Temporarily use a different branch of cisagov/setup-env-github-action" This reverts commit ddbf6f7f6c7cb928ce9bc22f1ae3b1796f0602e7. This can be done now that cisagov/setup-env-github-action#65 has been merged. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ec15b7d..34c3c6c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest steps: - id: setup-env - uses: cisagov/setup-env-github-action@add-go-actions-for-go-skeleton + uses: cisagov/setup-env-github-action@develop - uses: actions/checkout@v3 - id: setup-python uses: actions/setup-python@v4 From 82db36a453deda26c99ee231879486b9e2d9ac1a Mon Sep 17 00:00:00 2001 From: Jason Odoom Date: Mon, 28 Aug 2023 11:06:36 -0400 Subject: [PATCH 123/326] Add nixfmt pre-commit hook --- .pre-commit-config.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6b87032..e7262f6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -79,6 +79,12 @@ repos: # GoSec - id: go-sec-repo-mod + # Nix hooks + - repo: https://github.com/nix-community/nixpkgs-fmt + rev: v1.3.0 + hooks: + - id: nixpkgs-fmt + # Shell script hooks - repo: https://github.com/cisagov/pre-commit-shfmt rev: v0.0.2 From c0b5d5bb6cb644bf85171a8f5c89dfc3c770662f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Sep 2023 18:19:17 +0000 Subject: [PATCH 124/326] Bump actions/checkout from 3 to 4 Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- .github/workflows/sync-labels.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c992e8e..3b38f76 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,7 +19,7 @@ jobs: steps: - id: setup-env uses: cisagov/setup-env-github-action@develop - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - id: setup-python uses: actions/setup-python@v4 with: diff --git a/.github/workflows/sync-labels.yml b/.github/workflows/sync-labels.yml index ae7c091..b6f2024 100644 --- a/.github/workflows/sync-labels.yml +++ b/.github/workflows/sync-labels.yml @@ -19,7 +19,7 @@ jobs: issues: write runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Sync repository labels if: success() uses: crazy-max/ghaction-github-labeler@v4 From b04654e922ec4eef978022b67374a2ab3cffab4c Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 11 Sep 2023 01:57:49 -0400 Subject: [PATCH 125/326] Bump crazy-max/ghaction-github-labeler from 4 to 5 --- .github/workflows/sync-labels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync-labels.yml b/.github/workflows/sync-labels.yml index ae7c091..1c107e4 100644 --- a/.github/workflows/sync-labels.yml +++ b/.github/workflows/sync-labels.yml @@ -22,7 +22,7 @@ jobs: - uses: actions/checkout@v3 - name: Sync repository labels if: success() - uses: crazy-max/ghaction-github-labeler@v4 + uses: crazy-max/ghaction-github-labeler@v5 with: # This is a hideous ternary equivalent so we only do a dry run unless # this workflow is triggered by the develop branch. From 49ac8c505181e79caefc12cfc2f8e906c1e88ab2 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 11 Sep 2023 01:59:09 -0400 Subject: [PATCH 126/326] Update the dependabot ignore configuration Add crazy-max/ghaction-github-labeler as a commented out dependency to ignore in the dependabot configuration file. This should be enabled in downstream projects to consolidate updating this Action to the cisagov/skeleton-generic repository. --- .github/dependabot.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index fef57f5..e1f2f34 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -17,6 +17,7 @@ updates: # - dependency-name: actions/checkout # - dependency-name: actions/setup-go # - dependency-name: actions/setup-python + # - dependency-name: crazy-max/ghaction-github-labeler # - dependency-name: hashicorp/setup-terraform # - dependency-name: mxschmitt/action-tmate From 94d753d10d8106be1e4634bac6a27c0fbfb51c40 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 3 Aug 2023 19:10:29 -0400 Subject: [PATCH 127/326] Update pre-commit hook versions This is done automatically with the `pre-commit autoupdate` command. --- .pre-commit-config.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c5b7035..dad86ae 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,13 +31,13 @@ repos: # Text file hooks - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.34.0 + rev: v0.36.0 hooks: - id: markdownlint args: - --config=.mdl_config.yaml - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.0.0-alpha.9-for-vscode + rev: v3.0.3 hooks: - id: prettier - repo: https://github.com/adrienverge/yamllint @@ -49,14 +49,14 @@ repos: # GitHub Actions hooks - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.23.1 + rev: 0.26.3 hooks: - id: check-github-actions - id: check-github-workflows # pre-commit hooks - repo: https://github.com/pre-commit/pre-commit - rev: v3.3.2 + rev: v3.4.0 hooks: - id: validate_manifest @@ -113,11 +113,11 @@ repos: args: - --config=.bandit.yml - repo: https://github.com/psf/black - rev: 23.3.0 + rev: 23.9.1 hooks: - id: black - repo: https://github.com/PyCQA/flake8 - rev: 6.0.0 + rev: 6.1.0 hooks: - id: flake8 additional_dependencies: @@ -127,24 +127,24 @@ repos: hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.3.0 + rev: v1.5.1 hooks: - id: mypy - repo: https://github.com/asottile/pyupgrade - rev: v3.4.0 + rev: v3.10.1 hooks: - id: pyupgrade # Ansible hooks - repo: https://github.com/ansible/ansible-lint - rev: v6.17.0 + rev: v6.19.0 hooks: - id: ansible-lint # files: molecule/default/playbook.yml # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.80.0 + rev: v1.83.2 hooks: - id: terraform_fmt - id: terraform_validate From 1bc2056836e544c813d944150932075bb7251e33 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 11 Sep 2023 17:50:36 -0400 Subject: [PATCH 128/326] Switch to the pre-commit mirror for black This mirror was created to leverage performance optimizations from mypyc wheels that are available if black is installed from PyPI. These wheels are not available if black is installed from source as it would be using the old URL. Please see psf/black#3828 and psf/black#3405 for more information. --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dad86ae..34f0253 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -112,7 +112,7 @@ repos: - id: bandit args: - --config=.bandit.yml - - repo: https://github.com/psf/black + - repo: https://github.com/psf/black-pre-commit-mirror rev: 23.9.1 hooks: - id: black From a62ebe79e749b4e66b1f2122b3f866fd6ab8c23e Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 29 Aug 2023 17:10:03 -0400 Subject: [PATCH 129/326] Add the crazy-max/ghaction-github-status GitHub action This action is added in a separate "diagnostics" job. As configured it will never fail, but it will print out the status of the various GitHub components. This information will sometimes be useful when determining why builds fail after the fact. Co-authored-by: Mark Feldhousen Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- .github/dependabot.yml | 1 + .github/workflows/build.yml | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 9ab2270..2fdd39d 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -14,6 +14,7 @@ updates: # - dependency-name: actions/setup-go # - dependency-name: actions/setup-python # - dependency-name: crazy-max/ghaction-github-labeler + # - dependency-name: crazy-max/ghaction-github-status # - dependency-name: hashicorp/setup-terraform # - dependency-name: mxschmitt/action-tmate package-ecosystem: github-actions diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c08880f..9382c48 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,6 +14,12 @@ env: RUN_TMATE: ${{ secrets.RUN_TMATE }} jobs: + diagnostics: + runs-on: ubuntu-latest + steps: + - id: github-status + name: Check GitHub status + uses: crazy-max/ghaction-github-status@v3 lint: runs-on: ubuntu-latest steps: From 3619c455e6bb559ed74f7f320026badcaa603ad9 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 30 Aug 2023 09:48:01 -0400 Subject: [PATCH 130/326] Make the lint job depend on the diagnostics job Even though the diagnostics job is not currently configured to fail due to the GitHub status, it is still true that if the job is unable to run that does not bode well for the lint job's successful execution. Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9382c48..1049da9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -21,6 +21,8 @@ jobs: name: Check GitHub status uses: crazy-max/ghaction-github-status@v3 lint: + needs: + - diagnostics runs-on: ubuntu-latest steps: - id: setup-env From f4370660795e7d97f3e5ef0d582ed5754b2c267f Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 30 Aug 2023 10:35:45 -0400 Subject: [PATCH 131/326] Add a GH Action to dump the context This can be useful when debugging why a GH Action failed. Co-authored-by: felddy --- .github/dependabot.yml | 1 + .github/workflows/build.yml | 3 +++ 2 files changed, 4 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 2fdd39d..dbb9b73 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -13,6 +13,7 @@ updates: # - dependency-name: actions/checkout # - dependency-name: actions/setup-go # - dependency-name: actions/setup-python + # - dependency-name: crazy-max/ghaction-dump-context # - dependency-name: crazy-max/ghaction-github-labeler # - dependency-name: crazy-max/ghaction-github-status # - dependency-name: hashicorp/setup-terraform diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1049da9..346fd81 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,6 +20,9 @@ jobs: - id: github-status name: Check GitHub status uses: crazy-max/ghaction-github-status@v3 + - id: dump-context + name: Dump context + uses: crazy-max/ghaction-dump-context@v2 lint: needs: - diagnostics From c5e56a21c70e033b2093355b630dd172f85614f2 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 30 Aug 2023 10:36:31 -0400 Subject: [PATCH 132/326] Give the diagnostics job a descriptive name --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 346fd81..53c76f7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,6 +15,7 @@ env: jobs: diagnostics: + name: Run diagnostics runs-on: ubuntu-latest steps: - id: github-status From 9afb516c20153c850088cfcc8318d7df3e68da92 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 30 Aug 2023 10:47:40 -0400 Subject: [PATCH 133/326] Add the step-security/harden-runner GH Action This GH Action is being configured to run in audit mode. It should warn us if an Action is reaching out to an unexpected web address, overwriting source code, etc. Co-authored-by: felddy --- .github/dependabot.yml | 1 + .github/workflows/build.yml | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index dbb9b73..17220c6 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -18,6 +18,7 @@ updates: # - dependency-name: crazy-max/ghaction-github-status # - dependency-name: hashicorp/setup-terraform # - dependency-name: mxschmitt/action-tmate + # - dependency-name: step-security/harden-runner package-ecosystem: github-actions schedule: interval: weekly diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 53c76f7..2fda8f7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,6 +18,11 @@ jobs: name: Run diagnostics runs-on: ubuntu-latest steps: + - id: harden-runner + name: Harden the runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit - id: github-status name: Check GitHub status uses: crazy-max/ghaction-github-status@v3 From 9dc773cae76fcb144a8fde5b26118e915dea1a03 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 30 Aug 2023 12:44:35 -0400 Subject: [PATCH 134/326] Add a harden-runner task to the lint job as well This task can only provide coverage for the job that contains it. --- .github/workflows/build.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2fda8f7..3fbeff2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,6 +34,11 @@ jobs: - diagnostics runs-on: ubuntu-latest steps: + - id: harden-runner + name: Harden the runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit - id: setup-env uses: cisagov/setup-env-github-action@develop - uses: actions/checkout@v4 From bb81ec3a0418359c28fc495524959f9f897557b8 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 30 Aug 2023 15:15:16 -0400 Subject: [PATCH 135/326] Add a reminder We need a reminder add the step-security/harden-runner action at the top of every job. Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3fbeff2..e7b75a4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,6 +18,8 @@ jobs: name: Run diagnostics runs-on: ubuntu-latest steps: + # Note that a duplicate of this step must be added at the top of + # each job. - id: harden-runner name: Harden the runner uses: step-security/harden-runner@v2 From b5e5c11bcd63fcf556d31e3753eb7b84e0a85e3e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 13 Sep 2023 02:01:21 +0000 Subject: [PATCH 136/326] Bump crazy-max/ghaction-github-status from 3 to 4 Bumps [crazy-max/ghaction-github-status](https://github.com/crazy-max/ghaction-github-status) from 3 to 4. - [Release notes](https://github.com/crazy-max/ghaction-github-status/releases) - [Commits](https://github.com/crazy-max/ghaction-github-status/compare/v3...v4) --- updated-dependencies: - dependency-name: crazy-max/ghaction-github-status dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7b75a4..894bb58 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,7 +27,7 @@ jobs: egress-policy: audit - id: github-status name: Check GitHub status - uses: crazy-max/ghaction-github-status@v3 + uses: crazy-max/ghaction-github-status@v4 - id: dump-context name: Dump context uses: crazy-max/ghaction-dump-context@v2 From 371179ef14354df4d00d7f961754ddb86c30929b Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 13 Sep 2023 12:48:59 -0400 Subject: [PATCH 137/326] Add a diagnostics job for the label syncing workflow Also add a runner hardening task to the labeler job. --- .github/workflows/sync-labels.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/.github/workflows/sync-labels.yml b/.github/workflows/sync-labels.yml index 44e8e19..5a20438 100644 --- a/.github/workflows/sync-labels.yml +++ b/.github/workflows/sync-labels.yml @@ -11,7 +11,26 @@ permissions: contents: read jobs: + diagnostics: + name: Run diagnostics + runs-on: ubuntu-latest + steps: + # Note that a duplicate of this step must be added at the top of + # each job. + - id: harden-runner + name: Harden the runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit + - id: github-status + name: Check GitHub status + uses: crazy-max/ghaction-github-status@v3 + - id: dump-context + name: Dump context + uses: crazy-max/ghaction-dump-context@v2 labeler: + needs: + - diagnostics permissions: # actions/checkout needs this to fetch code contents: read @@ -19,6 +38,11 @@ jobs: issues: write runs-on: ubuntu-latest steps: + - id: harden-runner + name: Harden the runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit - uses: actions/checkout@v4 - name: Sync repository labels if: success() From 730485747b7df3f325b8949ffeeac1a4493d3607 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 13 Sep 2023 15:47:26 -0400 Subject: [PATCH 138/326] Enable dependabot ignore directives Enable the new dependabot ignore directives that were added in cisagov/skeleton-generic. --- .github/dependabot.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 633713c..ac4c7a2 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -18,12 +18,12 @@ updates: - dependency-name: actions/checkout - dependency-name: actions/setup-go - dependency-name: actions/setup-python - # - dependency-name: crazy-max/ghaction-dump-context - # - dependency-name: crazy-max/ghaction-github-labeler - # - dependency-name: crazy-max/ghaction-github-status + - dependency-name: crazy-max/ghaction-dump-context + - dependency-name: crazy-max/ghaction-github-labeler + - dependency-name: crazy-max/ghaction-github-status - dependency-name: hashicorp/setup-terraform - dependency-name: mxschmitt/action-tmate - # - dependency-name: step-security/harden-runner + - dependency-name: step-security/harden-runner # Managed by cisagov/skeleton-docker # - dependency-name: actions/download-artifact # - dependency-name: actions/github-script From d86ac5b31a735d399dda9f5d1bf665d4995a7408 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 13 Sep 2023 15:47:55 -0400 Subject: [PATCH 139/326] Remove unnecessary quotes in the dependabot configuration We generally only use quotes when they are strictly necessary to ensure data is interpreted as a string value. This mirrors what was done to the configurations inherited from cisagov/skeleton-generic. --- .github/dependabot.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index ac4c7a2..6594ea3 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -6,10 +6,10 @@ # and Lineage processes these changes. updates: - - package-ecosystem: "docker" - directory: "/" + - package-ecosystem: docker + directory: / schedule: - interval: "weekly" + interval: weekly - directory: / ignore: From 4186c1a39deae02ee9a6a2f5a016d9d81094cb77 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 13 Sep 2023 15:48:33 -0400 Subject: [PATCH 140/326] Sort the keys in the Dependabot configuration Our standard practice for YAML files is to sort keys alphabetically. This mirrors what was done to the configurations inherited from cisagov/skeleton-generic. --- .github/dependabot.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 6594ea3..336eaec 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -6,8 +6,8 @@ # and Lineage processes these changes. updates: - - package-ecosystem: docker - directory: / + - directory: / + package-ecosystem: docker schedule: interval: weekly From 95cb00540a72411011b593dde0d0142f7af38cfb Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 13 Sep 2023 15:49:56 -0400 Subject: [PATCH 141/326] Bump actions/checkout from 3 to 4 This updates the remaining declarations to match what was pulled down from cisagov/skeleton-generic. --- .github/workflows/build.yml | 8 ++++---- .github/workflows/codeql-analysis.yml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1f16c18..2a11aed 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -213,7 +213,7 @@ jobs: source_version: ${{ steps.prep.outputs.source_version }} tags: ${{ steps.prep.outputs.tags }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Gather repository metadata id: repo uses: actions/github-script@v6 @@ -272,7 +272,7 @@ jobs: needs: [prepare] steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up QEMU uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx @@ -341,7 +341,7 @@ jobs: runs-on: ubuntu-latest needs: [build] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - id: setup-python uses: actions/setup-python@v4 with: @@ -404,7 +404,7 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up QEMU uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index bf0d148..05364ee 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -38,7 +38,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL From c9f43e4a04c102eb4eace787579a3e628ccbc2fb Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 13 Sep 2023 15:54:02 -0400 Subject: [PATCH 142/326] Update job dependencies for the build.yml workflow Add the `diagnostics` job as a dependency for the other jobs. Reformat the dependencies for those jobs to match the formatting of the `lint` job. This aligns with what was pulled down from cisagov/skeleton-generic. --- .github/workflows/build.yml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2a11aed..dd6674b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -206,6 +206,8 @@ jobs: # # Scheduled builds are tagged with `:nightly`. name: "Prepare build variables" + needs: + - diagnostics runs-on: ubuntu-latest outputs: created: ${{ steps.prep.outputs.created }} @@ -269,7 +271,9 @@ jobs: # as an artifact and loaded by the test job. name: "Build test image" runs-on: ubuntu-latest - needs: [prepare] + needs: + - diagnostics + - prepare steps: - name: Checkout uses: actions/checkout@v4 @@ -339,7 +343,9 @@ jobs: # Executes tests on the single-platform image created in the "build" job. name: "Test image" runs-on: ubuntu-latest - needs: [build] + needs: + - diagnostics + - build steps: - uses: actions/checkout@v4 - id: setup-python @@ -385,7 +391,11 @@ jobs: # triggering event is a pull request. name: "Build and push all platforms" runs-on: ubuntu-latest - needs: [lint, prepare, test] + needs: + - diagnostics + - lint + - prepare + - test if: github.event_name != 'pull_request' # When Dependabot creates a PR it requires this permission in # order to push Docker images to ghcr.io. From 138fc0f77abde1ae3d012a3d4f974ab5fce134c4 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 13 Sep 2023 15:56:49 -0400 Subject: [PATCH 143/326] Adjust key ordering for jobs in the build.yml workflow Ensure that top-level keys except for `steps` are alphabetically sorted. --- .github/workflows/build.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index dd6674b..843798c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -208,12 +208,12 @@ jobs: name: "Prepare build variables" needs: - diagnostics - runs-on: ubuntu-latest outputs: created: ${{ steps.prep.outputs.created }} repometa: ${{ steps.repo.outputs.result }} source_version: ${{ steps.prep.outputs.source_version }} tags: ${{ steps.prep.outputs.tags }} + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Gather repository metadata @@ -270,10 +270,10 @@ jobs: # Builds a single test image for the native platform. This image is saved # as an artifact and loaded by the test job. name: "Build test image" - runs-on: ubuntu-latest needs: - diagnostics - prepare + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -342,10 +342,10 @@ jobs: test: # Executes tests on the single-platform image created in the "build" job. name: "Test image" - runs-on: ubuntu-latest needs: - diagnostics - build + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - id: setup-python @@ -389,18 +389,18 @@ jobs: # GitHub Container Registry. The contents of README.md are pushed as the # image's description to Docker Hub. This job is skipped when the # triggering event is a pull request. + if: github.event_name != 'pull_request' name: "Build and push all platforms" - runs-on: ubuntu-latest needs: - diagnostics - lint - prepare - test - if: github.event_name != 'pull_request' # When Dependabot creates a PR it requires this permission in # order to push Docker images to ghcr.io. permissions: packages: write + runs-on: ubuntu-latest steps: - name: Login to Docker Hub uses: docker/login-action@v2 From 7eefae068b22a3c0ee2ea116073f5a60dc8c7f93 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 13 Sep 2023 15:58:11 -0400 Subject: [PATCH 144/326] Remove unnecessary quotes in the build.yml workflow We generally only use quotes when they are strictly necessary to ensure data is interpreted as a string value. This mirrors what was done to the configurations inherited from cisagov/skeleton-generic. --- .github/workflows/build.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 843798c..91ab0d4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -55,7 +55,7 @@ jobs: lint: # Checks out the source and runs pre-commit hooks. Detects coding errors # and style deviations. - name: "Lint sources" + name: Lint sources needs: - diagnostics runs-on: ubuntu-latest @@ -205,7 +205,7 @@ jobs: # with the value specified by the user. # # Scheduled builds are tagged with `:nightly`. - name: "Prepare build variables" + name: Prepare build variables needs: - diagnostics outputs: @@ -269,7 +269,7 @@ jobs: build: # Builds a single test image for the native platform. This image is saved # as an artifact and loaded by the test job. - name: "Build test image" + name: Build test image needs: - diagnostics - prepare @@ -341,7 +341,7 @@ jobs: if: env.RUN_TMATE test: # Executes tests on the single-platform image created in the "build" job. - name: "Test image" + name: Test image needs: - diagnostics - build @@ -390,7 +390,7 @@ jobs: # image's description to Docker Hub. This job is skipped when the # triggering event is a pull request. if: github.event_name != 'pull_request' - name: "Build and push all platforms" + name: Build and push all platforms needs: - diagnostics - lint From 4380bec4127a2bcec24a4da54c8d595afb62b4ad Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 13 Sep 2023 16:31:44 -0400 Subject: [PATCH 145/326] Add runner hardening to all jobs in our workflows This aligns with what was done to the `lint` job of the build.yml workflow that was inherited from cisagov/skeleton-generic. --- .github/workflows/build.yml | 20 ++++++++++++++++++++ .github/workflows/codeql-analysis.yml | 6 ++++++ 2 files changed, 26 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 91ab0d4..fd30a13 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -215,6 +215,11 @@ jobs: tags: ${{ steps.prep.outputs.tags }} runs-on: ubuntu-latest steps: + - id: harden-runner + name: Harden the runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit - uses: actions/checkout@v4 - name: Gather repository metadata id: repo @@ -275,6 +280,11 @@ jobs: - prepare runs-on: ubuntu-latest steps: + - id: harden-runner + name: Harden the runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit - name: Checkout uses: actions/checkout@v4 - name: Set up QEMU @@ -347,6 +357,11 @@ jobs: - build runs-on: ubuntu-latest steps: + - id: harden-runner + name: Harden the runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit - uses: actions/checkout@v4 - id: setup-python uses: actions/setup-python@v4 @@ -402,6 +417,11 @@ jobs: packages: write runs-on: ubuntu-latest steps: + - id: harden-runner + name: Harden the runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit - name: Login to Docker Hub uses: docker/login-action@v2 with: diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 05364ee..ef6bfb7 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -37,6 +37,12 @@ jobs: # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection steps: + - id: harden-runner + name: Harden the runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit + - name: Checkout repository uses: actions/checkout@v4 From f0f3e1105babf9b6595b3b54a899e6b545315e92 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 13 Sep 2023 16:35:34 -0400 Subject: [PATCH 146/326] Remove unnecessary quotes in the codeql-analysis.yml workflow We generally only use quotes when they are strictly necessary to ensure data is interpreted as a string value. This mirrors what was done to the configurations inherited from cisagov/skeleton-generic. --- .github/workflows/codeql-analysis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index ef6bfb7..41309ce 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -4,7 +4,7 @@ # # You may wish to alter this file to override the set of languages analyzed, # or to provide custom queries or build logic. -name: "CodeQL" +name: CodeQL on: push: From 1f611fc67710100e3e6efeb395d67b82e3f52eaa Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Thu, 14 Sep 2023 15:45:17 -0400 Subject: [PATCH 147/326] Make the dev team the owners of the linter configuration files Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- .github/CODEOWNERS | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 8f5c8c5..a22502d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -8,3 +8,12 @@ # These folks own any files in the .github directory at the root of # the repository and any of its subdirectories. /.github/ @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj + +# These folks own all linting configuration files. +/.*.cfg @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.*.yaml @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.*.yml @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.ansible-lint @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.flake8 @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.prettierignore @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.yamllint @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj From c356768305b027d915db7ef7be1126ee687757ad Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Thu, 14 Sep 2023 15:59:22 -0400 Subject: [PATCH 148/326] Make dev team members the codeowners of the requirements*.txt and setup-env files Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- .github/CODEOWNERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index a22502d..b4e689e 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -17,3 +17,5 @@ /.flake8 @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj /.prettierignore @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj /.yamllint @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/requirements*.txt @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/setup-env @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj From 0195005cccab3e087ec8353c5049ea633d252835 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 15 Sep 2023 10:26:45 -0400 Subject: [PATCH 149/326] Explicitly list the linter config files the dev team should own @mcdonnnj correctly pointed out that other projects add their own configuration files that match, e.g., the /.*.yaml pattern. We want to ensure that we only own the linter configuration files from the skeleton. Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- .github/CODEOWNERS | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b4e689e..229920c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -10,12 +10,15 @@ /.github/ @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj # These folks own all linting configuration files. -/.*.cfg @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/.*.yaml @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/.*.yml @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj /.ansible-lint @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.bandit.yml @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj /.flake8 @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.isort.cfg @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.mdl_config.yaml @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.pre-commit-config.yaml @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj /.prettierignore @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj /.yamllint @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/requirements*.txt @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/requirements.txt @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/requirements-dev.txt @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/requirements-test.txt @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj /setup-env @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj From 6ee9882f5218d33bf1b2f63a2647b83719fb13d1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Sep 2023 17:56:41 +0000 Subject: [PATCH 150/326] Bump docker/login-action from 2 to 3 Bumps [docker/login-action](https://github.com/docker/login-action) from 2 to 3. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/v2...v3) --- updated-dependencies: - dependency-name: docker/login-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d8f6d6d..478ab81 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -354,12 +354,12 @@ jobs: packages: write steps: - name: Login to Docker Hub - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Login to GitHub Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} From 05bb5927b660a045581883916668af6df0ba316c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Sep 2023 17:56:49 +0000 Subject: [PATCH 151/326] Bump docker/setup-buildx-action from 2 to 3 Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 2 to 3. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/v2...v3) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d8f6d6d..8077439 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -237,7 +237,7 @@ jobs: - name: Set up QEMU uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Cache Docker layers uses: actions/cache@v3 env: @@ -369,7 +369,7 @@ jobs: - name: Set up QEMU uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Cache Docker layers uses: actions/cache@v3 env: From f05a983491df939d1db9f6a7b0adf118aecd83dd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Sep 2023 17:56:52 +0000 Subject: [PATCH 152/326] Bump docker/setup-qemu-action from 2 to 3 Bumps [docker/setup-qemu-action](https://github.com/docker/setup-qemu-action) from 2 to 3. - [Release notes](https://github.com/docker/setup-qemu-action/releases) - [Commits](https://github.com/docker/setup-qemu-action/compare/v2...v3) --- updated-dependencies: - dependency-name: docker/setup-qemu-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d8f6d6d..0a07f34 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -235,7 +235,7 @@ jobs: - name: Checkout uses: actions/checkout@v3 - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - name: Cache Docker layers @@ -367,7 +367,7 @@ jobs: - name: Checkout uses: actions/checkout@v3 - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - name: Cache Docker layers From 768b6cbc9baee5e8829035524e3d88d18776e0a0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Oct 2023 17:36:17 +0000 Subject: [PATCH 153/326] Bump python from 3.11.4-alpine to 3.12.0-alpine Bumps python from 3.11.4-alpine to 3.12.0-alpine. --- updated-dependencies: - dependency-name: python dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 4aea26a..fc7b5c2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ ARG VERSION=unspecified -FROM python:3.11.4-alpine +FROM python:3.12.0-alpine ARG VERSION From 8c6b0961738d6036f61035c253b5fc9933a5800c Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 11 Oct 2023 16:15:52 -0400 Subject: [PATCH 154/326] Add a diagnostics job to the CodeQL workflow It's good to agree everywhere with the changes we made to the build.yml workflow in cisagov/skeleton-generic#144. --- .github/workflows/codeql-analysis.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index bf0d148..2a1ca7f 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -20,8 +20,27 @@ on: - cron: '0 21 * * 6' jobs: + diagnostics: + name: Run diagnostics + runs-on: ubuntu-latest + steps: + # Note that a duplicate of this step must be added at the top of + # each job. + - id: harden-runner + name: Harden the runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit + - id: github-status + name: Check GitHub status + uses: crazy-max/ghaction-github-status@v3 + - id: dump-context + name: Dump context + uses: crazy-max/ghaction-dump-context@v2 analyze: name: Analyze + needs: + - diagnostics runs-on: ubuntu-latest permissions: # required for all workflows @@ -37,6 +56,12 @@ jobs: # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection steps: + - id: harden-runner + name: Harden the runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit + - name: Checkout repository uses: actions/checkout@v3 From b768a289e1702e9e46d81ad5a59f51abe666a976 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Oct 2023 18:41:36 +0000 Subject: [PATCH 155/326] Bump hashicorp/setup-terraform from 2 to 3 Bumps [hashicorp/setup-terraform](https://github.com/hashicorp/setup-terraform) from 2 to 3. - [Release notes](https://github.com/hashicorp/setup-terraform/releases) - [Changelog](https://github.com/hashicorp/setup-terraform/blob/main/CHANGELOG.md) - [Commits](https://github.com/hashicorp/setup-terraform/compare/v2...v3) --- updated-dependencies: - dependency-name: hashicorp/setup-terraform dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7b75a4..1ff72a9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -101,7 +101,7 @@ jobs: ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" sudo mv /usr/local/bin/packer /usr/local/bin/packer-default sudo ln -s /opt/packer/packer /usr/local/bin/packer - - uses: hashicorp/setup-terraform@v2 + - uses: hashicorp/setup-terraform@v3 with: terraform_version: ${{ steps.setup-env.outputs.terraform-version }} - name: Install go-critic From 9f317002ac85066adc1efa05d9c2f55275ef7d9c Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 2 Nov 2023 15:49:06 -0400 Subject: [PATCH 156/326] Prefer block style to flow style We prefer block style to flow style for sequences and mappings in YAML. --- .github/workflows/build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7b75a4..9332263 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -5,7 +5,8 @@ on: push: pull_request: repository_dispatch: - types: [apb] + types: + - apb env: CURL_CACHE_DIR: ~/.cache/curl From 696433ae2c3b3f9f2b3613562d3889c753c7bc34 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 2 Nov 2023 15:57:37 -0400 Subject: [PATCH 157/326] Alphabetize entries in the build workflow We prefer to alphabetize mapping keys in YAML documents whenever possible. --- .github/workflows/build.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9332263..2fef5d5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,8 +2,8 @@ name: build on: - push: pull_request: + push: repository_dispatch: types: - apb @@ -58,8 +58,8 @@ jobs: # it relies on the existence of a go.sum file. cache: false go-version: "1.20" - - name: Lookup Go cache directory - id: go-cache + - id: go-cache + name: Lookup Go cache directory run: | echo "dir=$(go env GOCACHE)" >> $GITHUB_OUTPUT - uses: actions/cache@v3 @@ -70,6 +70,10 @@ jobs: packer${{ steps.setup-env.outputs.packer-version }}-\ tf${{ steps.setup-env.outputs.terraform-version }}-" with: + key: "${{ env.BASE_CACHE_KEY }}\ + ${{ hashFiles('**/requirements-test.txt') }}-\ + ${{ hashFiles('**/requirements.txt') }}-\ + ${{ hashFiles('**/.pre-commit-config.yaml') }}" # Note that the .terraform directory IS NOT included in the # cache because if we were caching, then we would need to use # the `-upgrade=true` option. This option blindly pulls down the @@ -81,10 +85,6 @@ jobs: ${{ env.PRE_COMMIT_CACHE_DIR }} ${{ env.CURL_CACHE_DIR }} ${{ steps.go-cache.outputs.dir }} - key: "${{ env.BASE_CACHE_KEY }}\ - ${{ hashFiles('**/requirements-test.txt') }}-\ - ${{ hashFiles('**/requirements.txt') }}-\ - ${{ hashFiles('**/.pre-commit-config.yaml') }}" restore-keys: | ${{ env.BASE_CACHE_KEY }} - name: Setup curl cache From 6503a9e7f67d908833ff5aa178a2fd28faf5e803 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 2 Nov 2023 16:00:00 -0400 Subject: [PATCH 158/326] Add a `merge_group` trigger to the build workflow This should improve compatibility with merge queues. We configure it to only trigger on the `checks_requested` type which is currently the only supported type for this trigger. If additional types are added in the future they should be added if appropriate. --- .github/workflows/build.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2fef5d5..1928cb0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,6 +2,9 @@ name: build on: + merge_group: + types: + - checks_requested pull_request: push: repository_dispatch: From b92cc514ab81c762de7dfe14dbc6d03871f1a929 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 6 Dec 2023 14:43:11 +0000 Subject: [PATCH 159/326] Bump actions/github-script from 6 to 7 Bumps [actions/github-script](https://github.com/actions/github-script) from 6 to 7. - [Release notes](https://github.com/actions/github-script/releases) - [Commits](https://github.com/actions/github-script/compare/v6...v7) --- updated-dependencies: - dependency-name: actions/github-script dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2c0062f..5bc2ffb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -223,7 +223,7 @@ jobs: - uses: actions/checkout@v4 - name: Gather repository metadata id: repo - uses: actions/github-script@v6 + uses: actions/github-script@v7 with: script: | const repo = await github.rest.repos.get(context.repo) From 0196a0d3bd133df8aa4449b60f269f16b6cf8dc3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 6 Dec 2023 16:27:00 +0000 Subject: [PATCH 160/326] Bump docker/build-push-action from 4 to 5 Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 4 to 5. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v4...v5) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2c0062f..234a1b8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -304,7 +304,7 @@ jobs: run: mkdir -p dist - name: Build image id: docker_build - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: build-args: | VERSION=${{ needs.prepare.outputs.source_version }} @@ -452,7 +452,7 @@ jobs: run: ./buildx-dockerfile.sh - name: Build and push platform images to registries id: docker_build - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: build-args: | VERSION=${{ needs.prepare.outputs.source_version }} From ef3438ffa481f187b2f9ef59b82015b834586e64 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 6 Dec 2023 11:32:45 -0500 Subject: [PATCH 161/326] Add commented out option to docker/build-push-action There is currently a potential incompatibility with the default behavior of the version of buildx being used. A default image generated is built with provenance, which is something we would like to have, but these default images can run on neither Google Cloud Run nor AWS Lambda. Please see docker/buildx#1533 for mroe information. Since we want to retain support for creating AWS Lambda images we add a commented out disabling of this functionality that can be enabled in a downstream repository if needed. --- .github/workflows/build.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 234a1b8..37c3d38 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -313,6 +313,11 @@ jobs: context: . file: ./Dockerfile outputs: type=docker,dest=dist/image.tar + # Uncomment the following option if you are building an image for use + # on Google Cloud Run or AWS Lambda. The current default image output + # is unable to run on either. Please see the following issue for more + # information: https://github.com/docker/buildx/issues/1533 + # provenance: false tags: ${{ env.IMAGE_NAME }}:latest # not to be pushed # For a list of pre-defined annotation keys and value types see: # https://github.com/opencontainers/image-spec/blob/master/annotations.md @@ -461,6 +466,11 @@ jobs: context: . file: ./Dockerfile-x platforms: ${{ env.PLATFORMS }} + # Uncomment the following option if you are building an image for use + # on Google Cloud Run or AWS Lambda. The current default image output + # is unable to run on either. Please see the following issue for more + # information: https://github.com/docker/buildx/issues/1533 + # provenance: false push: true tags: ${{ needs.prepare.outputs.tags }} # For a list of pre-defined annotation keys and value types see: From 193e799a2601d030a0dd7bb6608752a076dc5981 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Dec 2023 18:59:21 +0000 Subject: [PATCH 162/326] Bump actions/setup-go from 4 to 5 Bumps [actions/setup-go](https://github.com/actions/setup-go) from 4 to 5. - [Release notes](https://github.com/actions/setup-go/releases) - [Commits](https://github.com/actions/setup-go/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/setup-go dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7b75a4..f208d82 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -51,7 +51,7 @@ jobs: # We need the Go version and Go cache location for the actions/cache step, # so the Go installation must happen before that. - id: setup-go - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: # There is no expectation for actual Go code so we disable caching as # it relies on the existence of a go.sum file. From 5c84295bb6811bc49d3ff29de067adb2d179f879 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Dec 2023 18:59:25 +0000 Subject: [PATCH 163/326] Bump actions/setup-python from 4 to 5 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4 to 5. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7b75a4..366a16b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -45,7 +45,7 @@ jobs: uses: cisagov/setup-env-github-action@develop - uses: actions/checkout@v4 - id: setup-python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.11" # We need the Go version and Go cache location for the actions/cache step, From aad19725f6d4924eec9366f3e9fe6c33e0af758f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Dec 2023 17:31:17 +0000 Subject: [PATCH 164/326] Bump github/codeql-action from 2 to 3 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 2 to 3. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/v2...v3) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index dc49271..ad0724a 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -67,7 +67,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} @@ -75,7 +75,7 @@ jobs: # Java). If this step fails, then you should remove it and run the build # manually (see below). - name: Autobuild - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@v3 # ℹī¸ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -89,4 +89,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v3 From 4a63dbe91c946d74d963c103b67d4f3c0746f35c Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 18 Jan 2024 16:25:35 -0500 Subject: [PATCH 165/326] Switch pre-commit hooks for running shfmt This hook bundles the binaries for shfmt with a Python package which removes the need to manually install the tool for the hook to function. --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 34f0253..ed254a0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -86,8 +86,8 @@ repos: - id: nixpkgs-fmt # Shell script hooks - - repo: https://github.com/cisagov/pre-commit-shfmt - rev: v0.0.2 + - repo: https://github.com/scop/pre-commit-shfmt + rev: v3.7.0-4 hooks: - id: shfmt args: From 3236b1ba94d1c833dc2f9508944c6bc6e6a49f48 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 18 Jan 2024 17:43:00 -0500 Subject: [PATCH 166/326] Remove installation of shfmt in the `build` workflow The new pre-commit hook provides `shfmt` binaries so we no longer need to ensure it is installed. --- .github/workflows/build.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7b75a4..b5d46cc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -114,11 +114,6 @@ jobs: PACKAGE_URL: github.com/securego/gosec/v2/cmd/gosec PACKAGE_VERSION: ${{ steps.setup-env.outputs.gosec-version }} run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - - name: Install shfmt - env: - PACKAGE_URL: mvdan.cc/sh/v3/cmd/shfmt - PACKAGE_VERSION: ${{ steps.setup-env.outputs.shfmt-version }} - run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - name: Install staticcheck env: PACKAGE_URL: honnef.co/go/tools/cmd/staticcheck From 5ddb14dab4e98019a6875278a1bc556c61e4bb69 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 18 Jan 2024 16:56:43 -0500 Subject: [PATCH 167/326] Use long options for shfmt arguments Since shfmt now supports long command line options we should use them as that is our preference. The single quotes for the number of spaces to indent is changed to double quotes to align with our usual quotation style. --- .pre-commit-config.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ed254a0..ca36e5d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -92,14 +92,14 @@ repos: - id: shfmt args: # Indent by two spaces - - -i - - '2' + - --indent + - "2" # Binary operators may start a line - - -bn + - --binary-next-line # Switch cases are indented - - -ci + - --case-indent # Redirect operators are followed by a space - - -sr + - --space-redirects - repo: https://github.com/detailyang/pre-commit-shell rev: 1.0.5 hooks: From 8ecd95718a6d1b0db176cb85bf0790e4810364fb Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 18 Jan 2024 17:34:18 -0500 Subject: [PATCH 168/326] Add additional shfmt options These options are baked into the functionality of the old hook but must be explicitly declared for the new hook. --- .pre-commit-config.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ca36e5d..6b35e4a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -91,6 +91,10 @@ repos: hooks: - id: shfmt args: + # List files that will be formatted + - --list + # Write result to file instead of stdout + - --write # Indent by two spaces - --indent - "2" From 242921b9856e3ede0fa1a941324c1a3f0ba116f7 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 21 Sep 2023 17:49:55 -0400 Subject: [PATCH 169/326] Set the default shell for all run steps in the build workflow This sets the default shell for any run steps in the build workflow to mirror our standard shellscript writing practices. In addition to enabling our standard options it will also enable errtrace and print any commands that are run which should make debugging/troubleshooting more straightforward. --- .github/workflows/build.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7b75a4..7299f6b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -7,6 +7,14 @@ on: repository_dispatch: types: [apb] +# Set a default shell for any run steps. The `-Eueo pipefail` sets errtrace, +# nounset, errexit, and pipefail. The `-x` will print all commands as they are +# run. Please see the GitHub Actions documentation for more information: +# https://docs.github.com/en/actions/using-jobs/setting-default-values-for-jobs +defaults: + run: + shell: bash -Eueo pipefail -x {0} + env: CURL_CACHE_DIR: ~/.cache/curl PIP_CACHE_DIR: ~/.cache/pip From c7b18dc7e57c5261720324bd5b1425f6992e965e Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 12 Jan 2024 17:55:34 -0500 Subject: [PATCH 170/326] Add linting with goimports to the pre-commit configuration This will run the Go tool `goimports` against the repository if it contains any Go files. This tool bundles the functionality of `go fmt` with the additional benefit of sorting Go imports much like the isort tool we use for Python code. --- .github/workflows/build.yml | 5 +++++ .pre-commit-config.yaml | 6 +++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7b75a4..8e14eab 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -109,6 +109,11 @@ jobs: PACKAGE_URL: github.com/go-critic/go-critic/cmd/gocritic PACKAGE_VERSION: ${{ steps.setup-env.outputs.go-critic-version }} run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} + - name: Install goimports + env: + PACKAGE_URL: golang.org/x/tools/cmd/goimports + PACKAGE_VERSION: ${{ steps.setup-env.outputs.goimports-version }} + run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - name: Install gosec env: PACKAGE_URL: github.com/securego/gosec/v2/cmd/gosec diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 34f0253..7bb1f8c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -78,7 +78,11 @@ repos: - id: go-vet-repo-mod # GoSec - id: go-sec-repo-mod - + # goimports + - id: go-imports-repo + args: + # Write changes to files + - -w # Nix hooks - repo: https://github.com/nix-community/nixpkgs-fmt rev: v1.3.0 From f6d9d6e29249c4b6246fca71c285d744c7409f38 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Mon, 22 Jan 2024 12:31:43 -0600 Subject: [PATCH 171/326] Add ATX Header Support for terraform-docs This is a temporary fix until @mcdonnnj has his PR approved and merged into the terraform-docs repo. This fix will perform a shallow clone of his forked branch, build the binary, and install it. --- .github/workflows/build.yml | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7b75a4..e86b966 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,9 +8,12 @@ on: types: [apb] env: + BRANCH_NAME: improvement/support_atx_closed_markdown_headers CURL_CACHE_DIR: ~/.cache/curl + DEPTH: 1 PIP_CACHE_DIR: ~/.cache/pip PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit + REPO_URL: https://github.com/mcdonnnj/terraform-docs.git RUN_TMATE: ${{ secrets.RUN_TMATE }} jobs: @@ -124,11 +127,20 @@ jobs: PACKAGE_URL: honnef.co/go/tools/cmd/staticcheck PACKAGE_VERSION: ${{ steps.setup-env.outputs.staticcheck-version }} run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - - name: Install Terraform-docs - env: - PACKAGE_URL: github.com/terraform-docs/terraform-docs - PACKAGE_VERSION: ${{ steps.setup-env.outputs.terraform-docs-version }} - run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} + # We are temporarily using @mcdonnnj's forked branch of terraform-docs + # until his PR: https://github.com/terraform-docs/terraform-docs/pull/745 + # is approved. This temporary fix will allow for ATX Header Support when + # terraform-docs is ran during lint. + - name: Clone ATX Headers branch from Terraform-docs Fork + run: | + git clone --branch $BRANCH_NAME --single-branch \ + --depth=$DEPTH $REPO_URL /tmp/terraform-docs + - name: Build and Install Terraform-docs Binary + run: | + cd /tmp/terraform-docs + GOBIN=$(go env GOPATH)/bin + go build -o $GOBIN/terraform-docs + echo "$GOBIN" >> $GITHUB_PATH - name: Install dependencies run: | python -m pip install --upgrade pip setuptools wheel From 544e4789e052978d00f982ef45d8d0de9f5e4748 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Mon, 22 Jan 2024 14:03:00 -0600 Subject: [PATCH 172/326] Add prepended names to variables to describe their function Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- .github/workflows/build.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e86b966..99b42c1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,13 +8,13 @@ on: types: [apb] env: - BRANCH_NAME: improvement/support_atx_closed_markdown_headers CURL_CACHE_DIR: ~/.cache/curl - DEPTH: 1 PIP_CACHE_DIR: ~/.cache/pip PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit - REPO_URL: https://github.com/mcdonnnj/terraform-docs.git RUN_TMATE: ${{ secrets.RUN_TMATE }} + TERRAFORM_DOCS_REPO_BRANCH_NAME: improvement/support_atx_closed_markdown_headers + TERRAFORM_DOCS_REPO_DEPTH: 1 + TERRAFORM_DOCS_REPO_URL: https://github.com/mcdonnnj/terraform-docs.git jobs: diagnostics: @@ -133,8 +133,10 @@ jobs: # terraform-docs is ran during lint. - name: Clone ATX Headers branch from Terraform-docs Fork run: | - git clone --branch $BRANCH_NAME --single-branch \ - --depth=$DEPTH $REPO_URL /tmp/terraform-docs + git clone --single-branch \ + --branch $TERRAFORM_DOCS_REPO_BRANCH_NAME \ + --depth $TERRAFORM_DOCS_REPO_DEPTH \ + $TERRAFORM_DOCS_REPO_URL /tmp/terraform-docs - name: Build and Install Terraform-docs Binary run: | cd /tmp/terraform-docs From f5fa0ff4ec1ded10a2906ade6300bba05cc2e81c Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Mon, 22 Jan 2024 14:05:01 -0600 Subject: [PATCH 173/326] Remove unnecessary capitalizations and fix grammar Co-authored-by: Shane Frasier --- .github/workflows/build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 99b42c1..ca719aa 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -129,15 +129,15 @@ jobs: run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} # We are temporarily using @mcdonnnj's forked branch of terraform-docs # until his PR: https://github.com/terraform-docs/terraform-docs/pull/745 - # is approved. This temporary fix will allow for ATX Header Support when - # terraform-docs is ran during lint. - - name: Clone ATX Headers branch from Terraform-docs Fork + # is approved. This temporary fix will allow for ATX header support when + # terraform-docs is run during linting. + - name: Clone ATX headers branch from terraform-docs fork run: | git clone --single-branch \ --branch $TERRAFORM_DOCS_REPO_BRANCH_NAME \ --depth $TERRAFORM_DOCS_REPO_DEPTH \ $TERRAFORM_DOCS_REPO_URL /tmp/terraform-docs - - name: Build and Install Terraform-docs Binary + - name: Build and install terraform-docs binary run: | cd /tmp/terraform-docs GOBIN=$(go env GOPATH)/bin From 36361dd1c74c9855b87f2b4aec5a9be1fa7c416e Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Mon, 22 Jan 2024 14:14:27 -0600 Subject: [PATCH 174/326] Simplify steps in the build/install portion of workflow PATH is handled by `setup-go` so we can refactor the code setting it. Also we are taking advantage of the -C switch to handle building from the cloned repository. Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- .github/workflows/build.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ca719aa..6ab90de 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -139,10 +139,9 @@ jobs: $TERRAFORM_DOCS_REPO_URL /tmp/terraform-docs - name: Build and install terraform-docs binary run: | - cd /tmp/terraform-docs - GOBIN=$(go env GOPATH)/bin - go build -o $GOBIN/terraform-docs - echo "$GOBIN" >> $GITHUB_PATH + go build \ + -C /tmp/terraform-docs \ + -o $(go env GOPATH)/bin/terraform-docs - name: Install dependencies run: | python -m pip install --upgrade pip setuptools wheel From 3711ebe461d81987b4121903d2f3f1b290fccad3 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 23 Jan 2024 08:04:07 -0600 Subject: [PATCH 175/326] Add TODO label --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6ab90de..f5ba773 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -51,7 +51,7 @@ jobs: uses: actions/setup-python@v4 with: python-version: "3.11" - # We need the Go version and Go cache location for the actions/cache step, + # TODO: We need the Go version and Go cache location for the actions/cache step, # so the Go installation must happen before that. - id: setup-go uses: actions/setup-go@v4 From d114fb4233f22cb9edb3e1a6555ca0a677bcfc7e Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 23 Jan 2024 08:24:12 -0600 Subject: [PATCH 176/326] Move TODO and add link to the issue `TODO` was placed on the wrong comment block. Also I am adding a link to the issue for the TODO. --- .github/workflows/build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f5ba773..0068f1f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -51,7 +51,7 @@ jobs: uses: actions/setup-python@v4 with: python-version: "3.11" - # TODO: We need the Go version and Go cache location for the actions/cache step, + # We need the Go version and Go cache location for the actions/cache step, # so the Go installation must happen before that. - id: setup-go uses: actions/setup-go@v4 @@ -127,6 +127,7 @@ jobs: PACKAGE_URL: honnef.co/go/tools/cmd/staticcheck PACKAGE_VERSION: ${{ steps.setup-env.outputs.staticcheck-version }} run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} + # TODO: https://github.com/cisagov/skeleton-generic/issues/165 # We are temporarily using @mcdonnnj's forked branch of terraform-docs # until his PR: https://github.com/terraform-docs/terraform-docs/pull/745 # is approved. This temporary fix will allow for ATX header support when From c907cfc82a8f329fbd4ad0ee17f3500922c51711 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 23 Jan 2024 08:41:41 -0600 Subject: [PATCH 177/326] Alphabetize switches Co-authored-by: dav3r --- .github/workflows/build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0068f1f..35391d5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -134,9 +134,10 @@ jobs: # terraform-docs is run during linting. - name: Clone ATX headers branch from terraform-docs fork run: | - git clone --single-branch \ + git clone \ --branch $TERRAFORM_DOCS_REPO_BRANCH_NAME \ --depth $TERRAFORM_DOCS_REPO_DEPTH \ + --single-branch \ $TERRAFORM_DOCS_REPO_URL /tmp/terraform-docs - name: Build and install terraform-docs binary run: | From 48db3e3381aa61bdbe67dab64454db3e7f332c75 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Thu, 25 Jan 2024 12:19:41 -0600 Subject: [PATCH 178/326] Allow setup-env to specify Python version This commit is introducing 2 new flags into the setup-env script. -l or --list-versions will list available Python versions and allow the user to select a version interactively. The second flag -v or --version will allow a user to set the version if installed. (e.g. ./setup-env -v 3.9.6) --- setup-env | 42 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/setup-env b/setup-env index 77926bf..b99ec5f 100755 --- a/setup-env +++ b/setup-env @@ -25,6 +25,8 @@ Options: -h --help Show this message. -i --install-hooks Install hook environments for all environments in the pre-commit config file. + -v --version Specify the Python version for the virtual environment. + -l --list-versions List available Python versions and select interactively. END_OF_LINE ) @@ -35,6 +37,13 @@ FORCE=0 # Positional parameters PARAMS="" +# Flags to allow a user to specify which version of Python they want to use +PYTHON_VERSION="" +LIST_VERSIONS=0 + +# Temp file that is used to search through available installed Python versions +TMPFILE=/tmp/versions.$$ + # Parse command line arguments while (("$#")); do case "$1" in @@ -50,6 +59,14 @@ while (("$#")); do INSTALL_HOOKS=1 shift ;; + -v | --version) + PYTHON_VERSION=$2 + shift 2 + ;; + -l | --list-versions) + LIST_VERSIONS=1 + shift + ;; -*) # unsupported flags echo "Error: Unsupported flag $1" >&2 exit 1 @@ -111,6 +128,25 @@ else fi set -o nounset +# List Python versions and select one interactively +if [ $LIST_VERSIONS -ne 0 ]; then + echo Available Python versions: + pyenv versions --bare --skip-aliases --skip-envs + read -p -r "Enter the desired Python version: " PYTHON_VERSION +fi + +# Check if PYTHON_VERSION isn't empty. If it is installed, set it locally. +pyenv versions --bare --skip-aliases --skip-envs > $TMPFILE +if [ -n "$PYTHON_VERSION" ]; then + if grep --fixed-strings --quiet "$PYTHON_VERSION" $TMPFILE; then + echo Using Python version "$PYTHON_VERSION" + pyenv local "$PYTHON_VERSION" + else + echo Error: Python version "$PYTHON_VERSION" is not installed. + fi + exit 1 +fi + # Remove any lingering local configuration. if [ $FORCE -ne 0 ]; then rm -f .python-version @@ -130,10 +166,10 @@ fi # Create a new virtual environment for this project if ! pyenv virtualenv "${env_name}"; then cat << END_OF_LINE - An existing virtual environment named $env_name was found. Either delete this - environment yourself or re-run with --force option to have it deleted. + An existing virtual environment named $env_name was found. Either delete this + environment yourself or re-run with --force option to have it deleted. - pyenv virtualenv-delete ${env_name} + pyenv virtualenv-delete ${env_name} END_OF_LINE exit 1 From c10929afae693990a3d4d034faa2c1ab1b2689ee Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Thu, 25 Jan 2024 14:15:40 -0600 Subject: [PATCH 179/326] Add /dev/null and remove TMPFILE This makes the code a bit cleaner and still accomplishes the same functionality Co-authored-by: Shane Frasier --- setup-env | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/setup-env b/setup-env index b99ec5f..769522a 100755 --- a/setup-env +++ b/setup-env @@ -41,9 +41,6 @@ PARAMS="" PYTHON_VERSION="" LIST_VERSIONS=0 -# Temp file that is used to search through available installed Python versions -TMPFILE=/tmp/versions.$$ - # Parse command line arguments while (("$#")); do case "$1" in @@ -136,9 +133,8 @@ if [ $LIST_VERSIONS -ne 0 ]; then fi # Check if PYTHON_VERSION isn't empty. If it is installed, set it locally. -pyenv versions --bare --skip-aliases --skip-envs > $TMPFILE if [ -n "$PYTHON_VERSION" ]; then - if grep --fixed-strings --quiet "$PYTHON_VERSION" $TMPFILE; then + if pyenv versions --bare --skip-aliases --skip-envs | grep --fixed-strings "$PYTHON_VERSION" >/dev/null; then echo Using Python version "$PYTHON_VERSION" pyenv local "$PYTHON_VERSION" else From adada40e6fa6aed6fa14f253a5b86f313b3dfa8f Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Thu, 25 Jan 2024 14:19:54 -0600 Subject: [PATCH 180/326] Place flags in the correct order for -r and -p --- setup-env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup-env b/setup-env index 769522a..2adbd5a 100755 --- a/setup-env +++ b/setup-env @@ -129,12 +129,12 @@ set -o nounset if [ $LIST_VERSIONS -ne 0 ]; then echo Available Python versions: pyenv versions --bare --skip-aliases --skip-envs - read -p -r "Enter the desired Python version: " PYTHON_VERSION + read -r -p "Enter the desired Python version: " PYTHON_VERSION fi # Check if PYTHON_VERSION isn't empty. If it is installed, set it locally. if [ -n "$PYTHON_VERSION" ]; then - if pyenv versions --bare --skip-aliases --skip-envs | grep --fixed-strings "$PYTHON_VERSION" >/dev/null; then + if pyenv versions --bare --skip-aliases --skip-envs | grep --fixed-strings "$PYTHON_VERSION" > /dev/null; then echo Using Python version "$PYTHON_VERSION" pyenv local "$PYTHON_VERSION" else From 1861b9b86778613bdaeff804418f2d00706f4a08 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Thu, 25 Jan 2024 14:21:49 -0600 Subject: [PATCH 181/326] Remove unneccessary spacing --- setup-env | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup-env b/setup-env index 2adbd5a..5abb624 100755 --- a/setup-env +++ b/setup-env @@ -162,10 +162,10 @@ fi # Create a new virtual environment for this project if ! pyenv virtualenv "${env_name}"; then cat << END_OF_LINE - An existing virtual environment named $env_name was found. Either delete this - environment yourself or re-run with --force option to have it deleted. + An existing virtual environment named $env_name was found. Either delete this + environment yourself or re-run with --force option to have it deleted. - pyenv virtualenv-delete ${env_name} + pyenv virtualenv-delete ${env_name} END_OF_LINE exit 1 From 3f623e493752c581829271a148cb9fff3ad4f4ab Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Thu, 25 Jan 2024 14:25:37 -0600 Subject: [PATCH 182/326] Alphabetize flags and descriptions Co-authored-by: Shane Frasier --- setup-env | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/setup-env b/setup-env index 5abb624..5282e49 100755 --- a/setup-env +++ b/setup-env @@ -25,8 +25,8 @@ Options: -h --help Show this message. -i --install-hooks Install hook environments for all environments in the pre-commit config file. - -v --version Specify the Python version for the virtual environment. -l --list-versions List available Python versions and select interactively. + -v --version Specify the Python version for the virtual environment. END_OF_LINE ) @@ -56,14 +56,14 @@ while (("$#")); do INSTALL_HOOKS=1 shift ;; - -v | --version) - PYTHON_VERSION=$2 - shift 2 - ;; -l | --list-versions) LIST_VERSIONS=1 shift ;; + -v | --version) + PYTHON_VERSION=$2 + shift 2 + ;; -*) # unsupported flags echo "Error: Unsupported flag $1" >&2 exit 1 From 9497dc276e9f148211e32cc800342846cfa25e33 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 26 Jan 2024 10:08:25 -0500 Subject: [PATCH 183/326] Move misplaced exit If the Python version exists then we want the script to continue execution. --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 5282e49..5d15679 100755 --- a/setup-env +++ b/setup-env @@ -139,8 +139,8 @@ if [ -n "$PYTHON_VERSION" ]; then pyenv local "$PYTHON_VERSION" else echo Error: Python version "$PYTHON_VERSION" is not installed. + exit 1 fi - exit 1 fi # Remove any lingering local configuration. From e1d0f28ba1d96b2da0af91e277648ca88d0c3f35 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 26 Jan 2024 10:10:23 -0500 Subject: [PATCH 184/326] Remove premature pyenv local command pyenv local is run below. --- setup-env | 1 - 1 file changed, 1 deletion(-) diff --git a/setup-env b/setup-env index 5d15679..a78f940 100755 --- a/setup-env +++ b/setup-env @@ -136,7 +136,6 @@ fi if [ -n "$PYTHON_VERSION" ]; then if pyenv versions --bare --skip-aliases --skip-envs | grep --fixed-strings "$PYTHON_VERSION" > /dev/null; then echo Using Python version "$PYTHON_VERSION" - pyenv local "$PYTHON_VERSION" else echo Error: Python version "$PYTHON_VERSION" is not installed. exit 1 From 517b336210269d890c28e5c05d1c13e866fb7b15 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 26 Jan 2024 10:11:50 -0500 Subject: [PATCH 185/326] Include PYTHON_VERSION when running pyenv virtualenv If PYTHON_VERSION is an empty string then the system Python will be used. --- setup-env | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/setup-env b/setup-env index a78f940..0aa173d 100755 --- a/setup-env +++ b/setup-env @@ -37,8 +37,8 @@ FORCE=0 # Positional parameters PARAMS="" -# Flags to allow a user to specify which version of Python they want to use -PYTHON_VERSION="" +# A flag to allow a user to specify which version of Python they want +# to use. LIST_VERSIONS=0 # Parse command line arguments @@ -132,8 +132,9 @@ if [ $LIST_VERSIONS -ne 0 ]; then read -r -p "Enter the desired Python version: " PYTHON_VERSION fi -# Check if PYTHON_VERSION isn't empty. If it is installed, set it locally. -if [ -n "$PYTHON_VERSION" ]; then +# Check if PYTHON_VERSION is defined. If it is defined then check that +# it is a valid value. +if [ -n "${PYTHON_VERSION+x}" ]; then if pyenv versions --bare --skip-aliases --skip-envs | grep --fixed-strings "$PYTHON_VERSION" > /dev/null; then echo Using Python version "$PYTHON_VERSION" else @@ -159,7 +160,15 @@ END_OF_LINE fi # Create a new virtual environment for this project -if ! pyenv virtualenv "${env_name}"; then +# +# If $PYTHON_VERSION is undefined then the system Python will be used. +# +# We can't quote ${PYTHON_VERSION:=} below since if the variable is +# undefined then we want nothing to appear; this is the reason for the +# "shellcheck disable" line below. +# +# shellcheck disable=SC2086 +if ! pyenv virtualenv ${PYTHON_VERSION:=} "${env_name}"; then cat << END_OF_LINE An existing virtual environment named $env_name was found. Either delete this environment yourself or re-run with --force option to have it deleted. From 2e5794cb2aeaa76e8c94e3113f903954a9564eb4 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 30 Jan 2024 12:51:51 -0600 Subject: [PATCH 186/326] Add getopt variables and short flags --- setup-env | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/setup-env b/setup-env index 0aa173d..63cebd3 100755 --- a/setup-env +++ b/setup-env @@ -34,9 +34,21 @@ END_OF_LINE # Flag to force deletion and creation of virtual environment FORCE=0 +# Initialize the all other flags +INSTALL_HOOKS=0 +LIST_VERSIONS=0 +PYTHON_VERSION="" +VENV_NAME="" + # Positional parameters PARAMS="" +# Define short options for getopt +OPTS="fhilv:n:" + +# Parse options using BSD getopt +OPTIND=1 + # A flag to allow a user to specify which version of Python they want # to use. LIST_VERSIONS=0 From 8a5003195b6cc3a069fb0c4ec678f20372c4e10d Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 30 Jan 2024 12:54:12 -0600 Subject: [PATCH 187/326] Remove redundant flag initialization --- setup-env | 4 ---- 1 file changed, 4 deletions(-) diff --git a/setup-env b/setup-env index 63cebd3..da640f0 100755 --- a/setup-env +++ b/setup-env @@ -49,10 +49,6 @@ OPTS="fhilv:n:" # Parse options using BSD getopt OPTIND=1 -# A flag to allow a user to specify which version of Python they want -# to use. -LIST_VERSIONS=0 - # Parse command line arguments while (("$#")); do case "$1" in From 0df0e6aca8252e6356762b9284f3165ee39e2017 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 30 Jan 2024 13:59:04 -0600 Subject: [PATCH 188/326] Add getopt functionality and -n flag This commit makes a couple changes. The first change is adding the BSD getopt tool to simplify the parsing of flags and arguments. Second, we are adding the -n flag so the user can specify the name of the virtual environment if they choose. --- setup-env | 57 +++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 21 deletions(-) diff --git a/setup-env b/setup-env index da640f0..e097c82 100755 --- a/setup-env +++ b/setup-env @@ -49,40 +49,55 @@ OPTS="fhilv:n:" # Parse options using BSD getopt OPTIND=1 +# Display installed python versions +python_versions() { + pyenv versions --bare --skip-aliases --skip-envs +} + # Parse command line arguments -while (("$#")); do - case "$1" in - -f | --force) +while getopts :$OPTS opt; do + case $opt in + f) FORCE=1 - shift ;; - -h | --help) - echo "${USAGE}" + h) + echo "$USAGE" exit 0 ;; - -i | --install-hooks) + i) INSTALL_HOOKS=1 - shift ;; - -l | --list-versions) + l) LIST_VERSIONS=1 - shift ;; - -v | --version) - PYTHON_VERSION=$2 - shift 2 + n) + VENV_NAME="$OPTARG" + ;; + v) + PYTHON_VERSION="$OPTARG" + # Check if Python version is valid and installed + if ! python_versions | grep --fixed-strings "$PYTHON_VERSION" > /dev/null; then + echo "Error: Python version $PYTHON_VERSION is not installed. Versions available:" + python_versions + exit 1 + fi ;; - -*) # unsupported flags - echo "Error: Unsupported flag $1" >&2 + \:) + echo Error: Option -$OPTARG requires an argument. + echo "$USAGE" exit 1 ;; - *) # preserve positional arguments - PARAMS="$PARAMS $1" - shift + \?) + echo -e "Invalid option please look through usage: \n" + echo "$USAGE" + exit 1 ;; + esac done +shift $((OPTIND-1)) + # set positional arguments in their proper place eval set -- "$PARAMS" @@ -124,9 +139,9 @@ fi set +o nounset # Determine the virtual environment name -if [ "$1" ]; then +if [ -n "$VENV_NAME" ]; then # Use the user-provided environment name - env_name=$1 + env_name="$VENV_NAME" else # Set the environment name to the last part of the working directory. env_name=${PWD##*/} @@ -136,7 +151,7 @@ set -o nounset # List Python versions and select one interactively if [ $LIST_VERSIONS -ne 0 ]; then echo Available Python versions: - pyenv versions --bare --skip-aliases --skip-envs + python_versions read -r -p "Enter the desired Python version: " PYTHON_VERSION fi From 60cad12a2a64508c1e5c80092118747b64bb32e6 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 30 Jan 2024 14:16:52 -0600 Subject: [PATCH 189/326] Update the usage and force documentation This commit will update the usage documentation so now it will display all the new flags, how to use them, and what they do. Since the long option isn't available for the BSD version of getopt we have to use short options only. The force documentation has been updated to reflect this change. Also removed some redundant code that wasn't necessary. --- setup-env | 47 +++++++++++++++++++---------------------------- 1 file changed, 19 insertions(+), 28 deletions(-) diff --git a/setup-env b/setup-env index e097c82..4ed12d9 100755 --- a/setup-env +++ b/setup-env @@ -17,16 +17,18 @@ It does the following: - Configures git upstream remote "lineage" repositories. Usage: - setup-env [options] [virt_env_name] - setup-env (-h | --help) + setup-env [-n] [virt_env_name] + setup-env [-v] [python_version] + setup-env (-h) Options: - -f --force Delete virtual enviroment if it already exists. - -h --help Show this message. - -i --install-hooks Install hook environments for all environments in the - pre-commit config file. - -l --list-versions List available Python versions and select interactively. - -v --version Specify the Python version for the virtual environment. + -f Delete virtual enviroment if it already exists. + -h Show this message. + -i Install hook environments for all environments in the + pre-commit config file. + -l List available Python versions and select interactively. + -n Choose the name of the virtual environment. + -v Specify the Python version for the virtual environment. END_OF_LINE ) @@ -49,9 +51,9 @@ OPTS="fhilv:n:" # Parse options using BSD getopt OPTIND=1 -# Display installed python versions +# Display installed python versions python_versions() { - pyenv versions --bare --skip-aliases --skip-envs + pyenv versions --bare --skip-aliases --skip-envs } # Parse command line arguments @@ -82,21 +84,21 @@ while getopts :$OPTS opt; do exit 1 fi ;; - \:) - echo Error: Option -$OPTARG requires an argument. + :) + echo -e "Error: Option -$OPTARG requires an argument. \n" echo "$USAGE" exit 1 ;; \?) - echo -e "Invalid option please look through usage: \n" + echo -e "Invalid option please look through usage: \n" echo "$USAGE" exit 1 ;; - + esac done -shift $((OPTIND-1)) +shift $((OPTIND - 1)) # set positional arguments in their proper place eval set -- "$PARAMS" @@ -155,17 +157,6 @@ if [ $LIST_VERSIONS -ne 0 ]; then read -r -p "Enter the desired Python version: " PYTHON_VERSION fi -# Check if PYTHON_VERSION is defined. If it is defined then check that -# it is a valid value. -if [ -n "${PYTHON_VERSION+x}" ]; then - if pyenv versions --bare --skip-aliases --skip-envs | grep --fixed-strings "$PYTHON_VERSION" > /dev/null; then - echo Using Python version "$PYTHON_VERSION" - else - echo Error: Python version "$PYTHON_VERSION" is not installed. - exit 1 - fi -fi - # Remove any lingering local configuration. if [ $FORCE -ne 0 ]; then rm -f .python-version @@ -173,7 +164,7 @@ if [ $FORCE -ne 0 ]; then elif [[ -f .python-version ]]; then cat << 'END_OF_LINE' An existing .python-version file was found. Either remove this file yourself - or re-run with --force option to have it deleted along with the associated + or re-run with -f (force) option to have it deleted along with the associated virtual environment. rm .python-version @@ -194,7 +185,7 @@ fi if ! pyenv virtualenv ${PYTHON_VERSION:=} "${env_name}"; then cat << END_OF_LINE An existing virtual environment named $env_name was found. Either delete this - environment yourself or re-run with --force option to have it deleted. + environment yourself or re-run with -f (force) option to have it deleted. pyenv virtualenv-delete ${env_name} From b6ab6d82fb63bece7926b9a524bbb59a96c0cb06 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 6 Feb 2024 19:58:49 -0600 Subject: [PATCH 190/326] Update usage with long options --- setup-env | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/setup-env b/setup-env index 4ed12d9..898907b 100755 --- a/setup-env +++ b/setup-env @@ -9,6 +9,8 @@ USAGE=$( Configure a development environment for this repository. It does the following: + - Allows user to specify Python version. + - Allows user to choose name for their virtual environment. - Verifies pyenv and pyenv-virtualenv are installed. - Creates a Python virtual environment. - Configures the activation of the virtual enviroment for the repo directory. @@ -17,18 +19,18 @@ It does the following: - Configures git upstream remote "lineage" repositories. Usage: - setup-env [-n] [virt_env_name] - setup-env [-v] [python_version] - setup-env (-h) + setup-env (-n | --name) [virt_env_name] + setup-env (-v | --version) [python_version] + setup-env (-h | --help) Options: - -f Delete virtual enviroment if it already exists. - -h Show this message. - -i Install hook environments for all environments in the - pre-commit config file. - -l List available Python versions and select interactively. - -n Choose the name of the virtual environment. - -v Specify the Python version for the virtual environment. + -f | --force Delete virtual enviroment if it already exists. + -h | --help Show this message. + -i | --install-hooks Install hook environments for all environments in the + pre-commit config file. + -l | --list-versions List available Python versions and select interactively. + -n | --name Choose the name of the virtual environment. + -v | --version Specify the Python version for the virtual environment. END_OF_LINE ) From d362614a13731f3884856fa2ee37ade0089fda47 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 6 Feb 2024 20:28:05 -0600 Subject: [PATCH 191/326] Add gnu-getopt functionality and error handling This commit will make it so long options are supported. It also includes some improved error handling. It will display a message if a Mac OS user doesn't have gnu-getopt installed on their system and suggest steps to take to get it installed via brew. --- setup-env | 107 ++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 68 insertions(+), 39 deletions(-) diff --git a/setup-env b/setup-env index 898907b..f4cdf75 100755 --- a/setup-env +++ b/setup-env @@ -35,6 +35,11 @@ Options: END_OF_LINE ) +# Display installed python versions +python_versions() { + pyenv versions --bare --skip-aliases --skip-envs +} + # Flag to force deletion and creation of virtual environment FORCE=0 @@ -44,67 +49,82 @@ LIST_VERSIONS=0 PYTHON_VERSION="" VENV_NAME="" -# Positional parameters -PARAMS="" +# Define long options +LONGOPTS="force,help,install-hooks,list-versions,name:,version:" # Define short options for getopt -OPTS="fhilv:n:" +SHORTOPTS="fhiln:v:" -# Parse options using BSD getopt -OPTIND=1 +# Check if GNU getopt is available +if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then + cat << 'END_OF_LINE' -# Display installed python versions -python_versions() { - pyenv versions --bare --skip-aliases --skip-envs -} + Gnu-getopt is not detected and is a dependency to run this script. + On the Mac, we recommend installing brew, https://brew.sh/. Then installation + is as simple as `brew install gnu-getopt` and adding this to your + profile: + + export PATH="/usr/local/opt/gnu-getopt/bin:$PATH" + +END_OF_LINE + exit 1 +fi -# Parse command line arguments -while getopts :$OPTS opt; do - case $opt in - f) +# Use GNU getopt to parse options +if ! PARSED=$(getopt --options $SHORTOPTS --longoptions $LONGOPTS --name "$0" -- "$@"); then + echo "Error parsing options" + exit 2 +fi +eval set -- "$PARSED" + +while true; do + case "$1" in + -f | --force) FORCE=1 + shift ;; - h) + -h | --help) echo "$USAGE" exit 0 ;; - i) + -i | --install-hooks) INSTALL_HOOKS=1 + shift ;; - l) + -l | --list-versions) LIST_VERSIONS=1 + shift ;; - n) - VENV_NAME="$OPTARG" + -n | --name) + VENV_NAME="$2" + shift 2 ;; - v) - PYTHON_VERSION="$OPTARG" - # Check if Python version is valid and installed - if ! python_versions | grep --fixed-strings "$PYTHON_VERSION" > /dev/null; then - echo "Error: Python version $PYTHON_VERSION is not installed. Versions available:" - python_versions - exit 1 + -v | --version) + PYTHON_VERSION="$2" + shift 2 + # Check the Python versions being passed in. + if [ -n "${PYTHON_VERSION+x}" ]; then + if python_versions | grep -E "^${PYTHON_VERSION}$" > /dev/null; then + echo Using Python version "$PYTHON_VERSION" + else + echo Error: Python version "$PYTHON_VERSION" is not installed. + echo Installed Python versions are: + python_versions + exit 1 + fi fi ;; - :) - echo -e "Error: Option -$OPTARG requires an argument. \n" - echo "$USAGE" - exit 1 + --) + shift + break ;; - \?) - echo -e "Invalid option please look through usage: \n" - echo "$USAGE" - exit 1 + *) + echo "Programming error" + exit 3 ;; - esac done -shift $((OPTIND - 1)) - -# set positional arguments in their proper place -eval set -- "$PARAMS" - # Check to see if pyenv is installed if [ -z "$(command -v pyenv)" ] || { [ -z "$(command -v pyenv-virtualenv)" ] && [ ! -f "$(pyenv root)/plugins/pyenv-virtualenv/bin/pyenv-virtualenv" ]; }; then echo "pyenv and pyenv-virtualenv are required." @@ -157,6 +177,15 @@ if [ $LIST_VERSIONS -ne 0 ]; then echo Available Python versions: python_versions read -r -p "Enter the desired Python version: " PYTHON_VERSION + # Check the Python versions being passed in. + if [ -n "${PYTHON_VERSION+x}" ]; then + if python_versions | grep -E "^${PYTHON_VERSION}$" > /dev/null; then + echo Using Python version "$PYTHON_VERSION" + else + echo Error: Python version "$PYTHON_VERSION" is not installed. + exit 1 + fi + fi fi # Remove any lingering local configuration. From f92458454394db61e8080b72b42fd340b3b64aa1 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 6 Feb 2024 20:49:34 -0600 Subject: [PATCH 192/326] Add documentation in CONTRIBUTING.md for gnu-getopt --- CONTRIBUTING.md | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 27bb9e7..dbbe417 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,9 +46,12 @@ There are a few ways to do this, but we prefer to use create and manage a Python virtual environment specific to this project. -If you already have `pyenv` and `pyenv-virtualenv` configured you can -take advantage of the `setup-env` tool in this repo to automate the -entire environment configuration process. +The `setup-env` tool in this repository is our recommended method +for automating the entire environment configuration process. The +dependencies required to run this tool are +[`gnu-getopt`](https://manned.org/getopt.1), `pyenv`, and +`pyenv-virtualenv`. If these tools are already configured on +your system, you can simply run the following command. ```console ./setup-env @@ -57,13 +60,14 @@ entire environment configuration process. Otherwise, follow the steps below to manually configure your environment. -#### Installing and using `pyenv` and `pyenv-virtualenv` #### +#### Installing and using `gnu-getopt`, `pyenv`, and `pyenv-virtualenv` #### On the Mac, we recommend installing [brew](https://brew.sh/). Then -installation is as simple as `brew install pyenv pyenv-virtualenv` and +installation is as simple as `brew install gnu-getopt pyenv pyenv-virtualenv` and adding this to your profile: ```bash +export PATH="/usr/local/opt/gnu-getopt/bin:$PATH" export PYENV_ROOT="$HOME/.pyenv" export PATH="$PYENV_ROOT/bin:$PATH" eval "$(pyenv init --path)" @@ -78,6 +82,8 @@ install the necessary tools. Before running this ensure that you have installed the prerequisites for your platform according to the [`pyenv` wiki page](https://github.com/pyenv/pyenv/wiki/common-build-problems). +`Gnu-getopt` is generally included in the core utilities of most +Linux distributions. On WSL you should treat your platform as whatever Linux distribution you've chosen to install. From ba86ead6c4f757f03610c6065df46e5ba4ad4dbd Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 7 Feb 2024 09:43:30 -0600 Subject: [PATCH 193/326] Fix grammar and capitalization errors Co-authored-by: Shane Frasier --- CONTRIBUTING.md | 2 +- setup-env | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dbbe417..d96a0e3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -82,7 +82,7 @@ install the necessary tools. Before running this ensure that you have installed the prerequisites for your platform according to the [`pyenv` wiki page](https://github.com/pyenv/pyenv/wiki/common-build-problems). -`Gnu-getopt` is generally included in the core utilities of most +`gnu-getopt` is generally included in the core utilities of most Linux distributions. On WSL you should treat your platform as whatever Linux distribution diff --git a/setup-env b/setup-env index f4cdf75..1054a64 100755 --- a/setup-env +++ b/setup-env @@ -9,10 +9,10 @@ USAGE=$( Configure a development environment for this repository. It does the following: - - Allows user to specify Python version. - - Allows user to choose name for their virtual environment. + - Allows the user to specify the Python version to use for the virtual environment. + - Allows the user to specify a name for the virtual environment. - Verifies pyenv and pyenv-virtualenv are installed. - - Creates a Python virtual environment. + - Creates the Python virtual environment. - Configures the activation of the virtual enviroment for the repo directory. - Installs the requirements needed for development. - Installs git pre-commit hooks. @@ -29,7 +29,7 @@ Options: -i | --install-hooks Install hook environments for all environments in the pre-commit config file. -l | --list-versions List available Python versions and select interactively. - -n | --name Choose the name of the virtual environment. + -n | --name Specify the name of the virtual environment. -v | --version Specify the Python version for the virtual environment. END_OF_LINE @@ -195,7 +195,7 @@ if [ $FORCE -ne 0 ]; then elif [[ -f .python-version ]]; then cat << 'END_OF_LINE' An existing .python-version file was found. Either remove this file yourself - or re-run with -f (force) option to have it deleted along with the associated + or re-run with the --force option to have it deleted along with the associated virtual environment. rm .python-version @@ -216,7 +216,7 @@ fi if ! pyenv virtualenv ${PYTHON_VERSION:=} "${env_name}"; then cat << END_OF_LINE An existing virtual environment named $env_name was found. Either delete this - environment yourself or re-run with -f (force) option to have it deleted. + environment yourself or re-run with the --force option to have it deleted. pyenv virtualenv-delete ${env_name} From ba0fc19811d2b9bcb2cc043660f6dcd9d6d9c64b Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 7 Feb 2024 09:48:01 -0600 Subject: [PATCH 194/326] Combine PATH exports to single line Co-authored-by: Shane Frasier --- CONTRIBUTING.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d96a0e3..1e6cc74 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -67,9 +67,8 @@ installation is as simple as `brew install gnu-getopt pyenv pyenv-virtualenv` an adding this to your profile: ```bash -export PATH="/usr/local/opt/gnu-getopt/bin:$PATH" export PYENV_ROOT="$HOME/.pyenv" -export PATH="$PYENV_ROOT/bin:$PATH" +export PATH="$PYENV_ROOT/bin:/usr/local/opt/gnu-getopt/bin:$PATH" eval "$(pyenv init --path)" eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" From 1240bdd98fe3523dada222782a55df8c2b258e5c Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 7 Feb 2024 09:48:39 -0600 Subject: [PATCH 195/326] Improve usage instructions Co-authored-by: Shane Frasier --- setup-env | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup-env b/setup-env index 1054a64..a0230ed 100755 --- a/setup-env +++ b/setup-env @@ -19,8 +19,7 @@ It does the following: - Configures git upstream remote "lineage" repositories. Usage: - setup-env (-n | --name) [virt_env_name] - setup-env (-v | --version) [python_version] + setup-env (-n | --name) [virt_env_name] (-v | --version) [python_version] setup-env (-h | --help) Options: From 297b5bd850f716372eb385c25ef29936b4bd6d7c Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 7 Feb 2024 11:03:35 -0600 Subject: [PATCH 196/326] Add $(brew --prefix) to PATH for getopt Co-authored-by: dav3r --- CONTRIBUTING.md | 2 +- setup-env | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1e6cc74..899f4ff 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -68,7 +68,7 @@ adding this to your profile: ```bash export PYENV_ROOT="$HOME/.pyenv" -export PATH="$PYENV_ROOT/bin:/usr/local/opt/gnu-getopt/bin:$PATH" +export PATH="$PYENV_ROOT/bin:$(brew --prefix)/opt/gnu-getopt/bin:$PATH" eval "$(pyenv init --path)" eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" diff --git a/setup-env b/setup-env index a0230ed..3ddc9c9 100755 --- a/setup-env +++ b/setup-env @@ -63,7 +63,7 @@ if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then is as simple as `brew install gnu-getopt` and adding this to your profile: - export PATH="/usr/local/opt/gnu-getopt/bin:$PATH" + export PATH="$(brew --prefix)/opt/gnu-getopt/bin:$PATH" END_OF_LINE exit 1 From 7af70f514fba159ed6e7c8ae85af7a0e14a667a1 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 7 Feb 2024 11:04:27 -0600 Subject: [PATCH 197/326] Fix confusing wording Co-authored-by: dav3r --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 3ddc9c9..1e2fd79 100755 --- a/setup-env +++ b/setup-env @@ -42,7 +42,7 @@ python_versions() { # Flag to force deletion and creation of virtual environment FORCE=0 -# Initialize the all other flags +# Initialize the other flags INSTALL_HOOKS=0 LIST_VERSIONS=0 PYTHON_VERSION="" From e5a2d14964a65c0f5b74e53e5f7e606778b5f003 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 7 Feb 2024 11:06:50 -0600 Subject: [PATCH 198/326] Replace virt_env_name w/ virtual_env_name for clarity Co-authored-by: dav3r --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 1e2fd79..ac20852 100755 --- a/setup-env +++ b/setup-env @@ -19,7 +19,7 @@ It does the following: - Configures git upstream remote "lineage" repositories. Usage: - setup-env (-n | --name) [virt_env_name] (-v | --version) [python_version] + setup-env (-n | --name) [virtual_env_name] (-v | --version) [python_version] setup-env (-h | --help) Options: From 82c70e089bfd43c84273a83228da38030d340c14 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 13 Feb 2024 14:31:35 -0600 Subject: [PATCH 199/326] Differentiate between GNU getopt and gnu-getopt brew formula This commit will clearly differentiate between GNU getopt the tool and gnu-getopt the Homebrew formula. Also updating the URL so that getopt points at the source repository and pyenv and pyenv-virtualenv point at their respective repositories. Updated the ending punctuation. Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- CONTRIBUTING.md | 13 +++++++------ setup-env | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 899f4ff..afb1ae8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -49,9 +49,10 @@ project. The `setup-env` tool in this repository is our recommended method for automating the entire environment configuration process. The dependencies required to run this tool are -[`gnu-getopt`](https://manned.org/getopt.1), `pyenv`, and -`pyenv-virtualenv`. If these tools are already configured on -your system, you can simply run the following command. +[GNU `getopt`](https://github.com/util-linux/util-linux/blob/master/misc-utils/getopt.1.adoc), +[`pyenv`](https://github.com/pyenv/pyenv), and [`pyenv-virtualenv`](https://github.com/pyenv/pyenv-virtualenv). +If these tools are already configured on your system, you can simply run the +following command: ```console ./setup-env @@ -60,7 +61,7 @@ your system, you can simply run the following command. Otherwise, follow the steps below to manually configure your environment. -#### Installing and using `gnu-getopt`, `pyenv`, and `pyenv-virtualenv` #### +#### Installing and using GNU `getopt`, `pyenv`, and `pyenv-virtualenv` #### On the Mac, we recommend installing [brew](https://brew.sh/). Then installation is as simple as `brew install gnu-getopt pyenv pyenv-virtualenv` and @@ -81,8 +82,8 @@ install the necessary tools. Before running this ensure that you have installed the prerequisites for your platform according to the [`pyenv` wiki page](https://github.com/pyenv/pyenv/wiki/common-build-problems). -`gnu-getopt` is generally included in the core utilities of most -Linux distributions. +GNU `getopt` is included in most Linux distributions as part of the +[`util-linux`](https://github.com/util-linux/util-linux) package. On WSL you should treat your platform as whatever Linux distribution you've chosen to install. diff --git a/setup-env b/setup-env index ac20852..0e9598b 100755 --- a/setup-env +++ b/setup-env @@ -58,7 +58,7 @@ SHORTOPTS="fhiln:v:" if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then cat << 'END_OF_LINE' - Gnu-getopt is not detected and is a dependency to run this script. + GNU getopt is not detected and is a dependency to run this script. On the Mac, we recommend installing brew, https://brew.sh/. Then installation is as simple as `brew install gnu-getopt` and adding this to your profile: From 493a4a3e5dfe4c705998395797c75eda756c6dc4 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 13 Feb 2024 14:58:43 -0600 Subject: [PATCH 200/326] Add parenthesis over brew link Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 0e9598b..f0d4b36 100755 --- a/setup-env +++ b/setup-env @@ -59,7 +59,7 @@ if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then cat << 'END_OF_LINE' GNU getopt is not detected and is a dependency to run this script. - On the Mac, we recommend installing brew, https://brew.sh/. Then installation + On the Mac, we recommend installing brew (https://brew.sh/). Then installation is as simple as `brew install gnu-getopt` and adding this to your profile: From 3bc9aeb12fb7a270fa6bc54fcc4deceb897906bc Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 14 Feb 2024 07:37:24 -0600 Subject: [PATCH 201/326] Refactor flag names for clarity and accuracy This commit will remove the previous flags -v or --version for -p or --python-version. Also it will replace -n or --name for -v or --venv-name. The usage menu has been updated to reflect these changes as well. --- setup-env | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/setup-env b/setup-env index f0d4b36..b7b08e6 100755 --- a/setup-env +++ b/setup-env @@ -19,7 +19,7 @@ It does the following: - Configures git upstream remote "lineage" repositories. Usage: - setup-env (-n | --name) [virtual_env_name] (-v | --version) [python_version] + setup-env [--venv-name virtual_env_name] [--python-version python_version] setup-env (-h | --help) Options: @@ -28,8 +28,8 @@ Options: -i | --install-hooks Install hook environments for all environments in the pre-commit config file. -l | --list-versions List available Python versions and select interactively. - -n | --name Specify the name of the virtual environment. - -v | --version Specify the Python version for the virtual environment. + -v | --venv-name Specify the name of the virtual environment. + -p | --python-version Specify the Python version for the virtual environment. END_OF_LINE ) @@ -49,10 +49,10 @@ PYTHON_VERSION="" VENV_NAME="" # Define long options -LONGOPTS="force,help,install-hooks,list-versions,name:,version:" +LONGOPTS="force,help,install-hooks,list-versions,python-version:,venv-name:" # Define short options for getopt -SHORTOPTS="fhiln:v:" +SHORTOPTS="fhilp:v:" # Check if GNU getopt is available if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then @@ -94,11 +94,7 @@ while true; do LIST_VERSIONS=1 shift ;; - -n | --name) - VENV_NAME="$2" - shift 2 - ;; - -v | --version) + -p | --python-version) PYTHON_VERSION="$2" shift 2 # Check the Python versions being passed in. @@ -113,6 +109,10 @@ while true; do fi fi ;; + -v | --venv-name) + VENV_NAME="$2" + shift 2 + ;; --) shift break From 0be1f6376dc914be4eff9ce5fde7649d085a2582 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 14 Feb 2024 08:08:07 -0600 Subject: [PATCH 202/326] Elaborate on message when checking for GNU getopt --- setup-env | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/setup-env b/setup-env index b7b08e6..6649b4d 100755 --- a/setup-env +++ b/setup-env @@ -58,7 +58,13 @@ SHORTOPTS="fhilp:v:" if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then cat << 'END_OF_LINE' - GNU getopt is not detected and is a dependency to run this script. + Please note, this script requires GNU getopt due to its enhanced + functionality and compatibility with certain script features that + are not supported by the POSIX getopt found in some systems, particularly + those with a non-GNU version of getopt. This distinction is crucial + as a system might have a non-GNU version of getopt installed by default, + which could lead to unexpected behavior or script failure. + On the Mac, we recommend installing brew (https://brew.sh/). Then installation is as simple as `brew install gnu-getopt` and adding this to your profile: From c8f0b1b996958a5fa965343fa7e6295b7160cbf4 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 14 Feb 2024 08:10:21 -0600 Subject: [PATCH 203/326] Remove unnecessary nounset flipping logic --- setup-env | 2 -- 1 file changed, 2 deletions(-) diff --git a/setup-env b/setup-env index 6649b4d..05b010b 100755 --- a/setup-env +++ b/setup-env @@ -166,7 +166,6 @@ END_OF_LINE exit 1 fi -set +o nounset # Determine the virtual environment name if [ -n "$VENV_NAME" ]; then # Use the user-provided environment name @@ -175,7 +174,6 @@ else # Set the environment name to the last part of the working directory. env_name=${PWD##*/} fi -set -o nounset # List Python versions and select one interactively if [ $LIST_VERSIONS -ne 0 ]; then From 495862aeaf3430b0a69f92cf54b6415818139db2 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 14 Feb 2024 08:20:56 -0600 Subject: [PATCH 204/326] Separate pyenv PATH from GNU getopt PATH --- CONTRIBUTING.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index afb1ae8..3a5e087 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -69,7 +69,8 @@ adding this to your profile: ```bash export PYENV_ROOT="$HOME/.pyenv" -export PATH="$PYENV_ROOT/bin:$(brew --prefix)/opt/gnu-getopt/bin:$PATH" +export PATH="$PYENV_ROOT/bin:$PATH" +export PATH="$(brew --prefix)/opt/gnu-getopt/bin:$PATH" eval "$(pyenv init --path)" eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" From 33582a11f73d4e5c3dc588a040e59941c381d09a Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 14 Feb 2024 12:59:30 -0600 Subject: [PATCH 205/326] Add checks for semantic python versions --- setup-env | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/setup-env b/setup-env index 05b010b..44368b3 100755 --- a/setup-env +++ b/setup-env @@ -39,6 +39,14 @@ python_versions() { pyenv versions --bare --skip-aliases --skip-envs } +check_semantic_version() { + local version=$1 + local regex="^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?\$" + + # Use Perl for regex matching and output true or false + echo "$version" | perl -ne "exit(!/$regex/)" +} + # Flag to force deletion and creation of virtual environment FORCE=0 @@ -103,16 +111,18 @@ while true; do -p | --python-version) PYTHON_VERSION="$2" shift 2 - # Check the Python versions being passed in. - if [ -n "${PYTHON_VERSION+x}" ]; then - if python_versions | grep -E "^${PYTHON_VERSION}$" > /dev/null; then - echo Using Python version "$PYTHON_VERSION" - else - echo Error: Python version "$PYTHON_VERSION" is not installed. - echo Installed Python versions are: - python_versions - exit 1 - fi + # Validate the semantic version format + if ! check_semantic_version "$PYTHON_VERSION"; then + echo "Error: The specified Python version $PYTHON_VERSION does not follow the semantic versioning standard." + echo "Example of a valid version: 3.8.1, 3.8.1-alpha.1, or 3.8.1+20130313144700" + exit 1 + elif ! python_versions | grep -E "^${PYTHON_VERSION}$" > /dev/null; then + echo "Error: Python version $PYTHON_VERSION is not installed." + echo "Installed Python versions are:" + python_versions + exit 1 + else + echo "Using Python version $PYTHON_VERSION" fi ;; -v | --venv-name) @@ -181,14 +191,7 @@ if [ $LIST_VERSIONS -ne 0 ]; then python_versions read -r -p "Enter the desired Python version: " PYTHON_VERSION # Check the Python versions being passed in. - if [ -n "${PYTHON_VERSION+x}" ]; then - if python_versions | grep -E "^${PYTHON_VERSION}$" > /dev/null; then - echo Using Python version "$PYTHON_VERSION" - else - echo Error: Python version "$PYTHON_VERSION" is not installed. - exit 1 - fi - fi + check_semantic_version "$PYTHON_VERSION" fi # Remove any lingering local configuration. From 94381940a9d28f87da2b85c5e1647a5a80d4a18d Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 14 Feb 2024 14:29:13 -0600 Subject: [PATCH 206/326] Refactor code for the semantic check This commit will make a few changes. The orginal version of the semantic checking function was a bit more difficult to read. It is now somewhat easier to follow how the regex is structured. Also the function has been renamed to check_python_version since it has 2 functions, making sure that the version is semantically correct and the second is to make sure that it is installed on the user's machine. This makes it easier to follow the logic for the flags, -p or --python-version and -l or --list-versions --- setup-env | 54 ++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 36 insertions(+), 18 deletions(-) diff --git a/setup-env b/setup-env index 44368b3..11ec170 100755 --- a/setup-env +++ b/setup-env @@ -39,12 +39,41 @@ python_versions() { pyenv versions --bare --skip-aliases --skip-envs } -check_semantic_version() { +check_python_version() { local version=$1 - local regex="^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?\$" - # Use Perl for regex matching and output true or false - echo "$version" | perl -ne "exit(!/$regex/)" + # Break down the regex into readable parts major.minor.patch + local major="0|[1-9]\\d*" + local minor="0|[1-9]\\d*" + local patch="0|[1-9]\\d*" + + # Splitting the prerelease part for readability + # Start of prerelease + local prerelease="(?:-" + # Numeric or alphanumeric identifiers + local prerelease+="(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)" + # Additional dot-separated identifiers + local prerelease+="(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*" + # End of prerelease, making it optional + local prerelease+=")?" + # Optional build metadata + local build="(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?" + + # Final regex composed of parts + local regex="^($major)\\.($minor)\\.($patch)$prerelease$build\$" + + if ! echo "$version" | perl -ne "exit(!/$regex/)"; then + echo "Error: The specified Python version $version does not follow the semantic versioning standard." + echo "Example of a valid version: 3.8.1, 3.8.1-alpha.1, or 3.8.1+20130313144700" + exit 1 + elif ! python_versions | grep -E "^${version}$" > /dev/null; then + echo "Error: Python version $version is not installed." + echo "Installed Python versions are:" + python_versions + exit 1 + else + echo "Using Python version $version" + fi } # Flag to force deletion and creation of virtual environment @@ -111,19 +140,8 @@ while true; do -p | --python-version) PYTHON_VERSION="$2" shift 2 - # Validate the semantic version format - if ! check_semantic_version "$PYTHON_VERSION"; then - echo "Error: The specified Python version $PYTHON_VERSION does not follow the semantic versioning standard." - echo "Example of a valid version: 3.8.1, 3.8.1-alpha.1, or 3.8.1+20130313144700" - exit 1 - elif ! python_versions | grep -E "^${PYTHON_VERSION}$" > /dev/null; then - echo "Error: Python version $PYTHON_VERSION is not installed." - echo "Installed Python versions are:" - python_versions - exit 1 - else - echo "Using Python version $PYTHON_VERSION" - fi + # Check the Python version being passed in. + check_python_version "$PYTHON_VERSION" ;; -v | --venv-name) VENV_NAME="$2" @@ -191,7 +209,7 @@ if [ $LIST_VERSIONS -ne 0 ]; then python_versions read -r -p "Enter the desired Python version: " PYTHON_VERSION # Check the Python versions being passed in. - check_semantic_version "$PYTHON_VERSION" + check_python_version "$PYTHON_VERSION" fi # Remove any lingering local configuration. From 4752b37ec887a65e967f01a1f9b7633e02a02af8 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 21 Feb 2024 10:57:38 -0600 Subject: [PATCH 207/326] Improve verbiage in comments Co-authored-by: Shane Frasier --- CONTRIBUTING.md | 4 ++-- setup-env | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3a5e087..9ff54e4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,8 +46,8 @@ There are a few ways to do this, but we prefer to use create and manage a Python virtual environment specific to this project. -The `setup-env` tool in this repository is our recommended method -for automating the entire environment configuration process. The +We recommend using the `setup-env` tool located in this repository, +as it automates the entire environment configuration process. The dependencies required to run this tool are [GNU `getopt`](https://github.com/util-linux/util-linux/blob/master/misc-utils/getopt.1.adoc), [`pyenv`](https://github.com/pyenv/pyenv), and [`pyenv-virtualenv`](https://github.com/pyenv/pyenv-virtualenv). diff --git a/setup-env b/setup-env index 05b010b..6d9fec0 100755 --- a/setup-env +++ b/setup-env @@ -27,7 +27,7 @@ Options: -h | --help Show this message. -i | --install-hooks Install hook environments for all environments in the pre-commit config file. - -l | --list-versions List available Python versions and select interactively. + -l | --list-versions List available Python versions and select one interactively. -v | --venv-name Specify the name of the virtual environment. -p | --python-version Specify the Python version for the virtual environment. @@ -63,7 +63,7 @@ if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then are not supported by the POSIX getopt found in some systems, particularly those with a non-GNU version of getopt. This distinction is crucial as a system might have a non-GNU version of getopt installed by default, - which could lead to unexpected behavior or script failure. + which could lead to unexpected behavior. On the Mac, we recommend installing brew (https://brew.sh/). Then installation is as simple as `brew install gnu-getopt` and adding this to your From 2e389979b0b0b34a6f32443d88f8ed0ba8c647cd Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 21 Feb 2024 11:22:53 -0600 Subject: [PATCH 208/326] Clarify between pyenv and GNU getopt setup Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- CONTRIBUTING.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9ff54e4..a844f86 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -68,9 +68,12 @@ installation is as simple as `brew install gnu-getopt pyenv pyenv-virtualenv` an adding this to your profile: ```bash +# Enable GNU getopt since it is keg-only +export PATH="$(brew --prefix)/opt/gnu-getopt/bin:$PATH" + +# Setup pyenv export PYENV_ROOT="$HOME/.pyenv" export PATH="$PYENV_ROOT/bin:$PATH" -export PATH="$(brew --prefix)/opt/gnu-getopt/bin:$PATH" eval "$(pyenv init --path)" eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" From f8824c8aa157cbee0ec95367b1754eeac67934c9 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 21 Feb 2024 12:14:23 -0600 Subject: [PATCH 209/326] Improve comment on conditional check for regex --- setup-env | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 6d9fec0..6d16958 100755 --- a/setup-env +++ b/setup-env @@ -54,7 +54,9 @@ LONGOPTS="force,help,install-hooks,list-versions,python-version:,venv-name:" # Define short options for getopt SHORTOPTS="fhilp:v:" -# Check if GNU getopt is available +# Check for GNU getopt by matching a specific pattern ("getopt from util-linux") +# in its version output. This approach presumes the output format remains stable. +# Be aware that format changes could invalidate this check. if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then cat << 'END_OF_LINE' From 88724e70167a5864495eccff5532f4c59fbefc1c Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 21 Feb 2024 12:25:49 -0600 Subject: [PATCH 210/326] Add comment explaining that GNU getopt is keg-only --- setup-env | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup-env b/setup-env index 6d16958..84a8305 100755 --- a/setup-env +++ b/setup-env @@ -73,6 +73,8 @@ if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then export PATH="$(brew --prefix)/opt/gnu-getopt/bin:$PATH" + This will enable GNU getopt since it is keg-only. + END_OF_LINE exit 1 fi From c1870be7a16f5e88f6e6540c290f674da4861e07 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 21 Feb 2024 16:28:54 -0600 Subject: [PATCH 211/326] Improve comments to better describe `keg-only` terminology Co-authored-by: Shane Frasier --- CONTRIBUTING.md | 3 ++- setup-env | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a844f86..7bb8881 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -68,7 +68,8 @@ installation is as simple as `brew install gnu-getopt pyenv pyenv-virtualenv` an adding this to your profile: ```bash -# Enable GNU getopt since it is keg-only +# GNU getopt must be explicitly added to the path since it is +# keg-only export PATH="$(brew --prefix)/opt/gnu-getopt/bin:$PATH" # Setup pyenv diff --git a/setup-env b/setup-env index 84a8305..3fa8c47 100755 --- a/setup-env +++ b/setup-env @@ -73,7 +73,8 @@ if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then export PATH="$(brew --prefix)/opt/gnu-getopt/bin:$PATH" - This will enable GNU getopt since it is keg-only. + GNU getopt must be explicitly added to the PATH since it + is keg-only. END_OF_LINE exit 1 From a3f69cd24e58077ee6f084bce900ed176c88cff7 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Mon, 26 Feb 2024 08:04:19 -0600 Subject: [PATCH 212/326] Change "'setup-env' tool" to "'setup-env' script" Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7bb8881..fca0cf7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,7 +46,7 @@ There are a few ways to do this, but we prefer to use create and manage a Python virtual environment specific to this project. -We recommend using the `setup-env` tool located in this repository, +We recommend using the `setup-env` script located in this repository, as it automates the entire environment configuration process. The dependencies required to run this tool are [GNU `getopt`](https://github.com/util-linux/util-linux/blob/master/misc-utils/getopt.1.adoc), From 8ff51796ec530e68d35362fab761608e676dcd3a Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Mon, 26 Feb 2024 08:07:16 -0600 Subject: [PATCH 213/326] Remove build-in error exit for generic error exit Exit code 2 usually relates to build-ins which does not fit what is failing here. Not sure what alternative would be best so we can just use the generic failure code Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 3fa8c47..f7ce620 100755 --- a/setup-env +++ b/setup-env @@ -83,7 +83,7 @@ fi # Use GNU getopt to parse options if ! PARSED=$(getopt --options $SHORTOPTS --longoptions $LONGOPTS --name "$0" -- "$@"); then echo "Error parsing options" - exit 2 + exit 1 fi eval set -- "$PARSED" From 1c21e2b9839980c75410584c69d0ac0a0651f097 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Mon, 26 Feb 2024 08:08:39 -0600 Subject: [PATCH 214/326] Change verbiage from 'tool' to 'script' for clarity Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fca0cf7..cdab589 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -48,7 +48,7 @@ project. We recommend using the `setup-env` script located in this repository, as it automates the entire environment configuration process. The -dependencies required to run this tool are +dependencies required to run this script are [GNU `getopt`](https://github.com/util-linux/util-linux/blob/master/misc-utils/getopt.1.adoc), [`pyenv`](https://github.com/pyenv/pyenv), and [`pyenv-virtualenv`](https://github.com/pyenv/pyenv-virtualenv). If these tools are already configured on your system, you can simply run the From 3acc8d6a17360221d6c59c306db1a14a98b48b8b Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Mon, 26 Feb 2024 10:04:42 -0600 Subject: [PATCH 215/326] Check for pyenv earlier in the script Since the new function, python_versions expects pyenv to be installed it makes sense to have it checked earlier in the script. --- setup-env | 72 +++++++++++++++++++++++++++---------------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/setup-env b/setup-env index f7ce620..3898ca8 100755 --- a/setup-env +++ b/setup-env @@ -80,6 +80,42 @@ END_OF_LINE exit 1 fi +# Check to see if pyenv is installed +if [ -z "$(command -v pyenv)" ] || { [ -z "$(command -v pyenv-virtualenv)" ] && [ ! -f "$(pyenv root)/plugins/pyenv-virtualenv/bin/pyenv-virtualenv" ]; }; then + echo "pyenv and pyenv-virtualenv are required." + if [[ "$OSTYPE" == "darwin"* ]]; then + cat << 'END_OF_LINE' + + On the Mac, we recommend installing brew, https://brew.sh/. Then installation + is as simple as `brew install pyenv pyenv-virtualenv` and adding this to your + profile: + + eval "$(pyenv init -)" + eval "$(pyenv virtualenv-init -)" + +END_OF_LINE + + fi + cat << 'END_OF_LINE' + For Linux, Windows Subsystem for Linux (WSL), or on the Mac (if you don't want + to use "brew") you can use https://github.com/pyenv/pyenv-installer to install + the necessary tools. Before running this ensure that you have installed the + prerequisites for your platform according to the pyenv wiki page, + https://github.com/pyenv/pyenv/wiki/common-build-problems. + + On WSL you should treat your platform as whatever Linux distribution you've + chosen to install. + + Once you have installed "pyenv" you will need to add the following lines to + your ".bashrc": + + export PATH="$PATH:$HOME/.pyenv/bin" + eval "$(pyenv init -)" + eval "$(pyenv virtualenv-init -)" +END_OF_LINE + exit 1 +fi + # Use GNU getopt to parse options if ! PARSED=$(getopt --options $SHORTOPTS --longoptions $LONGOPTS --name "$0" -- "$@"); then echo "Error parsing options" @@ -135,42 +171,6 @@ while true; do esac done -# Check to see if pyenv is installed -if [ -z "$(command -v pyenv)" ] || { [ -z "$(command -v pyenv-virtualenv)" ] && [ ! -f "$(pyenv root)/plugins/pyenv-virtualenv/bin/pyenv-virtualenv" ]; }; then - echo "pyenv and pyenv-virtualenv are required." - if [[ "$OSTYPE" == "darwin"* ]]; then - cat << 'END_OF_LINE' - - On the Mac, we recommend installing brew, https://brew.sh/. Then installation - is as simple as `brew install pyenv pyenv-virtualenv` and adding this to your - profile: - - eval "$(pyenv init -)" - eval "$(pyenv virtualenv-init -)" - -END_OF_LINE - - fi - cat << 'END_OF_LINE' - For Linux, Windows Subsystem for Linux (WSL), or on the Mac (if you don't want - to use "brew") you can use https://github.com/pyenv/pyenv-installer to install - the necessary tools. Before running this ensure that you have installed the - prerequisites for your platform according to the pyenv wiki page, - https://github.com/pyenv/pyenv/wiki/common-build-problems. - - On WSL you should treat your platform as whatever Linux distribution you've - chosen to install. - - Once you have installed "pyenv" you will need to add the following lines to - your ".bashrc": - - export PATH="$PATH:$HOME/.pyenv/bin" - eval "$(pyenv init -)" - eval "$(pyenv virtualenv-init -)" -END_OF_LINE - exit 1 -fi - # Determine the virtual environment name if [ -n "$VENV_NAME" ]; then # Use the user-provided environment name From b377ce71fa3dd5f56697d415c5230f3fe18e8090 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Mon, 26 Feb 2024 11:02:39 -0600 Subject: [PATCH 216/326] Explain -r and -p in Python version prompt --- setup-env | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 3898ca8..5c2b580 100755 --- a/setup-env +++ b/setup-env @@ -180,10 +180,12 @@ else env_name=${PWD##*/} fi -# List Python versions and select one interactively +# List Python versions and select one interactively. if [ $LIST_VERSIONS -ne 0 ]; then echo Available Python versions: python_versions + # Read the user's desired Python version. + # -r: treat backslashes as literal, -p: display prompt before input. read -r -p "Enter the desired Python version: " PYTHON_VERSION # Check the Python versions being passed in. if [ -n "${PYTHON_VERSION+x}" ]; then From 74838a256e134e00208a23bb6dded7012cfb8534 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Mon, 26 Feb 2024 11:58:44 -0600 Subject: [PATCH 217/326] Refine exit code to 64 with gnu-getopt note - Set exit code to 64 per sysexits.h for clarity. - Added comment on gnu-getopt handling. Aligns with Unix standards and enhances readability. --- setup-env | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 5c2b580..549c079 100755 --- a/setup-env +++ b/setup-env @@ -165,8 +165,9 @@ while true; do break ;; *) + # Unreachable due to gnu-getopt handling all options echo "Programming error" - exit 3 + exit 64 ;; esac done From 487126e34bd85e231b7021b70d225154694ad24d Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 28 Feb 2024 06:32:35 -0600 Subject: [PATCH 218/326] Rename gnu-getopt tool to GNU getopt formula Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 549c079..d2db6e2 100755 --- a/setup-env +++ b/setup-env @@ -165,7 +165,7 @@ while true; do break ;; *) - # Unreachable due to gnu-getopt handling all options + # Unreachable due to GNU getopt handling all options echo "Programming error" exit 64 ;; From 6c82a8d1bb86a90f0e903dd2d90efdb6198fba33 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 28 Feb 2024 07:49:16 -0600 Subject: [PATCH 219/326] Fix whitespace for usage menu Co-authored-by: Shane Frasier --- setup-env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup-env b/setup-env index d2db6e2..c93eb25 100755 --- a/setup-env +++ b/setup-env @@ -28,8 +28,8 @@ Options: -i | --install-hooks Install hook environments for all environments in the pre-commit config file. -l | --list-versions List available Python versions and select one interactively. - -v | --venv-name Specify the name of the virtual environment. - -p | --python-version Specify the Python version for the virtual environment. + -v | --venv-name Specify the name of the virtual environment. + -p | --python-version Specify the Python version for the virtual environment. END_OF_LINE ) From 324f6d426a149cb2c7bfc30ffcf7c1810b8ae1fa Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 28 Feb 2024 11:26:24 -0600 Subject: [PATCH 220/326] Add link to brew terminology Co-authored-by: dav3r --- CONTRIBUTING.md | 2 +- setup-env | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cdab589..bc32a55 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -69,7 +69,7 @@ adding this to your profile: ```bash # GNU getopt must be explicitly added to the path since it is -# keg-only +# keg-only (https://docs.brew.sh/FAQ#what-does-keg-only-mean) export PATH="$(brew --prefix)/opt/gnu-getopt/bin:$PATH" # Setup pyenv diff --git a/setup-env b/setup-env index c93eb25..bb689f3 100755 --- a/setup-env +++ b/setup-env @@ -74,7 +74,7 @@ if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then export PATH="$(brew --prefix)/opt/gnu-getopt/bin:$PATH" GNU getopt must be explicitly added to the PATH since it - is keg-only. + is keg-only (https://docs.brew.sh/FAQ#what-does-keg-only-mean). END_OF_LINE exit 1 From a26d0e36f0c33a7f472b91d66697e4cecb8f9557 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 28 Feb 2024 15:52:20 -0600 Subject: [PATCH 221/326] Rephrase comment to improve clarity Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index bb689f3..3fddb36 100755 --- a/setup-env +++ b/setup-env @@ -16,7 +16,7 @@ It does the following: - Configures the activation of the virtual enviroment for the repo directory. - Installs the requirements needed for development. - Installs git pre-commit hooks. - - Configures git upstream remote "lineage" repositories. + - Configures git remotes for upstream "lineage" repositories. Usage: setup-env [--venv-name virtual_env_name] [--python-version python_version] From 05108705fa4120db51f261ad59347bb01134eafc Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 28 Feb 2024 15:58:43 -0600 Subject: [PATCH 222/326] Improve comment for clarity Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 3fddb36..c8680ea 100755 --- a/setup-env +++ b/setup-env @@ -34,7 +34,7 @@ Options: END_OF_LINE ) -# Display installed python versions +# Display pyenv's installed Python versions python_versions() { pyenv versions --bare --skip-aliases --skip-envs } From 01abde6f580cf549d77c81af9378d4a18812fa8f Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 28 Feb 2024 16:03:47 -0600 Subject: [PATCH 223/326] Improve verbiage in comment --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index c8680ea..534d47f 100755 --- a/setup-env +++ b/setup-env @@ -217,7 +217,7 @@ fi # Create a new virtual environment for this project # -# If $PYTHON_VERSION is undefined then the system Python will be used. +# If $PYTHON_VERSION is undefined then the global version of Python will be used. # # We can't quote ${PYTHON_VERSION:=} below since if the variable is # undefined then we want nothing to appear; this is the reason for the From 0989d17f49a089e03e23b72ab638595d538add3f Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 28 Feb 2024 16:09:15 -0600 Subject: [PATCH 224/326] Change comments for macOS and venv_name --- CONTRIBUTING.md | 4 ++-- setup-env | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bc32a55..617db85 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -63,7 +63,7 @@ environment. #### Installing and using GNU `getopt`, `pyenv`, and `pyenv-virtualenv` #### -On the Mac, we recommend installing [brew](https://brew.sh/). Then +On macOS, we recommend installing [brew](https://brew.sh/). Then installation is as simple as `brew install gnu-getopt pyenv pyenv-virtualenv` and adding this to your profile: @@ -80,7 +80,7 @@ eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" ``` -For Linux, Windows Subsystem for Linux (WSL), or on the Mac (if you +For Linux, Windows Subsystem for Linux (WSL), or on macOS (if you don't want to use `brew`) you can use [pyenv/pyenv-installer](https://github.com/pyenv/pyenv-installer) to install the necessary tools. Before running this ensure that you have diff --git a/setup-env b/setup-env index 534d47f..d2cd111 100755 --- a/setup-env +++ b/setup-env @@ -19,7 +19,7 @@ It does the following: - Configures git remotes for upstream "lineage" repositories. Usage: - setup-env [--venv-name virtual_env_name] [--python-version python_version] + setup-env [--venv-name venv_name] [--python-version python_version] setup-env (-h | --help) Options: @@ -67,7 +67,7 @@ if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then as a system might have a non-GNU version of getopt installed by default, which could lead to unexpected behavior. - On the Mac, we recommend installing brew (https://brew.sh/). Then installation + On macOS, we recommend installing brew (https://brew.sh/). Then installation is as simple as `brew install gnu-getopt` and adding this to your profile: @@ -86,7 +86,7 @@ if [ -z "$(command -v pyenv)" ] || { [ -z "$(command -v pyenv-virtualenv)" ] && if [[ "$OSTYPE" == "darwin"* ]]; then cat << 'END_OF_LINE' - On the Mac, we recommend installing brew, https://brew.sh/. Then installation + On macOS, we recommend installing brew, https://brew.sh/. Then installation is as simple as `brew install pyenv pyenv-virtualenv` and adding this to your profile: @@ -97,7 +97,7 @@ END_OF_LINE fi cat << 'END_OF_LINE' - For Linux, Windows Subsystem for Linux (WSL), or on the Mac (if you don't want + For Linux, Windows Subsystem for Linux (WSL), or on mac OS (if you don't want to use "brew") you can use https://github.com/pyenv/pyenv-installer to install the necessary tools. Before running this ensure that you have installed the prerequisites for your platform according to the pyenv wiki page, From a9c6ed88c3c993db890a09ece5433b33e5d5d206 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Thu, 29 Feb 2024 09:19:01 -0600 Subject: [PATCH 225/326] Improve comments for clarity Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- CONTRIBUTING.md | 2 +- setup-env | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 617db85..f9fa53a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -80,7 +80,7 @@ eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" ``` -For Linux, Windows Subsystem for Linux (WSL), or on macOS (if you +For Linux, Windows Subsystem for Linux (WSL), or macOS (if you don't want to use `brew`) you can use [pyenv/pyenv-installer](https://github.com/pyenv/pyenv-installer) to install the necessary tools. Before running this ensure that you have diff --git a/setup-env b/setup-env index d2cd111..3a22d43 100755 --- a/setup-env +++ b/setup-env @@ -97,7 +97,7 @@ END_OF_LINE fi cat << 'END_OF_LINE' - For Linux, Windows Subsystem for Linux (WSL), or on mac OS (if you don't want + For Linux, Windows Subsystem for Linux (WSL), or macOS (if you don't want to use "brew") you can use https://github.com/pyenv/pyenv-installer to install the necessary tools. Before running this ensure that you have installed the prerequisites for your platform according to the pyenv wiki page, @@ -217,7 +217,7 @@ fi # Create a new virtual environment for this project # -# If $PYTHON_VERSION is undefined then the global version of Python will be used. +# If $PYTHON_VERSION is undefined then the current pyenv Python version will be used. # # We can't quote ${PYTHON_VERSION:=} below since if the variable is # undefined then we want nothing to appear; this is the reason for the From b9c729f6159defe55490d2198390ba2ceab5924f Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 4 Jan 2024 15:03:15 -0500 Subject: [PATCH 226/326] Update pre-commit hook versions This is done automatically with the `pre-commit autoupdate` command. The pre-commit/mirrors-prettier was manually held back because the latest tags are for alpha releases of the next major version. --- .pre-commit-config.yaml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 34f0253..e7f5d0a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ default_language_version: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: check-case-conflict - id: check-executables-have-shebangs @@ -31,7 +31,7 @@ repos: # Text file hooks - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.36.0 + rev: v0.39.0 hooks: - id: markdownlint args: @@ -41,7 +41,7 @@ repos: hooks: - id: prettier - repo: https://github.com/adrienverge/yamllint - rev: v1.32.0 + rev: v1.35.1 hooks: - id: yamllint args: @@ -49,14 +49,14 @@ repos: # GitHub Actions hooks - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.26.3 + rev: 0.28.0 hooks: - id: check-github-actions - id: check-github-workflows # pre-commit hooks - repo: https://github.com/pre-commit/pre-commit - rev: v3.4.0 + rev: v3.6.2 hooks: - id: validate_manifest @@ -107,44 +107,44 @@ repos: # Python hooks - repo: https://github.com/PyCQA/bandit - rev: 1.7.5 + rev: 1.7.7 hooks: - id: bandit args: - --config=.bandit.yml - repo: https://github.com/psf/black-pre-commit-mirror - rev: 23.9.1 + rev: 24.2.0 hooks: - id: black - repo: https://github.com/PyCQA/flake8 - rev: 6.1.0 + rev: 7.0.0 hooks: - id: flake8 additional_dependencies: - flake8-docstrings - repo: https://github.com/PyCQA/isort - rev: 5.12.0 + rev: 5.13.2 hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.5.1 + rev: v1.8.0 hooks: - id: mypy - repo: https://github.com/asottile/pyupgrade - rev: v3.10.1 + rev: v3.15.1 hooks: - id: pyupgrade # Ansible hooks - repo: https://github.com/ansible/ansible-lint - rev: v6.19.0 + rev: v24.2.0 hooks: - id: ansible-lint # files: molecule/default/playbook.yml # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.83.2 + rev: v1.88.0 hooks: - id: terraform_fmt - id: terraform_validate From 4c933958bca75c291bcb3b28fa1ba821d1d1f6f5 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 4 Jan 2024 15:08:30 -0500 Subject: [PATCH 227/326] Manually update the prettier hook The hook is manually updated to the last v3 tag available from the pre-commit/mirrors-prettier repository. We then add the latest release of v3 available from NPM as an additional depdency. --- .pre-commit-config.yaml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e7f5d0a..8e3941d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,9 +37,16 @@ repos: args: - --config=.mdl_config.yaml - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.0.3 + # This is the last version of v3 available from the mirror. We should hold + # here until v4, which is currently in alpha, is more stable. + rev: v3.1.0 hooks: - id: prettier + # This is the latest version of v3 available from NPM. The pre-commit + # mirror does not pull tags for old major versions once a new major + # version tag is published. + additional_dependencies: + - prettier@3.2.5 - repo: https://github.com/adrienverge/yamllint rev: v1.35.1 hooks: From 7169dcf65670f1d42b9498b68624a1638de08531 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Sat, 11 Nov 2023 16:20:14 -0500 Subject: [PATCH 228/326] Use Python and Go versions provided by cisagov/setup-env-github-action --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4911745..de6937d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -59,7 +59,7 @@ jobs: - id: setup-python uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: ${{ steps.setup-env.outputs.python-version }} # We need the Go version and Go cache location for the actions/cache step, # so the Go installation must happen before that. - id: setup-go @@ -68,7 +68,7 @@ jobs: # There is no expectation for actual Go code so we disable caching as # it relies on the existence of a go.sum file. cache: false - go-version: "1.20" + go-version: ${{ steps.setup-env.outputs.go-version }} - id: go-cache name: Lookup Go cache directory run: | From 035cf86cf76f40ed8b4bf67ff1b14c420701e145 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 27 Feb 2024 17:17:39 -0500 Subject: [PATCH 229/326] Switch pre-commit hooks for running shellcheck This hook bundles the binaries for shellcheck with a Python package which removes the need to ensure the tool is installed for the hook to function. It also ties the version of shellcheck used to the hook which will help guarantee consistency. --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1d12a0d..2c5b3c8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -115,10 +115,10 @@ repos: - --case-indent # Redirect operators are followed by a space - --space-redirects - - repo: https://github.com/detailyang/pre-commit-shell - rev: 1.0.5 + - repo: https://github.com/shellcheck-py/shellcheck-py + rev: v0.9.0.6 hooks: - - id: shell-lint + - id: shellcheck # Python hooks - repo: https://github.com/PyCQA/bandit From cea8edc5bcdcec8a06b6b810514b0222fc03f42e Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 14 Feb 2024 12:59:30 -0600 Subject: [PATCH 230/326] Add checks for semantic python versions --- setup-env | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/setup-env b/setup-env index 3a22d43..5e537bc 100755 --- a/setup-env +++ b/setup-env @@ -39,6 +39,14 @@ python_versions() { pyenv versions --bare --skip-aliases --skip-envs } +check_semantic_version() { + local version=$1 + local regex="^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?\$" + + # Use Perl for regex matching and output true or false + echo "$version" | perl -ne "exit(!/$regex/)" +} + # Flag to force deletion and creation of virtual environment FORCE=0 @@ -144,16 +152,18 @@ while true; do -p | --python-version) PYTHON_VERSION="$2" shift 2 - # Check the Python versions being passed in. - if [ -n "${PYTHON_VERSION+x}" ]; then - if python_versions | grep -E "^${PYTHON_VERSION}$" > /dev/null; then - echo Using Python version "$PYTHON_VERSION" - else - echo Error: Python version "$PYTHON_VERSION" is not installed. - echo Installed Python versions are: - python_versions - exit 1 - fi + # Validate the semantic version format + if ! check_semantic_version "$PYTHON_VERSION"; then + echo "Error: The specified Python version $PYTHON_VERSION does not follow the semantic versioning standard." + echo "Example of a valid version: 3.8.1, 3.8.1-alpha.1, or 3.8.1+20130313144700" + exit 1 + elif ! python_versions | grep -E "^${PYTHON_VERSION}$" > /dev/null; then + echo "Error: Python version $PYTHON_VERSION is not installed." + echo "Installed Python versions are:" + python_versions + exit 1 + else + echo "Using Python version $PYTHON_VERSION" fi ;; -v | --venv-name) @@ -189,14 +199,7 @@ if [ $LIST_VERSIONS -ne 0 ]; then # -r: treat backslashes as literal, -p: display prompt before input. read -r -p "Enter the desired Python version: " PYTHON_VERSION # Check the Python versions being passed in. - if [ -n "${PYTHON_VERSION+x}" ]; then - if python_versions | grep -E "^${PYTHON_VERSION}$" > /dev/null; then - echo Using Python version "$PYTHON_VERSION" - else - echo Error: Python version "$PYTHON_VERSION" is not installed. - exit 1 - fi - fi + check_semantic_version "$PYTHON_VERSION" fi # Remove any lingering local configuration. From d5c7c4a566f88f7575f06ff2e0829f257a00cb08 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 14 Feb 2024 14:29:13 -0600 Subject: [PATCH 231/326] Refactor code for the semantic check This commit will make a few changes. The orginal version of the semantic checking function was a bit more difficult to read. It is now somewhat easier to follow how the regex is structured. Also the function has been renamed to check_python_version since it has 2 functions, making sure that the version is semantically correct and the second is to make sure that it is installed on the user's machine. This makes it easier to follow the logic for the flags, -p or --python-version and -l or --list-versions --- setup-env | 54 ++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 36 insertions(+), 18 deletions(-) diff --git a/setup-env b/setup-env index 5e537bc..92540d1 100755 --- a/setup-env +++ b/setup-env @@ -39,12 +39,41 @@ python_versions() { pyenv versions --bare --skip-aliases --skip-envs } -check_semantic_version() { +check_python_version() { local version=$1 - local regex="^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?\$" - # Use Perl for regex matching and output true or false - echo "$version" | perl -ne "exit(!/$regex/)" + # Break down the regex into readable parts major.minor.patch + local major="0|[1-9]\\d*" + local minor="0|[1-9]\\d*" + local patch="0|[1-9]\\d*" + + # Splitting the prerelease part for readability + # Start of prerelease + local prerelease="(?:-" + # Numeric or alphanumeric identifiers + local prerelease+="(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)" + # Additional dot-separated identifiers + local prerelease+="(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*" + # End of prerelease, making it optional + local prerelease+=")?" + # Optional build metadata + local build="(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?" + + # Final regex composed of parts + local regex="^($major)\\.($minor)\\.($patch)$prerelease$build\$" + + if ! echo "$version" | perl -ne "exit(!/$regex/)"; then + echo "Error: The specified Python version $version does not follow the semantic versioning standard." + echo "Example of a valid version: 3.8.1, 3.8.1-alpha.1, or 3.8.1+20130313144700" + exit 1 + elif ! python_versions | grep -E "^${version}$" > /dev/null; then + echo "Error: Python version $version is not installed." + echo "Installed Python versions are:" + python_versions + exit 1 + else + echo "Using Python version $version" + fi } # Flag to force deletion and creation of virtual environment @@ -152,19 +181,8 @@ while true; do -p | --python-version) PYTHON_VERSION="$2" shift 2 - # Validate the semantic version format - if ! check_semantic_version "$PYTHON_VERSION"; then - echo "Error: The specified Python version $PYTHON_VERSION does not follow the semantic versioning standard." - echo "Example of a valid version: 3.8.1, 3.8.1-alpha.1, or 3.8.1+20130313144700" - exit 1 - elif ! python_versions | grep -E "^${PYTHON_VERSION}$" > /dev/null; then - echo "Error: Python version $PYTHON_VERSION is not installed." - echo "Installed Python versions are:" - python_versions - exit 1 - else - echo "Using Python version $PYTHON_VERSION" - fi + # Check the Python version being passed in. + check_python_version "$PYTHON_VERSION" ;; -v | --venv-name) VENV_NAME="$2" @@ -199,7 +217,7 @@ if [ $LIST_VERSIONS -ne 0 ]; then # -r: treat backslashes as literal, -p: display prompt before input. read -r -p "Enter the desired Python version: " PYTHON_VERSION # Check the Python versions being passed in. - check_semantic_version "$PYTHON_VERSION" + check_python_version "$PYTHON_VERSION" fi # Remove any lingering local configuration. From 327ab733aeaaad6a4916eb86b20d86618c9351e3 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Mon, 18 Mar 2024 12:36:02 -0500 Subject: [PATCH 232/326] Remove example of correct semantic version --- setup-env | 1 - 1 file changed, 1 deletion(-) diff --git a/setup-env b/setup-env index 92540d1..bacd2d5 100755 --- a/setup-env +++ b/setup-env @@ -64,7 +64,6 @@ check_python_version() { if ! echo "$version" | perl -ne "exit(!/$regex/)"; then echo "Error: The specified Python version $version does not follow the semantic versioning standard." - echo "Example of a valid version: 3.8.1, 3.8.1-alpha.1, or 3.8.1+20130313144700" exit 1 elif ! python_versions | grep -E "^${version}$" > /dev/null; then echo "Error: Python version $version is not installed." From 4dedf50886fd47c67895deb07367fca5c36ca33f Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 20 Mar 2024 12:58:03 -0500 Subject: [PATCH 233/326] Refactor the error message for the user --- setup-env | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup-env b/setup-env index bacd2d5..d7824cb 100755 --- a/setup-env +++ b/setup-env @@ -63,7 +63,9 @@ check_python_version() { local regex="^($major)\\.($minor)\\.($patch)$prerelease$build\$" if ! echo "$version" | perl -ne "exit(!/$regex/)"; then - echo "Error: The specified Python version $version does not follow the semantic versioning standard." + echo "Invalid version of Python: Python follows semantic versioning, " \ + "so any version string that is not a valid semantic version is an " \ + "invalid version of Python." exit 1 elif ! python_versions | grep -E "^${version}$" > /dev/null; then echo "Error: Python version $version is not installed." From e84deea5181f27471f01343113c91dc2b13e159e Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 20 Mar 2024 14:52:16 -0500 Subject: [PATCH 234/326] Improve the semantic error message --- setup-env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup-env b/setup-env index d7824cb..bba5f9e 100755 --- a/setup-env +++ b/setup-env @@ -63,8 +63,8 @@ check_python_version() { local regex="^($major)\\.($minor)\\.($patch)$prerelease$build\$" if ! echo "$version" | perl -ne "exit(!/$regex/)"; then - echo "Invalid version of Python: Python follows semantic versioning, " \ - "so any version string that is not a valid semantic version is an " \ + echo "Invalid version of Python: Python follows semantic versioning," \ + "so any version string that is not a valid semantic version is an" \ "invalid version of Python." exit 1 elif ! python_versions | grep -E "^${version}$" > /dev/null; then From 5fdc7befc1d1d4811c4550ca1e4c65a711971c21 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 20 Mar 2024 15:39:07 -0500 Subject: [PATCH 235/326] Fix grammar Co-authored-by: dav3r --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index bba5f9e..b93810c 100755 --- a/setup-env +++ b/setup-env @@ -217,7 +217,7 @@ if [ $LIST_VERSIONS -ne 0 ]; then # Read the user's desired Python version. # -r: treat backslashes as literal, -p: display prompt before input. read -r -p "Enter the desired Python version: " PYTHON_VERSION - # Check the Python versions being passed in. + # Check the Python version being passed in. check_python_version "$PYTHON_VERSION" fi From 42ef8c2d7b54cde82d4390a0050622cddfccf92a Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Thu, 21 Mar 2024 09:19:42 -0500 Subject: [PATCH 236/326] Refactor regex, add link, and improve comments --- setup-env | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/setup-env b/setup-env index b93810c..2f30021 100755 --- a/setup-env +++ b/setup-env @@ -42,31 +42,38 @@ python_versions() { check_python_version() { local version=$1 + # This is a valid regex for semantically correct Python version strings. + # For more information see here: https://regex101.com/r/vkijKf/1/. # Break down the regex into readable parts major.minor.patch - local major="0|[1-9]\\d*" - local minor="0|[1-9]\\d*" - local patch="0|[1-9]\\d*" + local major="0|[1-9]\d*" + local minor="0|[1-9]\d*" + local patch="0|[1-9]\d*" # Splitting the prerelease part for readability - # Start of prerelease + # Start of the prerelease local prerelease="(?:-" # Numeric or alphanumeric identifiers - local prerelease+="(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)" + local prerelease+="(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)" # Additional dot-separated identifiers - local prerelease+="(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*" - # End of prerelease, making it optional + local prerelease+="(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*" + # End of the prerelease, making it optional local prerelease+=")?" # Optional build metadata - local build="(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?" + local build="(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?" # Final regex composed of parts - local regex="^($major)\\.($minor)\\.($patch)$prerelease$build\$" + local regex="^($major)\.($minor)\.($patch)$prerelease$build$" + # This checks if the Python version does not match the regex pattern specified in $regex, + # using Perl for regex matching. If the pattern is not found, then prompt the user with + # the invalid version message. if ! echo "$version" | perl -ne "exit(!/$regex/)"; then echo "Invalid version of Python: Python follows semantic versioning," \ "so any version string that is not a valid semantic version is an" \ "invalid version of Python." exit 1 + # Else if the Python version isn't installed then notify the user. + # grep -E is used for searching through text lines that match the specific verison. elif ! python_versions | grep -E "^${version}$" > /dev/null; then echo "Error: Python version $version is not installed." echo "Installed Python versions are:" From a77e5e1c9a8752a2072a6a974d4164be116069e9 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Thu, 21 Mar 2024 10:13:11 -0500 Subject: [PATCH 237/326] Update link to use semver.org over regex101.com --- setup-env | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 2f30021..8d7b347 100755 --- a/setup-env +++ b/setup-env @@ -43,7 +43,8 @@ check_python_version() { local version=$1 # This is a valid regex for semantically correct Python version strings. - # For more information see here: https://regex101.com/r/vkijKf/1/. + # For more information see here: + # https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string. # Break down the regex into readable parts major.minor.patch local major="0|[1-9]\d*" local minor="0|[1-9]\d*" From 5fe14c7c6066d30381f6746eb313a56e4d447ac5 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Thu, 21 Mar 2024 10:29:58 -0500 Subject: [PATCH 238/326] Remove unnecessary period Co-authored-by: dav3r --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 8d7b347..059ccad 100755 --- a/setup-env +++ b/setup-env @@ -44,7 +44,7 @@ check_python_version() { # This is a valid regex for semantically correct Python version strings. # For more information see here: - # https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string. + # https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string # Break down the regex into readable parts major.minor.patch local major="0|[1-9]\d*" local minor="0|[1-9]\d*" From b7896a0a2790cc121842c6ac1602734bbd5dd726 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Sat, 20 Apr 2024 04:11:57 -0400 Subject: [PATCH 239/326] Add a meta hook to the pre-commit configuration Add the `check-useless-excludes` meta hook to verify that any defined `exclude` directives apply to at least one file in the repository. --- .pre-commit-config.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2c5b3c8..de8c587 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,6 +4,11 @@ default_language_version: python: python3 repos: + # Check the pre-commit configuration + - repo: meta + hooks: + - id: check-useless-excludes + - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.5.0 hooks: From 260566f177520175530963c469e50d124e5bc0e4 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Sat, 20 Apr 2024 04:15:52 -0400 Subject: [PATCH 240/326] Remove `exclude` directive that does not apply to any files --- .pre-commit-config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index de8c587..5ec468e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,7 +24,6 @@ repos: - --allow-missing-credentials - id: detect-private-key - id: end-of-file-fixer - exclude: files/(issue|motd) - id: mixed-line-ending args: - --fix=lf From 07e2b60f912ada6de4de0dcf0573bbecc0f2037c Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Sat, 4 May 2024 10:09:12 -0400 Subject: [PATCH 241/326] Pin ansible-core when running the ansible-lint linter New versions of ansible-core (2.16.7 and 2.17.0) have been released that do not suffer from the bug discussed in ansible/ansible#82702. This bug broke any symlinked files in vars, tasks, etc. for any Ansible role installed via ansible-galaxy. All versions later than ansible-core 2.16.7 and 2.17.0 should function as expected. Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- .pre-commit-config.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2c5b3c8..895384c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -155,6 +155,17 @@ repos: rev: v24.2.0 hooks: - id: ansible-lint + additional_dependencies: + # ansible-core 2.16.3 through 2.16.6 suffer from the bug + # discussed in ansible/ansible#82702, which breaks any + # symlinked files in vars, tasks, etc. for any Ansible role + # installed via ansible-galaxy. Hence we never want to + # install those versions. + # + # Note that any changes made to this dependency must also be + # made in requirements.txt in cisagov/skeleton-packer and + # requirements-test.txt in cisagov/skeleton-ansible-role. + - ansible-core>=2.16.7 # files: molecule/default/playbook.yml # Terraform hooks From c74e5db75b7e3785a3f7196365b7ba99f9a004ea Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Sat, 4 May 2024 10:11:08 -0400 Subject: [PATCH 242/326] Remove unnecessary line The line is not only unnecessary, it was commented out to boot! --- .pre-commit-config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 895384c..11772d9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -166,7 +166,6 @@ repos: # made in requirements.txt in cisagov/skeleton-packer and # requirements-test.txt in cisagov/skeleton-ansible-role. - ansible-core>=2.16.7 - # files: molecule/default/playbook.yml # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform From 2e53e0de98caa5d5a42320618d8e604c770da7d9 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 7 May 2024 15:21:06 -0400 Subject: [PATCH 243/326] Explain why ansible may need to be added as a dependency for ansible-lint On its own ansible-lint does not pull in ansible, only ansible-core. Therefore, if an Ansible module lives in ansible instead of ansible-core, the linter will complain that the module is unknown. In these cases it is necessary to add the ansible package itself as an additional dependency, with the same pinning as is done in requirements-test.txt of cisagov/skeleton-ansible-role. --- .pre-commit-config.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 11772d9..a48e196 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -156,6 +156,14 @@ repos: hooks: - id: ansible-lint additional_dependencies: + # On its own ansible-lint does not pull in ansible, only + # ansible-core. Therefore, if an Ansible module lives in + # ansible instead of ansible-core, the linter will complain + # that the module is unknown. In these cases it is + # necessary to add the ansible package itself as an + # additional dependency, with the same pinning as is done in + # requirements-test.txt of cisagov/skeleton-ansible-role. + # - ansible>=8,<10 # ansible-core 2.16.3 through 2.16.6 suffer from the bug # discussed in ansible/ansible#82702, which breaks any # symlinked files in vars, tasks, etc. for any Ansible role From f51fe623bb34e68bb874cd30756fee2692d34b5a Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Sat, 11 May 2024 00:38:50 -0400 Subject: [PATCH 244/326] Update pre-commit hook versions This is done automatically with the `pre-commit autoupdate` command. The pre-commit/mirrors-prettier was manually held back because the latest tags are for alpha releases of the next major version. --- .pre-commit-config.yaml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2c5b3c8..d315345 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ default_language_version: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v4.6.0 hooks: - id: check-case-conflict - id: check-executables-have-shebangs @@ -31,7 +31,7 @@ repos: # Text file hooks - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.39.0 + rev: v0.41.0 hooks: - id: markdownlint args: @@ -56,14 +56,14 @@ repos: # GitHub Actions hooks - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.28.0 + rev: 0.28.4 hooks: - id: check-github-actions - id: check-github-workflows # pre-commit hooks - repo: https://github.com/pre-commit/pre-commit - rev: v3.6.2 + rev: v3.7.1 hooks: - id: validate_manifest @@ -98,7 +98,7 @@ repos: # Shell script hooks - repo: https://github.com/scop/pre-commit-shfmt - rev: v3.7.0-4 + rev: v3.8.0-1 hooks: - id: shfmt args: @@ -116,19 +116,19 @@ repos: # Redirect operators are followed by a space - --space-redirects - repo: https://github.com/shellcheck-py/shellcheck-py - rev: v0.9.0.6 + rev: v0.10.0.1 hooks: - id: shellcheck # Python hooks - repo: https://github.com/PyCQA/bandit - rev: 1.7.7 + rev: 1.7.8 hooks: - id: bandit args: - --config=.bandit.yml - repo: https://github.com/psf/black-pre-commit-mirror - rev: 24.2.0 + rev: 24.4.2 hooks: - id: black - repo: https://github.com/PyCQA/flake8 @@ -142,24 +142,24 @@ repos: hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.8.0 + rev: v1.10.0 hooks: - id: mypy - repo: https://github.com/asottile/pyupgrade - rev: v3.15.1 + rev: v3.15.2 hooks: - id: pyupgrade # Ansible hooks - repo: https://github.com/ansible/ansible-lint - rev: v24.2.0 + rev: v24.6.0 hooks: - id: ansible-lint # files: molecule/default/playbook.yml # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.88.0 + rev: v1.90.0 hooks: - id: terraform_fmt - id: terraform_validate From 8e55b8e24e047008b1fa80bc87a76163888e4e5e Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 6 Jun 2024 13:58:38 -0400 Subject: [PATCH 245/326] Manually update the prettier hook Use the latest v3 release available from NPM. --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d315345..2ece2c0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -46,7 +46,7 @@ repos: # mirror does not pull tags for old major versions once a new major # version tag is published. additional_dependencies: - - prettier@3.2.5 + - prettier@3.3.1 - repo: https://github.com/adrienverge/yamllint rev: v1.35.1 hooks: From c617bb92af0bd01d0ef9bb7e51c007e34f91a915 Mon Sep 17 00:00:00 2001 From: Shane Frasier Date: Thu, 6 Jun 2024 16:42:24 -0400 Subject: [PATCH 246/326] Correct commented-out ansible pin The pin now agrees with what is in cisagov/skeleton-ansible-role. Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a48e196..3071c44 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -163,7 +163,7 @@ repos: # necessary to add the ansible package itself as an # additional dependency, with the same pinning as is done in # requirements-test.txt of cisagov/skeleton-ansible-role. - # - ansible>=8,<10 + # - ansible>=9,<10 # ansible-core 2.16.3 through 2.16.6 suffer from the bug # discussed in ansible/ansible#82702, which breaks any # symlinked files in vars, tasks, etc. for any Ansible role From a68994d17dcc11e9b90132c50fe52732d5fda07b Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Mon, 1 Jul 2024 16:19:46 -0400 Subject: [PATCH 247/326] Add a lower-bound pin for flake8-docstrings --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 386c83f..74c9c76 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -136,7 +136,7 @@ repos: hooks: - id: flake8 additional_dependencies: - - flake8-docstrings + - flake8-docstrings>=1.7.0 - repo: https://github.com/PyCQA/isort rev: 5.13.2 hooks: From 43b91c74754e912172c702e20f12ba9f767ac202 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 12 Aug 2024 06:24:06 -0400 Subject: [PATCH 248/326] Use the hashicorp/setup-packer GitHub Action Instead of manually installing Packer we can instead leverage the hashicorp/setup-packer Action just as we do for Terraform. --- .github/workflows/build.yml | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9bb221a..e12b842 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,7 +20,6 @@ defaults: shell: bash -Eueo pipefail -x {0} env: - CURL_CACHE_DIR: ~/.cache/curl PIP_CACHE_DIR: ~/.cache/pip PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit RUN_TMATE: ${{ secrets.RUN_TMATE }} @@ -97,25 +96,12 @@ jobs: path: | ${{ env.PIP_CACHE_DIR }} ${{ env.PRE_COMMIT_CACHE_DIR }} - ${{ env.CURL_CACHE_DIR }} ${{ steps.go-cache.outputs.dir }} restore-keys: | ${{ env.BASE_CACHE_KEY }} - - name: Setup curl cache - run: mkdir -p ${{ env.CURL_CACHE_DIR }} - - name: Install Packer - env: - PACKER_VERSION: ${{ steps.setup-env.outputs.packer-version }} - run: | - PACKER_ZIP="packer_${PACKER_VERSION}_linux_amd64.zip" - curl --output ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \ - --time-cond ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \ - --location \ - "https://releases.hashicorp.com/packer/${PACKER_VERSION}/${PACKER_ZIP}" - sudo unzip -d /opt/packer \ - ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" - sudo mv /usr/local/bin/packer /usr/local/bin/packer-default - sudo ln -s /opt/packer/packer /usr/local/bin/packer + - uses: hashicorp/setup-packer@v3 + with: + version: ${{ steps.setup-env.outputs.packer-version }} - uses: hashicorp/setup-terraform@v3 with: terraform_version: ${{ steps.setup-env.outputs.terraform-version }} From 8ada75d419c3ea546843fc0772d9d0b678beeea4 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 23 Aug 2024 00:54:54 -0400 Subject: [PATCH 249/326] Remove @jasonodoom as a codeowner He is no longer a member of @cisagov/vm-dev. --- .github/CODEOWNERS | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 229920c..3af99ba 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,22 +3,22 @@ # These owners will be the default owners for everything in the # repo. Unless a later match takes precedence, these owners will be # requested for review when someone opens a pull request. -* @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +* @dav3r @felddy @jsf9k @mcdonnnj # These folks own any files in the .github directory at the root of # the repository and any of its subdirectories. -/.github/ @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.github/ @dav3r @felddy @jsf9k @mcdonnnj # These folks own all linting configuration files. -/.ansible-lint @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/.bandit.yml @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/.flake8 @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/.isort.cfg @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/.mdl_config.yaml @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/.pre-commit-config.yaml @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/.prettierignore @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/.yamllint @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/requirements.txt @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/requirements-dev.txt @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/requirements-test.txt @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/setup-env @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.ansible-lint @dav3r @felddy @jsf9k @mcdonnnj +/.bandit.yml @dav3r @felddy @jsf9k @mcdonnnj +/.flake8 @dav3r @felddy @jsf9k @mcdonnnj +/.isort.cfg @dav3r @felddy @jsf9k @mcdonnnj +/.mdl_config.yaml @dav3r @felddy @jsf9k @mcdonnnj +/.pre-commit-config.yaml @dav3r @felddy @jsf9k @mcdonnnj +/.prettierignore @dav3r @felddy @jsf9k @mcdonnnj +/.yamllint @dav3r @felddy @jsf9k @mcdonnnj +/requirements.txt @dav3r @felddy @jsf9k @mcdonnnj +/requirements-dev.txt @dav3r @felddy @jsf9k @mcdonnnj +/requirements-test.txt @dav3r @felddy @jsf9k @mcdonnnj +/setup-env @dav3r @felddy @jsf9k @mcdonnnj From 293020830fb6830a7324b5eacb8c3122979d9882 Mon Sep 17 00:00:00 2001 From: Shane Frasier Date: Mon, 26 Aug 2024 09:27:58 -0400 Subject: [PATCH 250/326] Pin to a specific version Previously we only provided a lower bound for the version, but pinning to a specific version aligns with what has been done with the prettier hook and how pre-commit hooks are pinned in general. The flake8-docstrings package is rarely updated, so there is no real downside to pinning to a specific version. Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 74c9c76..236eeda 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -136,7 +136,7 @@ repos: hooks: - id: flake8 additional_dependencies: - - flake8-docstrings>=1.7.0 + - flake8-docstrings==1.7.0 - repo: https://github.com/PyCQA/isort rev: 5.13.2 hooks: From 46e055367c1e34711ed0980b2934b9df54bf33fe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 24 Sep 2024 18:23:01 +0000 Subject: [PATCH 251/326] Bump actions/cache from 3 to 4 Bumps [actions/cache](https://github.com/actions/cache) from 3 to 4. - [Release notes](https://github.com/actions/cache/releases) - [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md) - [Commits](https://github.com/actions/cache/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/cache dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9bb221a..a403ea9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -76,7 +76,7 @@ jobs: name: Lookup Go cache directory run: | echo "dir=$(go env GOCACHE)" >> $GITHUB_OUTPUT - - uses: actions/cache@v3 + - uses: actions/cache@v4 env: BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ py${{ steps.setup-python.outputs.python-version }}-\ From 3167421109abf3fe94dc801203587e1bf3ce33a3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 24 Sep 2024 18:23:14 +0000 Subject: [PATCH 252/326] Bump crazy-max/ghaction-github-status from 3 to 4 Bumps [crazy-max/ghaction-github-status](https://github.com/crazy-max/ghaction-github-status) from 3 to 4. - [Release notes](https://github.com/crazy-max/ghaction-github-status/releases) - [Commits](https://github.com/crazy-max/ghaction-github-status/compare/v3...v4) --- updated-dependencies: - dependency-name: crazy-max/ghaction-github-status dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/sync-labels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync-labels.yml b/.github/workflows/sync-labels.yml index 5a20438..e83bd41 100644 --- a/.github/workflows/sync-labels.yml +++ b/.github/workflows/sync-labels.yml @@ -24,7 +24,7 @@ jobs: egress-policy: audit - id: github-status name: Check GitHub status - uses: crazy-max/ghaction-github-status@v3 + uses: crazy-max/ghaction-github-status@v4 - id: dump-context name: Dump context uses: crazy-max/ghaction-dump-context@v2 From 6a58c2c24ef1eb15c7a69a44f16c63964f1c7f82 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 12 Aug 2024 16:23:58 -0400 Subject: [PATCH 253/326] Update pre-commit hook versions This is done automatically with the `pre-commit autoupdate` command. The pre-commit/mirrors-prettier hook was manually held back because the latest tags are for alpha releases of the next major version. --- .pre-commit-config.yaml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 386c83f..81f3276 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,7 +31,7 @@ repos: # Text file hooks - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.41.0 + rev: v0.42.0 hooks: - id: markdownlint args: @@ -56,14 +56,14 @@ repos: # GitHub Actions hooks - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.28.4 + rev: 0.29.2 hooks: - id: check-github-actions - id: check-github-workflows # pre-commit hooks - repo: https://github.com/pre-commit/pre-commit - rev: v3.7.1 + rev: v3.8.0 hooks: - id: validate_manifest @@ -98,7 +98,7 @@ repos: # Shell script hooks - repo: https://github.com/scop/pre-commit-shfmt - rev: v3.8.0-1 + rev: v3.9.0-1 hooks: - id: shfmt args: @@ -122,17 +122,17 @@ repos: # Python hooks - repo: https://github.com/PyCQA/bandit - rev: 1.7.8 + rev: 1.7.10 hooks: - id: bandit args: - --config=.bandit.yml - repo: https://github.com/psf/black-pre-commit-mirror - rev: 24.4.2 + rev: 24.8.0 hooks: - id: black - repo: https://github.com/PyCQA/flake8 - rev: 7.0.0 + rev: 7.1.1 hooks: - id: flake8 additional_dependencies: @@ -142,17 +142,17 @@ repos: hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.10.0 + rev: v1.11.2 hooks: - id: mypy - repo: https://github.com/asottile/pyupgrade - rev: v3.15.2 + rev: v3.17.0 hooks: - id: pyupgrade # Ansible hooks - repo: https://github.com/ansible/ansible-lint - rev: v24.6.0 + rev: v24.9.2 hooks: - id: ansible-lint additional_dependencies: @@ -177,7 +177,7 @@ repos: # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.90.0 + rev: v1.96.1 hooks: - id: terraform_fmt - id: terraform_validate @@ -190,7 +190,7 @@ repos: # Packer hooks - repo: https://github.com/cisagov/pre-commit-packer - rev: v0.0.2 + rev: v0.1.0 hooks: - id: packer_validate - id: packer_fmt From 553efcb0d4e755ebd47abb49c865367ed6d0a236 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 12 Aug 2024 16:30:49 -0400 Subject: [PATCH 254/326] Manually update the prettier hook Use the latest v3 release available from NPM. --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 81f3276..2104775 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -46,7 +46,7 @@ repos: # mirror does not pull tags for old major versions once a new major # version tag is published. additional_dependencies: - - prettier@3.3.1 + - prettier@3.3.3 - repo: https://github.com/adrienverge/yamllint rev: v1.35.1 hooks: From 045a998dcf14dc7e3de9301ba7ee2103272b0ac4 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 18 Jan 2024 16:11:15 -0500 Subject: [PATCH 255/326] Add a pre-commit hook to run pip-audit The pip-audit tool will audit any supplied pip requirements files for vulnerable packages. --- .pre-commit-config.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2c5b3c8..78140ff 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -145,6 +145,18 @@ repos: rev: v1.8.0 hooks: - id: mypy + - repo: https://github.com/pypa/pip-audit + rev: v2.7.3 + hooks: + - id: pip-audit + args: + # Add any pip requirements files to scan + - --requirement + - requirements-dev.txt + - --requirement + - requirements-test.txt + - --requirement + - requirements.txt - repo: https://github.com/asottile/pyupgrade rev: v3.15.1 hooks: From c502f1ab7cca8bd383a34360ce456b50fd6e8b21 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 12 Aug 2024 16:32:02 -0400 Subject: [PATCH 256/326] Use the rbubley/mirrors-prettier hook for prettier This replaces the now archived pre-commit/mirrors-prettier hook. --- .pre-commit-config.yaml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ca59d6f..3cb1f85 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -40,17 +40,10 @@ repos: - id: markdownlint args: - --config=.mdl_config.yaml - - repo: https://github.com/pre-commit/mirrors-prettier - # This is the last version of v3 available from the mirror. We should hold - # here until v4, which is currently in alpha, is more stable. - rev: v3.1.0 + - repo: https://github.com/rbubley/mirrors-prettier + rev: v3.3.3 hooks: - id: prettier - # This is the latest version of v3 available from NPM. The pre-commit - # mirror does not pull tags for old major versions once a new major - # version tag is published. - additional_dependencies: - - prettier@3.3.3 - repo: https://github.com/adrienverge/yamllint rev: v1.35.1 hooks: From 4f86062dfc30c6a5e2d6ae0f6f9564e08112b964 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 1 Oct 2024 03:43:21 -0400 Subject: [PATCH 257/326] Pull in the `bump-version` script from cisagov/pre-commit-packer --- bump-version | 172 +++++++++++++++++++++++++++++++++++++++++++ requirements-dev.txt | 2 +- 2 files changed, 173 insertions(+), 1 deletion(-) create mode 100755 bump-version diff --git a/bump-version b/bump-version new file mode 100755 index 0000000..b19230e --- /dev/null +++ b/bump-version @@ -0,0 +1,172 @@ +#!/usr/bin/env bash + +# bump-version [--push] [--label LABEL] (major | minor | patch | prerelease | build | finalize | show) +# bump-version --list-files + +set -o nounset +set -o errexit +set -o pipefail + +# Stores the canonical version for the project. +VERSION_FILE=config/version.txt +# Files that should be updated with the new version. +VERSION_FILES=("$VERSION_FILE" README.md) + +USAGE=$( + cat << END_OF_LINE +Update the version of the project. + +Usage: + ${0##*/} [--push] [--label LABEL] (major | minor | patch | prerelease | build | finalize | show) + ${0##*/} --list-files + ${0##*/} (-h | --help) + +Options: + -h | --help Show this message. + --push Perform a \`git push\` after updating the version. + --label LABEL Specify the label to use when updating the build or prerelease version. + --list-files List the files that will be updated when the version is bumped. +END_OF_LINE +) + +old_version=$(< "$VERSION_FILE") +# Comment out periods so they are interpreted as periods and don't +# just match any character +old_version_regex=${old_version//\./\\\.} +new_version="$old_version" + +bump_part="" +label="" +commit_prefix="Bump" +with_push=false +commands_with_label=("build" "prerelease") +commands_with_prerelease=("major" "minor" "patch") +with_prerelease=false + +####################################### +# Display an error message, the help information, and exit with a non-zero status. +# Arguments: +# Error message. +####################################### +function invalid_option() { + echo "$1" + echo "$USAGE" + exit 1 +} + +####################################### +# Bump the version using the provided command. +# Arguments: +# The version to bump. +# The command to bump the version. +# Returns: +# The new version. +####################################### +function bump_version() { + local temp_version + temp_version=$(python -c "import semver; print(semver.parse_version_info('$1').${2})") + echo "$temp_version" +} + +if [ $# -eq 0 ]; then + echo "$USAGE" + exit 1 +else + while [ $# -gt 0 ]; do + case $1 in + --push) + if [ "$with_push" = true ]; then + invalid_option "Push has already been set." + fi + + with_push=true + shift + ;; + --label) + if [ -n "$label" ]; then + invalid_option "Label has already been set." + fi + + label="$2" + shift 2 + ;; + build | finalize | major | minor | patch) + if [ -n "$bump_part" ]; then + invalid_option "Only one version part should be bumped at a time." + fi + + bump_part="$1" + shift + ;; + prerelease) + with_prerelease=true + shift + ;; + show) + echo "$old_version" + exit 0 + ;; + -h | --help) + echo "$USAGE" + exit 0 + ;; + --list-files) + printf '%s\n' "${VERSION_FILES[@]}" + exit 0 + ;; + *) + invalid_option "Invalid option: $1" + ;; + esac + done +fi + +if [ -n "$label" ] && [ "$with_prerelease" = false ] && [[ ! " ${commands_with_label[*]} " =~ [[:space:]]${bump_part}[[:space:]] ]]; then + invalid_option "Setting the label is only allowed for the following commands: ${commands_with_label[*]}" +fi + +if [ "$with_prerelease" = true ] && [[ ! " ${commands_with_prerelease[*]} " =~ [[:space:]]${bump_part}[[:space:]] ]]; then + invalid_option "Changing the prerelease is only allowed in conjunction with the following commands: ${commands_with_prerelease[*]}" +fi + +label_option="" +if [ -n "$label" ]; then + label_option="token='$label'" +fi + +if [ -n "$bump_part" ]; then + if [ "$bump_part" = "finalize" ]; then + commit_prefix="Finalize" + bump_command="finalize_version()" + elif [ "$bump_part" = "build" ]; then + bump_command="bump_${bump_part}($label_option)" + else + bump_command="bump_${bump_part}()" + fi + new_version=$(bump_version "$old_version" "$bump_command") + echo Changing version from "$old_version" to "$new_version" +fi + +if [ "$with_prerelease" = true ]; then + bump_command="bump_prerelease($label_option)" + temp_version=$(bump_version "$new_version" "$bump_command") + echo Changing version from "$new_version" to "$temp_version" + new_version="$temp_version" +fi + +tmp_file=/tmp/version.$$ +for version_file in "${VERSION_FILES[@]}"; do + if [ ! -f "$version_file" ]; then + echo Missing expected file: "$version_file" + exit 1 + fi + sed "s/$old_version_regex/$new_version/" "$version_file" > $tmp_file + mv $tmp_file "$version_file" +done + +git add "${VERSION_FILES[@]}" +git commit --message "$commit_prefix version from $old_version to $new_version" + +if [ "$with_push" = true ]; then + git push +fi diff --git a/requirements-dev.txt b/requirements-dev.txt index cb51627..de5eb3b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,3 @@ --requirement requirements-test.txt ipython -semver +semver>=3 From b780601c2ba9949edccc128dfd0df3c749701ff6 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 1 Oct 2024 03:47:01 -0400 Subject: [PATCH 258/326] Modify the `bump-version` script for this repository Copy over necessary changes from the `bump_version.sh` script. --- bump-version | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bump-version b/bump-version index b19230e..600265b 100755 --- a/bump-version +++ b/bump-version @@ -8,7 +8,7 @@ set -o errexit set -o pipefail # Stores the canonical version for the project. -VERSION_FILE=config/version.txt +VERSION_FILE=src/version.txt # Files that should be updated with the new version. VERSION_FILES=("$VERSION_FILE" README.md) @@ -29,7 +29,7 @@ Options: END_OF_LINE ) -old_version=$(< "$VERSION_FILE") +old_version=$(sed -n "s/^__version__ = \"\(.*\)\"$/\1/p" $VERSION_FILE) # Comment out periods so they are interpreted as periods and don't # just match any character old_version_regex=${old_version//\./\\\.} From e3bc34a02d32849341e255ba35026c4e1f935bd8 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 1 Oct 2024 03:47:33 -0400 Subject: [PATCH 259/326] Remove the `bump_version.sh` script This functionality has been replaced by the `bump-version` script. --- .github/workflows/build.yml | 4 +- bump_version.sh | 81 ------------------------------------- tag.sh | 2 +- 3 files changed, 3 insertions(+), 84 deletions(-) delete mode 100755 bump_version.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d8ba132..8078b05 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -173,7 +173,7 @@ jobs: # repometa: # The json metadata describing this repository. # source_version: - # The source version as reported by the `bump_version.sh show` command. + # The source version as reported by the `bump-version show` command. # tags: # A comma separated list of Docker tags to be applied to the images on # Docker Hub. The tags will vary depending on: @@ -265,7 +265,7 @@ jobs: TAGS="${TAGS},ghcr.io/${i}" done echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT - echo "source_version=$(./bump_version.sh show)" >> $GITHUB_OUTPUT + echo "source_version=$(./bump-version show)" >> $GITHUB_OUTPUT echo "tags=${TAGS}" >> $GITHUB_OUTPUT echo tags=${TAGS} - name: Setup tmate debug session diff --git a/bump_version.sh b/bump_version.sh deleted file mode 100755 index 0071670..0000000 --- a/bump_version.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env bash - -# Usage: -# bump_version.sh (show|major|minor|patch|finalize) -# bump_version.sh (build|prerelease) [token] -# Notes: -# - If you specify a token it will only be used if the current version is -# tokenless or if the provided token matches the token used in the current -# version. - -set -o nounset -set -o errexit -set -o pipefail - -VERSION_FILE=src/version.txt -README_FILE=README.md - -function usage { - cat << HELP -Usage: - ${0##*/} (show|major|minor|patch|finalize) - ${0##*/} (build|prerelease) [token] - -Notes: - - If you specify a token it will only be used if the current version is - tokenless or if the provided token matches the token used in the current - version. -HELP - exit 1 -} - -function update_version { - # Comment out periods so they are interpreted as periods and don't - # just match any character - old_version_regex=${1//\./\\\.} - - echo Changing version from "$1" to "$2" - tmp_file=/tmp/version.$$ - sed "s/$old_version_regex/$2/" $VERSION_FILE > $tmp_file - mv $tmp_file $VERSION_FILE - sed "s/$old_version_regex/$2/" $README_FILE > $tmp_file - mv $tmp_file $README_FILE - git add $VERSION_FILE $README_FILE - git commit --message "$3" -} - -if [ $# -lt 1 ] || [ $# -gt 2 ]; then - usage -else - old_version=$(sed -n "s/^__version__ = \"\(.*\)\"$/\1/p" $VERSION_FILE) - case $1 in - major | minor | patch) - if [ $# -ne 1 ]; then - usage - fi - new_version=$(python -c "import semver; print(semver.bump_$1('$old_version'))") - update_version "$old_version" "$new_version" "Bump version from $old_version to $new_version" - ;; - build | prerelease) - if [ $# -eq 2 ]; then - new_version=$(python -c "import semver; print(semver.bump_$1('$old_version', token='$2'))") - else - new_version=$(python -c "import semver; print(semver.bump_$1('$old_version'))") - fi - update_version "$old_version" "$new_version" "Bump version from $old_version to $new_version" - ;; - finalize) - if [ $# -ne 1 ]; then - usage - fi - new_version=$(python -c "import semver; print(semver.finalize_version('$old_version'))") - update_version "$old_version" "$new_version" "Finalize version from $old_version to $new_version" - ;; - show) - echo "$old_version" - ;; - *) - usage - ;; - esac -fi diff --git a/tag.sh b/tag.sh index e1f7447..0a0e607 100755 --- a/tag.sh +++ b/tag.sh @@ -4,6 +4,6 @@ set -o nounset set -o errexit set -o pipefail -version=$(./bump_version.sh show) +version=$(./bump-version show) git tag "v$version" && git push --tags From 942c0dc98f605282fdf3c0ac6b9a549647f89f41 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 13 Aug 2024 06:17:33 -0400 Subject: [PATCH 260/326] Add a new trigger for the sync-labels GitHub Actions workflow Add a `workflow_dispatch` trigger so we can manually run the workflow if needed. --- .github/workflows/sync-labels.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/sync-labels.yml b/.github/workflows/sync-labels.yml index e83bd41..59aefe4 100644 --- a/.github/workflows/sync-labels.yml +++ b/.github/workflows/sync-labels.yml @@ -6,6 +6,7 @@ on: paths: - '.github/labels.yml' - '.github/workflows/sync-labels.yml' + workflow_dispatch: permissions: contents: read From a267662455c30986086d4ca14173cc20af7161d4 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 13 Aug 2024 06:19:38 -0400 Subject: [PATCH 261/326] Remove unnecessary quotes in the sync-labels workflow --- .github/workflows/sync-labels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sync-labels.yml b/.github/workflows/sync-labels.yml index 59aefe4..5d5ab41 100644 --- a/.github/workflows/sync-labels.yml +++ b/.github/workflows/sync-labels.yml @@ -4,8 +4,8 @@ name: sync-labels on: push: paths: - - '.github/labels.yml' - - '.github/workflows/sync-labels.yml' + - .github/labels.yml + - .github/workflows/sync-labels.yml workflow_dispatch: permissions: From dc7f09e29b8466af0fa2f788761e22dd2fcbd0ce Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Sat, 14 Sep 2024 18:44:01 -0400 Subject: [PATCH 262/326] Add four new hooks from pre-commit/pre-commit-hooks --- .pre-commit-config.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3cb1f85..c98ded8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,9 +16,13 @@ repos: - id: check-executables-have-shebangs - id: check-json - id: check-merge-conflict + - id: check-shebang-scripts-are-executable + - id: check-symlinks - id: check-toml + - id: check-vcs-permalinks - id: check-xml - id: debug-statements + - id: destroyed-symlinks - id: detect-aws-credentials args: - --allow-missing-credentials From 343d2ccbd1cd983374235e5d3bfcecd3187c00d5 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 28 Oct 2024 17:47:53 -0400 Subject: [PATCH 263/326] Add the GitHubSecurityLab/actions-permissions/monitor Action This Action will provide information about the usage of GITHUB_TOKEN in the workflow. It should be added to _every_ job in _any_ workflow to provide information for analysis. --- .github/dependabot.yml | 1 + .github/workflows/build.yml | 10 ++++++++++ .github/workflows/sync-labels.yml | 10 ++++++++++ 3 files changed, 21 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 17220c6..4a6667f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -16,6 +16,7 @@ updates: # - dependency-name: crazy-max/ghaction-dump-context # - dependency-name: crazy-max/ghaction-github-labeler # - dependency-name: crazy-max/ghaction-github-status + # - dependency-name: GitHubSecurityLab/actions-permissions # - dependency-name: hashicorp/setup-terraform # - dependency-name: mxschmitt/action-tmate # - dependency-name: step-security/harden-runner diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7a60b2..2cdd921 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,6 +34,12 @@ jobs: steps: # Note that a duplicate of this step must be added at the top of # each job. + - uses: GitHubSecurityLab/actions-permissions/monitor@v1 + with: + # Uses the organization variable unless overridden + config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} + # Note that a duplicate of this step must be added at the top of + # each job. - id: harden-runner name: Harden the runner uses: step-security/harden-runner@v2 @@ -50,6 +56,10 @@ jobs: - diagnostics runs-on: ubuntu-latest steps: + - uses: GitHubSecurityLab/actions-permissions/monitor@v1 + with: + # Uses the organization variable unless overridden + config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} - id: harden-runner name: Harden the runner uses: step-security/harden-runner@v2 diff --git a/.github/workflows/sync-labels.yml b/.github/workflows/sync-labels.yml index e83bd41..d2458d1 100644 --- a/.github/workflows/sync-labels.yml +++ b/.github/workflows/sync-labels.yml @@ -17,6 +17,12 @@ jobs: steps: # Note that a duplicate of this step must be added at the top of # each job. + - uses: GitHubSecurityLab/actions-permissions/monitor@v1 + with: + # Uses the organization variable unless overridden + config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} + # Note that a duplicate of this step must be added at the top of + # each job. - id: harden-runner name: Harden the runner uses: step-security/harden-runner@v2 @@ -38,6 +44,10 @@ jobs: issues: write runs-on: ubuntu-latest steps: + - uses: GitHubSecurityLab/actions-permissions/monitor@v1 + with: + # Uses the organization variable unless overridden + config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} - id: harden-runner name: Harden the runner uses: step-security/harden-runner@v2 From 8a77a8b77a7d5e5247e8ff563d93a14510e09b9a Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 28 Oct 2024 17:09:15 -0400 Subject: [PATCH 264/326] Restrict permissions of GITHUB_TOKEN This changes the default permissions for the GITHUB_TOKEN used in our GitHub Actions configuration to the minimum required to successfully run. --- .github/workflows/build.yml | 5 +++++ .github/workflows/sync-labels.yml | 2 ++ 2 files changed, 7 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7a60b2..d4340af 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -30,6 +30,8 @@ env: jobs: diagnostics: name: Run diagnostics + # This job does not need any permissions + permissions: {} runs-on: ubuntu-latest steps: # Note that a duplicate of this step must be added at the top of @@ -48,6 +50,9 @@ jobs: lint: needs: - diagnostics + permissions: + # actions/checkout needs this to fetch code + contents: read runs-on: ubuntu-latest steps: - id: harden-runner diff --git a/.github/workflows/sync-labels.yml b/.github/workflows/sync-labels.yml index e83bd41..39e7379 100644 --- a/.github/workflows/sync-labels.yml +++ b/.github/workflows/sync-labels.yml @@ -13,6 +13,8 @@ permissions: jobs: diagnostics: name: Run diagnostics + # This job does not need any permissions + permissions: {} runs-on: ubuntu-latest steps: # Note that a duplicate of this step must be added at the top of From 3b1d4ef0fae08e6444e9b414ce1315841e681322 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 16 Oct 2024 12:53:42 -0400 Subject: [PATCH 265/326] Update pre-commit hook versions This is done automatically with the `pre-commit autoupdate` command. --- .pre-commit-config.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3cb1f85..26b399d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,7 +10,7 @@ repos: - id: check-useless-excludes - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v5.0.0 hooks: - id: check-case-conflict - id: check-executables-have-shebangs @@ -53,14 +53,14 @@ repos: # GitHub Actions hooks - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.29.2 + rev: 0.29.4 hooks: - id: check-github-actions - id: check-github-workflows # pre-commit hooks - repo: https://github.com/pre-commit/pre-commit - rev: v3.8.0 + rev: v4.0.1 hooks: - id: validate_manifest @@ -95,7 +95,7 @@ repos: # Shell script hooks - repo: https://github.com/scop/pre-commit-shfmt - rev: v3.9.0-1 + rev: v3.10.0-1 hooks: - id: shfmt args: @@ -125,7 +125,7 @@ repos: args: - --config=.bandit.yml - repo: https://github.com/psf/black-pre-commit-mirror - rev: 24.8.0 + rev: 24.10.0 hooks: - id: black - repo: https://github.com/PyCQA/flake8 @@ -139,7 +139,7 @@ repos: hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.11.2 + rev: v1.13.0 hooks: - id: mypy - repo: https://github.com/pypa/pip-audit @@ -155,7 +155,7 @@ repos: - --requirement - requirements.txt - repo: https://github.com/asottile/pyupgrade - rev: v3.17.0 + rev: v3.19.0 hooks: - id: pyupgrade @@ -199,7 +199,7 @@ repos: # Packer hooks - repo: https://github.com/cisagov/pre-commit-packer - rev: v0.1.0 + rev: v0.3.0 hooks: - id: packer_validate - id: packer_fmt From 1d285f2d851926effdbfbdcf58853ce70d1bf016 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 29 Oct 2024 16:36:27 -0400 Subject: [PATCH 266/326] Sort hook ids in each pre-commit hook entry Ensure that all hook ids are sorted alphabetically in each hook entry in our pre-commit configuration. --- .pre-commit-config.yaml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3cb1f85..0fd3234 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -68,25 +68,25 @@ repos: - repo: https://github.com/TekWizely/pre-commit-golang rev: v1.0.0-rc.1 hooks: - # Style Checkers - - id: go-critic - # StaticCheck - - id: go-staticcheck-repo-mod # Go Build - id: go-build-repo-mod + # Style Checkers + - id: go-critic + # goimports + - id: go-imports-repo + args: + # Write changes to files + - -w # Go Mod Tidy - id: go-mod-tidy-repo + # GoSec + - id: go-sec-repo-mod + # StaticCheck + - id: go-staticcheck-repo-mod # Go Test - id: go-test-repo-mod # Go Vet - id: go-vet-repo-mod - # GoSec - - id: go-sec-repo-mod - # goimports - - id: go-imports-repo - args: - # Write changes to files - - -w # Nix hooks - repo: https://github.com/nix-community/nixpkgs-fmt rev: v1.3.0 @@ -201,5 +201,5 @@ repos: - repo: https://github.com/cisagov/pre-commit-packer rev: v0.1.0 hooks: - - id: packer_validate - id: packer_fmt + - id: packer_validate From 8824475dfadd1a9cbc9ce9bd1c9f31e4a688994b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 1 Nov 2024 12:25:35 -0400 Subject: [PATCH 267/326] Update the commented out dependabot ignore directives Add a directive for hashicorp/setup-packer that was missed when it was added to the `build` workflow. Add a directive for cisagov/setup-env-github-action that is not strictly necessary since we currently just pull from the `develop` branch, but is good to have in case we were to change that in the future. --- .github/dependabot.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 4a6667f..81cd6bd 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -13,10 +13,12 @@ updates: # - dependency-name: actions/checkout # - dependency-name: actions/setup-go # - dependency-name: actions/setup-python + # - dependency-name: cisagov/setup-env-github-action # - dependency-name: crazy-max/ghaction-dump-context # - dependency-name: crazy-max/ghaction-github-labeler # - dependency-name: crazy-max/ghaction-github-status # - dependency-name: GitHubSecurityLab/actions-permissions + # - dependency-name: hashicorp/setup-packer # - dependency-name: hashicorp/setup-terraform # - dependency-name: mxschmitt/action-tmate # - dependency-name: step-security/harden-runner From 12a91ad97e76cd2f221fffaef4f66956533f6540 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 8 Nov 2024 13:40:44 -0500 Subject: [PATCH 268/326] Bump up the lower bound on ansible-core This is being done because the pip-audit pre-commit hook identifies a vulnerability in ansible-core version 2.16.13. Note that this requires that we bump up ansible to version 10 since all versions of ansible 9 have a dependency on ~=2.16.X. --- .pre-commit-config.yaml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c5e1096..8b402fb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -176,17 +176,25 @@ repos: # necessary to add the ansible package itself as an # additional dependency, with the same pinning as is done in # requirements-test.txt of cisagov/skeleton-ansible-role. - # - ansible>=9,<10 + # + # Version 10 is required because the pip-audit pre-commit + # hook identifies a vulnerability in ansible-core 2.16.13, + # but all versions of ansible 9 have a dependency on + # ~=2.16.X. + # - ansible>=10,<11 # ansible-core 2.16.3 through 2.16.6 suffer from the bug # discussed in ansible/ansible#82702, which breaks any # symlinked files in vars, tasks, etc. for any Ansible role # installed via ansible-galaxy. Hence we never want to # install those versions. # + # Note that the pip-audit pre-commit hook identifies a + # vulnerability in ansible-core 2.16.13. + # # Note that any changes made to this dependency must also be # made in requirements.txt in cisagov/skeleton-packer and # requirements-test.txt in cisagov/skeleton-ansible-role. - - ansible-core>=2.16.7 + - ansible-core>2.16.13 # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform From b9f798d03afb72f33ffa625982dd5b548dea5132 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 13 Nov 2024 10:29:42 -0500 Subject: [PATCH 269/326] Update the version of the ansible-lint pre-commit hook Version 24.10.0 is the first version that supports Fedora 41 as a valid platform. --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c5e1096..ebd6138 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -165,7 +165,7 @@ repos: # Ansible hooks - repo: https://github.com/ansible/ansible-lint - rev: v24.9.2 + rev: v24.10.0 hooks: - id: ansible-lint additional_dependencies: From cca133a2710c5ed99e4c0ce3d06a57ec118bcf13 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 13 Nov 2024 21:33:32 -0500 Subject: [PATCH 270/326] Adjust pin for ansible-core The pin of ansible-core was originally put in place because the pip-audit pre-commit hook identifies a vulnerability in ansible-core 2.16.13. Normally we would pin ansible-core to >2.16.13, but in the spirit of the earlier, optional pin of ansible>=10 we pin ansible-core to >=2.17. This effectively also pins ansible to >=10. Co-authored-by: Nick M <50747025+mcdonnnj@users.noreply.github.com> --- .pre-commit-config.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8b402fb..b61a8f5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -189,12 +189,14 @@ repos: # install those versions. # # Note that the pip-audit pre-commit hook identifies a - # vulnerability in ansible-core 2.16.13. + # vulnerability in ansible-core 2.16.13. The pin of + # ansible-core to >=2.17 effectively also pins ansible to + # >=10. # # Note that any changes made to this dependency must also be # made in requirements.txt in cisagov/skeleton-packer and # requirements-test.txt in cisagov/skeleton-ansible-role. - - ansible-core>2.16.13 + - ansible-core>=2.17 # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform From bd852610595fdd2eee77f489d4b184f88d90643b Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 20 Nov 2024 12:21:14 -0500 Subject: [PATCH 271/326] Add comments about looming EOL issues for ansible and ansible-core This adds even more evidence for why it is a good idea to go ahead and upgrade ansible and ansible-core, in addition to the vulnerability that pip-audit turned up. Co-authored-by: Nick M <50747025+mcdonnnj@users.noreply.github.com> --- .pre-commit-config.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b61a8f5..97fbf1c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -181,6 +181,10 @@ repos: # hook identifies a vulnerability in ansible-core 2.16.13, # but all versions of ansible 9 have a dependency on # ~=2.16.X. + # + # It is also a good idea to go ahead and upgrade to version + # 10 since version 9 is going EOL at the end of November: + # https://endoflife.date/ansible # - ansible>=10,<11 # ansible-core 2.16.3 through 2.16.6 suffer from the bug # discussed in ansible/ansible#82702, which breaks any @@ -193,6 +197,11 @@ repos: # ansible-core to >=2.17 effectively also pins ansible to # >=10. # + # It is also a good idea to go ahead and upgrade to + # ansible-core 2.17 since security support for ansible-core + # 2.16 ends this month: + # https://docs.ansible.com/ansible/devel/reference_appendices/release_and_maintenance.html#ansible-core-support-matrix + # # Note that any changes made to this dependency must also be # made in requirements.txt in cisagov/skeleton-packer and # requirements-test.txt in cisagov/skeleton-ansible-role. From 26ccf289eb4c09a4be02bd7e18c018c6c4b77a9b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 21 Nov 2024 16:58:03 -0500 Subject: [PATCH 272/326] Enable new dependabot ignore directives --- .github/dependabot.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index b4a8084..d01227a 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -18,12 +18,12 @@ updates: - dependency-name: actions/checkout - dependency-name: actions/setup-go - dependency-name: actions/setup-python - # - dependency-name: cisagov/setup-env-github-action + - dependency-name: cisagov/setup-env-github-action - dependency-name: crazy-max/ghaction-dump-context - dependency-name: crazy-max/ghaction-github-labeler - dependency-name: crazy-max/ghaction-github-status - # - dependency-name: GitHubSecurityLab/actions-permissions - # - dependency-name: hashicorp/setup-packer + - dependency-name: GitHubSecurityLab/actions-permissions + - dependency-name: hashicorp/setup-packer - dependency-name: hashicorp/setup-terraform - dependency-name: mxschmitt/action-tmate - dependency-name: step-security/harden-runner From ec72a4b015fc418a0be625fb1f0ad2316e028811 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 25 Sep 2024 09:51:30 -0400 Subject: [PATCH 273/326] Upgrade instances of actions/cache to v4 --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2b2bd00..ad412e0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -320,7 +320,7 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Cache Docker layers - uses: actions/cache@v3 + uses: actions/cache@v4 env: BASE_CACHE_KEY: buildx-${{ runner.os }}- with: @@ -401,7 +401,7 @@ jobs: with: python-version: "3.11" - name: Cache testing environments - uses: actions/cache@v3 + uses: actions/cache@v4 env: BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ py${{ steps.setup-python.outputs.python-version }}-" @@ -473,7 +473,7 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Cache Docker layers - uses: actions/cache@v3 + uses: actions/cache@v4 env: BASE_CACHE_KEY: buildx-${{ runner.os }}- with: From 4301e1c8f1475f7b179c325a1ce765b46065e4ec Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 25 Sep 2024 09:52:05 -0400 Subject: [PATCH 274/326] Upgrade instance of actions/setup-python to v5 Also pull the Python version from steps.setup-env.outputs instead of hardcoding it. --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ad412e0..fa6bc81 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -397,9 +397,9 @@ jobs: egress-policy: audit - uses: actions/checkout@v4 - id: setup-python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: ${{ steps.setup-env.outputs.python-version }} - name: Cache testing environments uses: actions/cache@v4 env: From 8aeb3b6d708ab1a064c5577cf867269dfe281122 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 21 Nov 2024 17:01:14 -0500 Subject: [PATCH 275/326] Upgrade instances of crazy-max/ghaction-github-status to v4 --- .github/workflows/codeql-analysis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index dc49271..382d3d1 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -33,7 +33,7 @@ jobs: egress-policy: audit - id: github-status name: Check GitHub status - uses: crazy-max/ghaction-github-status@v3 + uses: crazy-max/ghaction-github-status@v4 - id: dump-context name: Dump context uses: crazy-max/ghaction-dump-context@v2 From 31adcba8d17211611389315dc92da966a319687d Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 21 Nov 2024 17:14:08 -0500 Subject: [PATCH 276/326] Alphabetize entries in the build workflow We prefer to alphabetize mapping keys in YAML documents whenever possible. This ensures any parts specific to this repository are now alphabetized. --- .github/workflows/build.yml | 64 ++++++++++++++++++------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fa6bc81..53f65cf 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -5,28 +5,28 @@ on: merge_group: types: - checks_requested + pull_request: push: branches: - "**" tags: - "v*.*.*" - pull_request: - schedule: - - cron: "0 10 * * *" # everyday at 10am repository_dispatch: # Respond to rebuild requests. See: https://github.com/cisagov/action-apb/ types: - apb + schedule: + - cron: "0 10 * * *" # everyday at 10am workflow_dispatch: inputs: - remote-shell: - description: "Debug with remote shell" - required: true - default: "false" image-tag: + default: "dispatch" description: "Tag to apply to pushed images" required: true - default: "dispatch" + remote-shell: + default: "false" + description: "Debug with remote shell" + required: true # Set a default shell for any run steps. The `-Eueo pipefail` sets errtrace, # nounset, errexit, and pipefail. The `-x` will print all commands as they are @@ -324,8 +324,8 @@ jobs: env: BASE_CACHE_KEY: buildx-${{ runner.os }}- with: - path: ${{ env.BUILDX_CACHE_DIR }} key: ${{ env.BASE_CACHE_KEY }}${{ github.sha }} + path: ${{ env.BUILDX_CACHE_DIR }} restore-keys: | ${{ env.BASE_CACHE_KEY }} - name: Create dist directory @@ -340,15 +340,6 @@ jobs: cache-to: type=local,dest=${{ env.BUILDX_CACHE_DIR }} context: . file: ./Dockerfile - outputs: type=docker,dest=dist/image.tar - # Uncomment the following option if you are building an image for use - # on Google Cloud Run or AWS Lambda. The current default image output - # is unable to run on either. Please see the following issue for more - # information: https://github.com/docker/buildx/issues/1533 - # provenance: false - tags: ${{ env.IMAGE_NAME }}:latest # not to be pushed - # For a list of pre-defined annotation keys and value types see: - # https://github.com/opencontainers/image-spec/blob/master/annotations.md labels: "\ org.opencontainers.image.created=${{ needs.prepare.outputs.created }} @@ -372,6 +363,15 @@ jobs: org.opencontainers.image.version=${{ needs.prepare.outputs.source_version }}" + outputs: type=docker,dest=dist/image.tar + # Uncomment the following option if you are building an image for use + # on Google Cloud Run or AWS Lambda. The current default image output + # is unable to run on either. Please see the following issue for more + # information: https://github.com/docker/buildx/issues/1533 + # provenance: false + tags: ${{ env.IMAGE_NAME }}:latest # not to be pushed + # For a list of pre-defined annotation keys and value types see: + # https://github.com/opencontainers/image-spec/blob/master/annotations.md - name: Compress image run: gzip dist/image.tar - name: Upload artifacts @@ -406,10 +406,10 @@ jobs: BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ py${{ steps.setup-python.outputs.python-version }}-" with: - path: ${{ env.PIP_CACHE_DIR }} key: "${{ env.BASE_CACHE_KEY }}\ ${{ hashFiles('**/requirements-test.txt') }}-\ ${{ hashFiles('**/requirements.txt') }}" + path: ${{ env.PIP_CACHE_DIR }} restore-keys: | ${{ env.BASE_CACHE_KEY }} - name: Install dependencies @@ -458,14 +458,14 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v3 with: - username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} + username: ${{ secrets.DOCKER_USERNAME }} - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: + password: ${{ secrets.GITHUB_TOKEN }} registry: ghcr.io username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - name: Checkout uses: actions/checkout@v4 - name: Set up QEMU @@ -477,8 +477,8 @@ jobs: env: BASE_CACHE_KEY: buildx-${{ runner.os }}- with: - path: ${{ env.BUILDX_CACHE_DIR }} key: ${{ env.BASE_CACHE_KEY }}${{ github.sha }} + path: ${{ env.BUILDX_CACHE_DIR }} restore-keys: | ${{ env.BASE_CACHE_KEY }} - name: Create cross-platform support Dockerfile-x @@ -493,16 +493,6 @@ jobs: cache-to: type=local,dest=${{ env.BUILDX_CACHE_DIR }} context: . file: ./Dockerfile-x - platforms: ${{ env.PLATFORMS }} - # Uncomment the following option if you are building an image for use - # on Google Cloud Run or AWS Lambda. The current default image output - # is unable to run on either. Please see the following issue for more - # information: https://github.com/docker/buildx/issues/1533 - # provenance: false - push: true - tags: ${{ needs.prepare.outputs.tags }} - # For a list of pre-defined annotation keys and value types see: - # https://github.com/opencontainers/image-spec/blob/master/annotations.md labels: "\ org.opencontainers.image.created=${{ needs.prepare.outputs.created }} @@ -526,6 +516,16 @@ jobs: org.opencontainers.image.version=${{ needs.prepare.outputs.source_version }}" + platforms: ${{ env.PLATFORMS }} + # Uncomment the following option if you are building an image for use + # on Google Cloud Run or AWS Lambda. The current default image output + # is unable to run on either. Please see the following issue for more + # information: https://github.com/docker/buildx/issues/1533 + # provenance: false + push: true + tags: ${{ needs.prepare.outputs.tags }} + # For a list of pre-defined annotation keys and value types see: + # https://github.com/opencontainers/image-spec/blob/master/annotations.md - name: Publish README.md to Docker Hub env: DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} From 458bbf5a874dfcffc9a40217b1bf81bba98d01cf Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 21 Nov 2024 17:31:33 -0500 Subject: [PATCH 277/326] Alphabetize entries in the CodeQL workflow We prefer to alphabetize mapping keys in YAML documents whenever possible. A comment was updated to reflect the change in order. --- .github/workflows/codeql-analysis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 382d3d1..2fdf6e9 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -7,15 +7,15 @@ name: CodeQL on: + pull_request: + # The branches here must be a subset of the ones in the push key + branches: + - develop push: # Dependabot triggered push events have read-only access, but uploading code # scanning requires write access. branches-ignore: - dependabot/** - pull_request: - # The branches below must be a subset of the branches above - branches: - - develop schedule: - cron: '0 21 * * 6' From 9342f0cd6709d53e3dd55cf9b53ee761bc4fdffd Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 21 Nov 2024 17:33:56 -0500 Subject: [PATCH 278/326] Add a merge_group trigger to the CodeQL workflow This should improve compatibility with merge queues. We configure it to only trigger on the `checks_requested` type which is currently the only supported type for this trigger. If additional types are added in the future they should be added if appropriate. --- .github/workflows/codeql-analysis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 2fdf6e9..f0df652 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -7,6 +7,9 @@ name: CodeQL on: + merge_group: + types: + - checks_requested pull_request: # The branches here must be a subset of the ones in the push key branches: From 7f04a42ba010435f6d0a0de763692bc4f034d238 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 21 Nov 2024 17:36:09 -0500 Subject: [PATCH 279/326] Monitor GITHUB_TOKEN permissions for all jobs in the build workflow This ensures the GitHubSecurityLab/actions-permissions/monitor Action is used in every job in the workflow. --- .github/workflows/build.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 53f65cf..aa071dc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -243,6 +243,10 @@ jobs: tags: ${{ steps.prep.outputs.tags }} runs-on: ubuntu-latest steps: + - uses: GitHubSecurityLab/actions-permissions/monitor@v1 + with: + # Uses the organization variable unless overridden + config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} - id: harden-runner name: Harden the runner uses: step-security/harden-runner@v2 @@ -308,6 +312,10 @@ jobs: - prepare runs-on: ubuntu-latest steps: + - uses: GitHubSecurityLab/actions-permissions/monitor@v1 + with: + # Uses the organization variable unless overridden + config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} - id: harden-runner name: Harden the runner uses: step-security/harden-runner@v2 @@ -390,6 +398,10 @@ jobs: - build runs-on: ubuntu-latest steps: + - uses: GitHubSecurityLab/actions-permissions/monitor@v1 + with: + # Uses the organization variable unless overridden + config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} - id: harden-runner name: Harden the runner uses: step-security/harden-runner@v2 @@ -450,6 +462,10 @@ jobs: packages: write runs-on: ubuntu-latest steps: + - uses: GitHubSecurityLab/actions-permissions/monitor@v1 + with: + # Uses the organization variable unless overridden + config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} - id: harden-runner name: Harden the runner uses: step-security/harden-runner@v2 From b9639f279ddd273bdcfc8f2be1e9f9f17ef8de9d Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 21 Nov 2024 17:41:43 -0500 Subject: [PATCH 280/326] Monitor GITHUB_TOKEN permissions for all jobs in the CodeQL workflow This ensures the GitHubSecurityLab/actions-permissions/monitor Action is used in every job in the workflow. --- .github/workflows/codeql-analysis.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index f0df652..3844efc 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -29,6 +29,12 @@ jobs: steps: # Note that a duplicate of this step must be added at the top of # each job. + - uses: GitHubSecurityLab/actions-permissions/monitor@v1 + with: + # Uses the organization variable unless overridden + config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} + # Note that a duplicate of this step must be added at the top of + # each job. - id: harden-runner name: Harden the runner uses: step-security/harden-runner@v2 @@ -59,6 +65,10 @@ jobs: # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection steps: + - uses: GitHubSecurityLab/actions-permissions/monitor@v1 + with: + # Uses the organization variable unless overridden + config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} - id: harden-runner name: Harden the runner uses: step-security/harden-runner@v2 From 805d0fd2569f8fdae405b95bdd51f9bd91b09e17 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 25 Sep 2024 09:49:19 -0400 Subject: [PATCH 281/326] Upgrade bandit to 1.7.10 We should use the same version of bandit throughout the pre-commit configuration. --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a4c10db..4d0c4a9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -133,7 +133,7 @@ repos: - --config=.bandit.yml # Run bandit on everything except the "tests" tree - repo: https://github.com/PyCQA/bandit - rev: 1.7.5 + rev: 1.7.10 hooks: - id: bandit name: bandit (everything else) From 05169bcf069a715ffa007c79457a587b146ebe6d Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 25 Sep 2024 09:50:10 -0400 Subject: [PATCH 282/326] Add whitespace to placate black pre-commit hook --- tests/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/conftest.py b/tests/conftest.py index 054f09d..29a8b92 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,6 +2,7 @@ https://docs.pytest.org/en/latest/writing_plugins.html#conftest-py-plugins """ + # Third-Party Libraries import pytest from python_on_whales import docker From 0bfb66241cb555b1e1b32adf53b7068aca32949a Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 22 Nov 2024 04:04:24 -0500 Subject: [PATCH 283/326] Remove needless shebang --- tests/container_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/container_test.py b/tests/container_test.py index c2e1874..00c8e82 100644 --- a/tests/container_test.py +++ b/tests/container_test.py @@ -1,4 +1,3 @@ -#!/usr/bin/env pytest -vs """Tests for example container.""" # Standard Python Libraries From f5dd0463743633e9e8f202a2dddfc1c9e522688d Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 22 Nov 2024 04:10:14 -0500 Subject: [PATCH 284/326] Add missing use of the cisagov/setup-env-github-action Action Now that the actions/setup-python Action in the test job expects the Python version to be supplied by cisagov/setup-env-github-action we need to make sure the Action is run in the job. --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index aa071dc..eb883ea 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -407,6 +407,8 @@ jobs: uses: step-security/harden-runner@v2 with: egress-policy: audit + - id: setup-env + uses: cisagov/setup-env-github-action@develop - uses: actions/checkout@v4 - id: setup-python uses: actions/setup-python@v5 From ee5a0af9c7ddadb1d22c9529a540b37159b1b4d3 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 22 Nov 2024 04:29:48 -0500 Subject: [PATCH 285/326] Change the port used as an example in the Docker composition Since no actual networking is going on inside the image this change is non-destructive. I am making it because we are now using the GitHubSecurityLab/actions-permissions/monitor Action and the proxy it configures uses port 8080. There are currently no options for configuring this in the Action and the easiest path forward in this case is to simply change the port number used in the Docker composition. --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index e746ae9..fa067b7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,8 +21,8 @@ services: environment: - ECHO_MESSAGE=Hello World from docker compose! ports: - - target: 8080 - published: 8080 + - target: 8421 + published: 8421 protocol: tcp mode: host secrets: From b342704ebf0892e198332048fdd2de9d13d9aca3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 22 Nov 2024 18:29:48 +0000 Subject: [PATCH 286/326] Bump docker/build-push-action from 5 to 6 Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 5 to 6. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v5...v6) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d8ba132..56c9a79 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -304,7 +304,7 @@ jobs: run: mkdir -p dist - name: Build image id: docker_build - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: build-args: | VERSION=${{ needs.prepare.outputs.source_version }} @@ -457,7 +457,7 @@ jobs: run: ./buildx-dockerfile.sh - name: Build and push platform images to registries id: docker_build - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: build-args: | VERSION=${{ needs.prepare.outputs.source_version }} From 26360aaca08a23c5738d96abb5527c4907d75534 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 22 Nov 2024 13:31:43 -0500 Subject: [PATCH 287/326] Update job permissions in the `build` workflow Adjust permissions based on the output of the GitHubSecurityLab/actions-permissions/monitor Action. We have been explicitly declaring the `read` permission for `content` if a job uses actions/checkout. --- .github/workflows/build.yml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index eb883ea..6edf624 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -241,6 +241,9 @@ jobs: repometa: ${{ steps.repo.outputs.result }} source_version: ${{ steps.prep.outputs.source_version }} tags: ${{ steps.prep.outputs.tags }} + permissions: + # actions/checkout needs this to fetch code + contents: read runs-on: ubuntu-latest steps: - uses: GitHubSecurityLab/actions-permissions/monitor@v1 @@ -310,6 +313,9 @@ jobs: needs: - diagnostics - prepare + permissions: + # actions/checkout needs this to fetch code + contents: read runs-on: ubuntu-latest steps: - uses: GitHubSecurityLab/actions-permissions/monitor@v1 @@ -396,6 +402,9 @@ jobs: needs: - diagnostics - build + permissions: + # actions/checkout needs this to fetch code + contents: read runs-on: ubuntu-latest steps: - uses: GitHubSecurityLab/actions-permissions/monitor@v1 @@ -458,9 +467,11 @@ jobs: - lint - prepare - test - # When Dependabot creates a PR it requires this permission in - # order to push Docker images to ghcr.io. permissions: + # actions/checkout needs this to fetch code + contents: read + # When Dependabot creates a PR it requires this permission in + # order to push Docker images to ghcr.io. packages: write runs-on: ubuntu-latest steps: From 77f9251baf3a3d0af3cdc4e065e21c88fff9021c Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 22 Nov 2024 13:34:45 -0500 Subject: [PATCH 288/326] Update job permissions in the `CodeQL` workflow Adjust permissions based on the output of the GitHubSecurityLab/actions-permissions/monitor Action. We have been explicitly declaring the `read` permission for `content` if a job uses actions/checkout. --- .github/workflows/codeql-analysis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 3844efc..19439d9 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,6 +25,8 @@ on: jobs: diagnostics: name: Run diagnostics + # This job does not need any permissions + permissions: {} runs-on: ubuntu-latest steps: # Note that a duplicate of this step must be added at the top of @@ -52,6 +54,8 @@ jobs: - diagnostics runs-on: ubuntu-latest permissions: + # actions/checkout needs this to fetch code + contents: read # required for all workflows security-events: write strategy: From cd1ee1d8bf9af0c47fb609193514e66cb2e6a98a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Dec 2023 17:31:21 +0000 Subject: [PATCH 289/326] Bump actions/download-artifact from 3 to 4 Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 3 to 4. - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/download-artifact dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d8ba132..3bbdb1b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -389,7 +389,7 @@ jobs: python -m pip install --upgrade pip setuptools wheel pip install --upgrade --requirement requirements-test.txt - name: Download docker image artifact - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: dist path: dist From 18815c74096728cf40a12b82eef25cd14b0e3d17 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Dec 2023 17:31:13 +0000 Subject: [PATCH 290/326] Bump actions/upload-artifact from 3 to 4 Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 3 to 4. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3bbdb1b..9e79e04 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -347,7 +347,7 @@ jobs: - name: Compress image run: gzip dist/image.tar - name: Upload artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: dist path: dist From 03e668f4ca285e5868d7513dc8137c69883f9d58 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 6 Dec 2024 04:39:31 -0500 Subject: [PATCH 291/326] Change the format of the version tracking file There is no reason to have any information but the version in the version tracking file for Docker image projects. The current format is simply an artifact of mirroring things over from the cisagov/skeleton-python-library project. --- bump-version | 2 +- src/version.txt | 2 +- tests/container_test.py | 12 +++--------- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/bump-version b/bump-version index 600265b..414552a 100755 --- a/bump-version +++ b/bump-version @@ -29,7 +29,7 @@ Options: END_OF_LINE ) -old_version=$(sed -n "s/^__version__ = \"\(.*\)\"$/\1/p" $VERSION_FILE) +old_version=$(< "$VERSION_FILE") # Comment out periods so they are interpreted as periods and don't # just match any character old_version_regex=${old_version//\./\\\.} diff --git a/src/version.txt b/src/version.txt index f102a9c..8acdd82 100644 --- a/src/version.txt +++ b/src/version.txt @@ -1 +1 @@ -__version__ = "0.0.1" +0.0.1 diff --git a/tests/container_test.py b/tests/container_test.py index 00c8e82..f58d325 100644 --- a/tests/container_test.py +++ b/tests/container_test.py @@ -62,10 +62,8 @@ def test_output(dockerc, main_container): ) def test_release_version(): """Verify that release tag version agrees with the module version.""" - pkg_vars = {} with open(VERSION_FILE) as f: - exec(f.read(), pkg_vars) # nosec - project_version = pkg_vars["__version__"] + project_version = f.read().strip() assert ( RELEASE_TAG == f"v{project_version}" ), "RELEASE_TAG does not match the project version" @@ -76,10 +74,8 @@ def test_log_version(dockerc, version_container): # make sure container exited if running test isolated dockerc.wait(version_container.id) log_output = version_container.logs().strip() - pkg_vars = {} with open(VERSION_FILE) as f: - exec(f.read(), pkg_vars) # nosec - project_version = pkg_vars["__version__"] + project_version = f.read().strip() assert ( log_output == project_version ), f"Container version output to log does not match project version file {VERSION_FILE}" @@ -87,10 +83,8 @@ def test_log_version(dockerc, version_container): def test_container_version_label_matches(version_container): """Verify the container version label is the correct version.""" - pkg_vars = {} with open(VERSION_FILE) as f: - exec(f.read(), pkg_vars) # nosec - project_version = pkg_vars["__version__"] + project_version = f.read().strip() assert ( version_container.config.labels["org.opencontainers.image.version"] == project_version From 414efb4c4e60c86083d416bdfd8e9c92a9898b0c Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 6 Dec 2024 10:22:06 -0500 Subject: [PATCH 292/326] DRY out getting the project version in tests We were previously getting the version in three different tests. It makes more sense to turn getting the project version into a fixture and to use that fixture anywhere the project version is needed. --- tests/conftest.py | 10 ++++++++++ tests/container_test.py | 12 +++--------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 29a8b92..6092a48 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,6 +10,8 @@ MAIN_SERVICE_NAME = "example" VERSION_SERVICE_NAME = f"{MAIN_SERVICE_NAME}-version" +VERSION_FILE = "src/version.txt" + @pytest.fixture(scope="session") def dockerc(): @@ -36,6 +38,14 @@ def version_container(dockerc): return dockerc.compose.ps(services=[VERSION_SERVICE_NAME], all=True)[0] +@pytest.fixture(scope="session") +def project_version(): + """Return the version of the project.""" + with open(VERSION_FILE) as f: + project_version = f.read().strip() + return project_version + + def pytest_addoption(parser): """Add new commandline options to pytest.""" parser.addoption( diff --git a/tests/container_test.py b/tests/container_test.py index f58d325..cf18333 100644 --- a/tests/container_test.py +++ b/tests/container_test.py @@ -60,31 +60,25 @@ def test_output(dockerc, main_container): @pytest.mark.skipif( RELEASE_TAG in [None, ""], reason="this is not a release (RELEASE_TAG not set)" ) -def test_release_version(): +def test_release_version(project_version): """Verify that release tag version agrees with the module version.""" - with open(VERSION_FILE) as f: - project_version = f.read().strip() assert ( RELEASE_TAG == f"v{project_version}" ), "RELEASE_TAG does not match the project version" -def test_log_version(dockerc, version_container): +def test_log_version(dockerc, project_version, version_container): """Verify the container outputs the correct version to the logs.""" # make sure container exited if running test isolated dockerc.wait(version_container.id) log_output = version_container.logs().strip() - with open(VERSION_FILE) as f: - project_version = f.read().strip() assert ( log_output == project_version ), f"Container version output to log does not match project version file {VERSION_FILE}" -def test_container_version_label_matches(version_container): +def test_container_version_label_matches(project_version, version_container): """Verify the container version label is the correct version.""" - with open(VERSION_FILE) as f: - project_version = f.read().strip() assert ( version_container.config.labels["org.opencontainers.image.version"] == project_version From b6b0b7c27b9c1e12f321dc16b89acbf90c00af19 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 2 Feb 2024 05:10:50 -0500 Subject: [PATCH 293/326] Use the full identifier for source Docker images This helps ensure that when a Docker image is built the expacted source image is used regardless of what repository is configured as the default on the host system. It also makes our Dockerfiles more seamlessly convertible to using the GitHub Container Registry or any other Open Container Initiative (OCI) compatible registry. --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index fc7b5c2..e4a610d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,8 @@ ARG VERSION=unspecified -FROM python:3.12.0-alpine +# Official Docker images are in the form library/ while non-official +# images are in the form /. +FROM docker.io/library/python:3.12.0-alpine ARG VERSION From 3b7a9ccdf02878c1d68c2fa8a26db02df0ba3b8c Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 20 Feb 2024 10:02:56 -0500 Subject: [PATCH 294/326] Install cisagov/skeleton-python-library directly Instead of downloading the source archive, extracting it, and then installing it with pip we instead just let pip directly install the package. --- Dockerfile | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9c939b7..7492a54 100644 --- a/Dockerfile +++ b/Dockerfile @@ -79,12 +79,7 @@ WORKDIR ${CISA_HOME} # cache. This results in a smaller final image, at the cost of # slightly longer install times. ### -RUN wget --output-document sourcecode.tgz \ - https://github.com/cisagov/skeleton-python-library/archive/v${VERSION}.tar.gz \ - && tar --extract --gzip --file sourcecode.tgz --strip-components=1 \ - && pip3 install --no-cache-dir --requirement requirements.txt \ - && ln -snf /run/secrets/quote.txt src/example/data/secret.txt \ - && rm sourcecode.tgz +RUN pip3 install --no-cache-dir https://github.com/cisagov/skeleton-python-library/archive/v${VERSION}.tar.gz ### # Prepare to run From db1970697171c6e530b8328f79071b022c14c848 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 13 Feb 2024 05:05:50 -0500 Subject: [PATCH 295/326] Use a specific version of Alpine Linux Use the full tag that includes the Alpine Linux version to ensure the pulled image is always the same. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e4a610d..9c939b7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ ARG VERSION=unspecified # Official Docker images are in the form library/ while non-official # images are in the form /. -FROM docker.io/library/python:3.12.0-alpine +FROM docker.io/library/python:3.12.0-alpine3.18 ARG VERSION From 9e6eef290a6dc61f0cb95357383770c635dec4c3 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 20 Feb 2024 10:05:26 -0500 Subject: [PATCH 296/326] Remove unused OS package dependencies Since we are now installing cisagov/skeleton-python-library directly with pip we no longer need these OS packages. --- Dockerfile | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7492a54..ce1aa88 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,19 +45,6 @@ RUN apk --update --no-cache --quiet upgrade RUN addgroup --system --gid ${CISA_GID} ${CISA_GROUP} \ && adduser --system --uid ${CISA_UID} --ingroup ${CISA_GROUP} ${CISA_USER} -### -# Dependencies -# -# Note that we use apk --no-cache to avoid writing to a local cache. -# This results in a smaller final image, at the cost of slightly -# longer install times. -### -ENV DEPS \ - ca-certificates \ - openssl \ - py-pip -RUN apk --no-cache --quiet add ${DEPS} - ### # Make sure pip, setuptools, and wheel are the latest versions # From c516e44a361bdb3ddf285203bfe487e2c9f94ef6 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 20 Feb 2024 13:16:14 -0500 Subject: [PATCH 297/326] Remove package upgrading We should not blindly upgrade all pre-installed packages. This can create inconsistent build results due to changes in installed versions. --- Dockerfile | 9 --------- 1 file changed, 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index ce1aa88..296d03d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,15 +30,6 @@ ARG CISA_USER="cisa" ENV CISA_GROUP=${CISA_USER} ENV CISA_HOME="/home/${CISA_USER}" -### -# Upgrade the system -# -# Note that we use apk --no-cache to avoid writing to a local cache. -# This results in a smaller final image, at the cost of slightly -# longer install times. -### -RUN apk --update --no-cache --quiet upgrade - ### # Create unprivileged user ### From 460eeec5f3861847966f525d6388adbfed8fc49d Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 20 Feb 2024 10:11:44 -0500 Subject: [PATCH 298/326] Change the secret message being checks in tests Now that we are not overwriting the internal Python package file the text we look for must match what is output by default. The Docker Compose secret configuration is left in place to continue to serve as an example and to be leveraged for a future update to cisagov/skeleton-python-library that can provide similar functionality to what was removed in this project. --- tests/container_test.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/container_test.py b/tests/container_test.py index cf18333..6451766 100644 --- a/tests/container_test.py +++ b/tests/container_test.py @@ -10,9 +10,7 @@ ENV_VAR = "ECHO_MESSAGE" ENV_VAR_VAL = "Hello World from docker compose!" READY_MESSAGE = "This is a debug message" -SECRET_QUOTE = ( - "There are no secrets better kept than the secrets everybody guesses." # nosec -) +SECRET_QUOTE = "Three may keep a secret, if two of them are dead." # nosec RELEASE_TAG = os.getenv("RELEASE_TAG") VERSION_FILE = "src/version.txt" From 001e85e96753b00cfc4909e691ee29930b834787 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 20 Feb 2024 13:21:51 -0500 Subject: [PATCH 299/326] Pin Python packages directly installed Pin the versions of the pip, setuptools, and wheel packages that are installed. --- Dockerfile | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 296d03d..7ba2012 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,6 +30,11 @@ ARG CISA_USER="cisa" ENV CISA_GROUP=${CISA_USER} ENV CISA_HOME="/home/${CISA_USER}" +# Versions of the Python packages installed directly +ENV PYTHON_PIP_VERSION=24.0 +ENV PYTHON_SETUPTOOLS_VERSION=69.1.0 +ENV PYTHON_WHEEL_VERSION=0.42.0 + ### # Create unprivileged user ### @@ -37,16 +42,16 @@ RUN addgroup --system --gid ${CISA_GID} ${CISA_GROUP} \ && adduser --system --uid ${CISA_UID} --ingroup ${CISA_GROUP} ${CISA_USER} ### -# Make sure pip, setuptools, and wheel are the latest versions +# Make sure the specified versions of pip, setuptools, and wheel are installed # # Note that we use pip3 --no-cache-dir to avoid writing to a local # cache. This results in a smaller final image, at the cost of # slightly longer install times. ### RUN pip3 install --no-cache-dir --upgrade \ - pip \ - setuptools \ - wheel + pip==${PYTHON_PIP_VERSION} \ + setuptools==${PYTHON_SETUPTOOLS_VERSION} \ + wheel==${PYTHON_WHEEL_VERSION} WORKDIR ${CISA_HOME} From 48fa1a5c61a0ef05db5bc352b68bd3d8cdcd2c94 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 26 Feb 2024 17:02:05 -0500 Subject: [PATCH 300/326] Prefer calling pip as a module Instead of relying on `pip3` being on the PATH we instead call the module through the Python executable. This ensures that the `pip` being used is in the same environment as the `python3` being used. --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index ca97b41..b07e712 100644 --- a/Dockerfile +++ b/Dockerfile @@ -44,11 +44,11 @@ RUN addgroup --system --gid ${CISA_GID} ${CISA_GROUP} \ ### # Make sure the specified versions of pip, setuptools, and wheel are installed # -# Note that we use pip3 --no-cache-dir to avoid writing to a local +# Note that we use the --no-cache-dir flag to avoid writing to a local # cache. This results in a smaller final image, at the cost of # slightly longer install times. ### -RUN pip3 install --no-cache-dir --upgrade \ +RUN python3 -m pip install --no-cache-dir --upgrade \ pip==${PYTHON_PIP_VERSION} \ setuptools==${PYTHON_SETUPTOOLS_VERSION} \ wheel==${PYTHON_WHEEL_VERSION} @@ -56,11 +56,11 @@ RUN pip3 install --no-cache-dir --upgrade \ ### # Install Python dependencies # -# Note that we use pip3 --no-cache-dir to avoid writing to a local +# Note that we use the --no-cache-dir flag to avoid writing to a local # cache. This results in a smaller final image, at the cost of # slightly longer install times. ### -RUN pip3 install --no-cache-dir https://github.com/cisagov/skeleton-python-library/archive/v${VERSION}.tar.gz +RUN python3 -m pip install --no-cache-dir https://github.com/cisagov/skeleton-python-library/archive/v${VERSION}.tar.gz ### # Prepare to run From e6f5798b9bdbe140aab760f0cedee7ee01324907 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 23 Feb 2024 13:14:34 -0500 Subject: [PATCH 301/326] Move WORKDIR instruction We can move this instruction to the end of the Dockerfile now that we are no longer working with files in the Docker container when building. --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7ba2012..ca97b41 100644 --- a/Dockerfile +++ b/Dockerfile @@ -53,8 +53,6 @@ RUN pip3 install --no-cache-dir --upgrade \ setuptools==${PYTHON_SETUPTOOLS_VERSION} \ wheel==${PYTHON_WHEEL_VERSION} -WORKDIR ${CISA_HOME} - ### # Install Python dependencies # @@ -68,6 +66,7 @@ RUN pip3 install --no-cache-dir https://github.com/cisagov/skeleton-python-libra # Prepare to run ### ENV ECHO_MESSAGE="Hello World from Dockerfile" +WORKDIR ${CISA_HOME} USER ${CISA_USER}:${CISA_GROUP} EXPOSE 8080/TCP VOLUME ["/var/log"] From 95d4a7a6f49a61216547f9f49c363f847bf99107 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 26 Feb 2024 17:13:30 -0500 Subject: [PATCH 302/326] Use a Python virtual environment in the Docker image Using a virtual environment is a Python best practice. We also consolidate all of the Python dependency installation steps into a single RUN instruction. This ensures that Python setup is cached in one layer and mirrors the logical organization of this being a single step. --- Dockerfile | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index b07e712..7416591 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,6 +29,7 @@ ARG CISA_GID=${CISA_UID} ARG CISA_USER="cisa" ENV CISA_GROUP=${CISA_USER} ENV CISA_HOME="/home/${CISA_USER}" +ENV VIRTUAL_ENV="${CISA_HOME}/.venv" # Versions of the Python packages installed directly ENV PYTHON_PIP_VERSION=24.0 @@ -42,25 +43,32 @@ RUN addgroup --system --gid ${CISA_GID} ${CISA_GROUP} \ && adduser --system --uid ${CISA_UID} --ingroup ${CISA_GROUP} ${CISA_USER} ### -# Make sure the specified versions of pip, setuptools, and wheel are installed +# Set up a Python virtual environment (venv); install the specified versions of pip, +# setuptools, and wheel into it; and then install the Python dependencies for +# the application. # # Note that we use the --no-cache-dir flag to avoid writing to a local # cache. This results in a smaller final image, at the cost of # slightly longer install times. ### -RUN python3 -m pip install --no-cache-dir --upgrade \ - pip==${PYTHON_PIP_VERSION} \ - setuptools==${PYTHON_SETUPTOOLS_VERSION} \ - wheel==${PYTHON_WHEEL_VERSION} +RUN python3 -m venv ${VIRTUAL_ENV} \ + && ${VIRTUAL_ENV}/bin/python3 -m pip install --no-cache-dir --upgrade \ + pip==${PYTHON_PIP_VERSION} \ + setuptools==${PYTHON_SETUPTOOLS_VERSION} \ + wheel==${PYTHON_WHEEL_VERSION} \ + && ${VIRTUAL_ENV}/bin/python3 -m pip install --no-cache-dir --upgrade \ + https://github.com/cisagov/skeleton-python-library/archive/v${VERSION}.tar.gz ### -# Install Python dependencies +# Sym-link the Python binary in the venv to the system-wide Python and add the venv to +# the PATH. # -# Note that we use the --no-cache-dir flag to avoid writing to a local -# cache. This results in a smaller final image, at the cost of -# slightly longer install times. +# Note that we sym-link the Python binary in the venv to the system-wide Python so that +# any calls to `python3` will use our virtual environment. We are using short flags +# because the ln binary in Alpine Linux does not support long flags. ### -RUN python3 -m pip install --no-cache-dir https://github.com/cisagov/skeleton-python-library/archive/v${VERSION}.tar.gz +RUN ln -sf "$(command -v python3)" "${VIRTUAL_ENV}"/bin/python3 +ENV PATH="${VIRTUAL_ENV}/bin:$PATH" ### # Prepare to run From 79a4adf8a55a81c8360b1a49c55ea5fd91ebdc49 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 27 Feb 2024 18:12:50 -0500 Subject: [PATCH 303/326] Add a pipenv configuration This configuration includes a Pipfile configuration file and the generated Pipfile.lock file that pins to specific versions for the Python dependencies for this project. This will help us ensure repeatable builds. The pipenv package is added as a developmental requirement to support these files. --- requirements-dev.txt | 1 + src/Pipfile | 13 +++++++++++++ src/Pipfile.lock | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+) create mode 100644 src/Pipfile create mode 100644 src/Pipfile.lock diff --git a/requirements-dev.txt b/requirements-dev.txt index de5eb3b..d7a04ed 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,4 @@ --requirement requirements-test.txt ipython +pipenv semver>=3 diff --git a/src/Pipfile b/src/Pipfile new file mode 100644 index 0000000..56f2fc9 --- /dev/null +++ b/src/Pipfile @@ -0,0 +1,13 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +# List any Python dependencies for the image here +[packages] +# This should match the version of the image +example = {file = "https://github.com/cisagov/skeleton-python-library/archive/v0.0.1.tar.gz"} + +# This version should match the version of Python in the image +[requires] +python_full_version = "3.12.0" diff --git a/src/Pipfile.lock b/src/Pipfile.lock new file mode 100644 index 0000000..d39d053 --- /dev/null +++ b/src/Pipfile.lock @@ -0,0 +1,38 @@ +{ + "_meta": { + "hash": { + "sha256": "654452851fea1eb2c8811649e5efe8873c8ff51f5c14dd27a4a8ebb5b15a27c4" + }, + "pipfile-spec": 6, + "requires": { + "python_full_version": "3.12.0" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "docopt": { + "hashes": [ + "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491" + ], + "version": "==0.6.2" + }, + "example": { + "file": "https://github.com/cisagov/skeleton-python-library/archive/v0.0.1.tar.gz" + }, + "setuptools": { + "hashes": [ + "sha256:02fa291a0471b3a18b2b2481ed902af520c69e8ae0919c13da936542754b4c56", + "sha256:5c0806c7d9af348e6dd3777b4f4dbb42c7ad85b190104837488eab9a7c945cf8" + ], + "markers": "python_version >= '3.8'", + "version": "==69.1.1" + } + }, + "develop": {} +} From d3895efb85ba686d8d4bfac0bbf6f0a674aeb145 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 26 Feb 2024 22:16:54 -0500 Subject: [PATCH 304/326] Explain `ln` options being used Since we cannot use long options on Alpine Linux we should explain what the short options we are using do. I also changed the order of options so that they are in alphabetical order. Co-authored-by: Shane Frasier --- Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7416591..1aae8db 100644 --- a/Dockerfile +++ b/Dockerfile @@ -65,9 +65,10 @@ RUN python3 -m venv ${VIRTUAL_ENV} \ # # Note that we sym-link the Python binary in the venv to the system-wide Python so that # any calls to `python3` will use our virtual environment. We are using short flags -# because the ln binary in Alpine Linux does not support long flags. +# because the ln binary in Alpine Linux does not support long flags. The -f instructs +# ln to remove the existing file and the -s instructs ln to create a symbolic link. ### -RUN ln -sf "$(command -v python3)" "${VIRTUAL_ENV}"/bin/python3 +RUN ln -fs "$(command -v python3)" "${VIRTUAL_ENV}"/bin/python3 ENV PATH="${VIRTUAL_ENV}/bin:$PATH" ### From 6b869bb2ee9ec0423008973746a498305787d23a Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 27 Feb 2024 19:01:45 -0500 Subject: [PATCH 305/326] Install Python dependencies using pipenv Now that we have a pipenv configuration we will use it to install the Python dependencies for the image. The `build` workflow is updated to no longer pass the VERSION build argument in line with this change. --- .github/workflows/build.yml | 4 ---- Dockerfile | 34 +++++++++++++++++++++++----------- README.md | 2 -- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f643b86..612806d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -348,8 +348,6 @@ jobs: id: docker_build uses: docker/build-push-action@v6 with: - build-args: | - VERSION=${{ needs.prepare.outputs.source_version }} cache-from: type=local,src=${{ env.BUILDX_CACHE_DIR }} cache-to: type=local,dest=${{ env.BUILDX_CACHE_DIR }} context: . @@ -516,8 +514,6 @@ jobs: id: docker_build uses: docker/build-push-action@v6 with: - build-args: | - VERSION=${{ needs.prepare.outputs.source_version }} cache-from: type=local,src=${{ env.BUILDX_CACHE_DIR }} cache-to: type=local,dest=${{ env.BUILDX_CACHE_DIR }} context: . diff --git a/Dockerfile b/Dockerfile index 1aae8db..c36ae46 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,7 @@ -ARG VERSION=unspecified - # Official Docker images are in the form library/ while non-official # images are in the form /. FROM docker.io/library/python:3.12.0-alpine3.18 -ARG VERSION - ### # For a list of pre-defined annotation keys and value types see: # https://github.com/opencontainers/image-spec/blob/master/annotations.md @@ -33,6 +29,7 @@ ENV VIRTUAL_ENV="${CISA_HOME}/.venv" # Versions of the Python packages installed directly ENV PYTHON_PIP_VERSION=24.0 +ENV PYTHON_PIPENV_VERSION=2023.12.1 ENV PYTHON_SETUPTOOLS_VERSION=69.1.0 ENV PYTHON_WHEEL_VERSION=0.42.0 @@ -43,21 +40,36 @@ RUN addgroup --system --gid ${CISA_GID} ${CISA_GROUP} \ && adduser --system --uid ${CISA_UID} --ingroup ${CISA_GROUP} ${CISA_USER} ### -# Set up a Python virtual environment (venv); install the specified versions of pip, -# setuptools, and wheel into it; and then install the Python dependencies for -# the application. +# Install the specified version of pipenv; set up a Python virtual environment (venv); +# and install the specified versions of pip, setuptools, and wheel into the venv. # # Note that we use the --no-cache-dir flag to avoid writing to a local # cache. This results in a smaller final image, at the cost of # slightly longer install times. ### -RUN python3 -m venv ${VIRTUAL_ENV} \ +RUN python3 -m pip install --no-cache-dir --upgrade pipenv==${PYTHON_PIPENV_VERSION} \ + # Manueally create the virtual environment + && python3 -m venv ${VIRTUAL_ENV} \ + # Ensure the core Python packages are installed in the virtual environment && ${VIRTUAL_ENV}/bin/python3 -m pip install --no-cache-dir --upgrade \ pip==${PYTHON_PIP_VERSION} \ setuptools==${PYTHON_SETUPTOOLS_VERSION} \ - wheel==${PYTHON_WHEEL_VERSION} \ - && ${VIRTUAL_ENV}/bin/python3 -m pip install --no-cache-dir --upgrade \ - https://github.com/cisagov/skeleton-python-library/archive/v${VERSION}.tar.gz + wheel==${PYTHON_WHEEL_VERSION} + +### +# Check the Pipfile configuration and then install the Python dependencies into +# the virtual environment. +# +# Note that pipenv will install into a virtual environment if the VIRTUAL_ENV +# environment variable is set. We are using short flags because the rm binary +# in Alpine Linux does not support long flags. The -f instructs rm to remove +# files without prompting. +### +WORKDIR /tmp +COPY src/Pipfile src/Pipfile.lock ./ +RUN pipenv check --verbose \ + && pipenv install --clear --deploy --extra-pip-args "--no-cache-dir" --verbose \ + && rm -f Pipfile* ### # Sym-link the Python binary in the venv to the system-wide Python and add the venv to diff --git a/README.md b/README.md index 57f8c30..8d926e4 100644 --- a/README.md +++ b/README.md @@ -196,7 +196,6 @@ Build the image locally using this git repository as the [build context](https:/ ```console docker build \ - --build-arg VERSION=0.0.1 \ --tag cisagov/example:0.0.1 \ https://github.com/cisagov/example.git#develop ``` @@ -227,7 +226,6 @@ Docker: docker buildx build \ --file Dockerfile-x \ --platform linux/amd64 \ - --build-arg VERSION=0.0.1 \ --output type=docker \ --tag cisagov/example:0.0.1 . ``` From 5cbb75beff797cc24c469cb24941d2541473923b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 27 Feb 2024 19:09:29 -0500 Subject: [PATCH 306/326] Use a multi-stage Docker build Switch to using a multi-stage build in the Dockerfile. This reduces image size since pipenv and its dependencices are not needed in the final image. It also ensures that the system Python environment is unmodified. --- Dockerfile | 71 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 41 insertions(+), 30 deletions(-) diff --git a/Dockerfile b/Dockerfile index c36ae46..ca0742e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,29 +1,11 @@ # Official Docker images are in the form library/ while non-official # images are in the form /. -FROM docker.io/library/python:3.12.0-alpine3.18 +FROM docker.io/library/python:3.12.0-alpine3.18 as compile-stage ### -# For a list of pre-defined annotation keys and value types see: -# https://github.com/opencontainers/image-spec/blob/master/annotations.md -# -# Note: Additional labels are added by the build workflow. -### -# github@cisa.dhs.gov is a very generic email distribution, and it is -# unlikely that anyone on that distribution is familiar with the -# particulars of your repository. It is therefore *strongly* -# suggested that you use an email address here that is specific to the -# person or group that maintains this repository; for example: -# LABEL org.opencontainers.image.authors="vm-fusion-dev-group@trio.dhs.gov" -LABEL org.opencontainers.image.authors="github@cisa.dhs.gov" -LABEL org.opencontainers.image.vendor="Cybersecurity and Infrastructure Security Agency" - -### -# Unprivileged user setup variables +# Unprivileged user variables ### -ARG CISA_UID=421 -ARG CISA_GID=${CISA_UID} ARG CISA_USER="cisa" -ENV CISA_GROUP=${CISA_USER} ENV CISA_HOME="/home/${CISA_USER}" ENV VIRTUAL_ENV="${CISA_HOME}/.venv" @@ -33,12 +15,6 @@ ENV PYTHON_PIPENV_VERSION=2023.12.1 ENV PYTHON_SETUPTOOLS_VERSION=69.1.0 ENV PYTHON_WHEEL_VERSION=0.42.0 -### -# Create unprivileged user -### -RUN addgroup --system --gid ${CISA_GID} ${CISA_GROUP} \ - && adduser --system --uid ${CISA_UID} --ingroup ${CISA_GROUP} ${CISA_USER} - ### # Install the specified version of pipenv; set up a Python virtual environment (venv); # and install the specified versions of pip, setuptools, and wheel into the venv. @@ -68,18 +44,53 @@ RUN python3 -m pip install --no-cache-dir --upgrade pipenv==${PYTHON_PIPENV_VERS WORKDIR /tmp COPY src/Pipfile src/Pipfile.lock ./ RUN pipenv check --verbose \ - && pipenv install --clear --deploy --extra-pip-args "--no-cache-dir" --verbose \ - && rm -f Pipfile* + && pipenv install --clear --deploy --extra-pip-args "--no-cache-dir" --verbose + +# Official Docker images are in the form library/ while non-official +# images are in the form /. +FROM docker.io/library/python:3.12.0-alpine3.18 as build-stage + +### +# For a list of pre-defined annotation keys and value types see: +# https://github.com/opencontainers/image-spec/blob/master/annotations.md +# +# Note: Additional labels are added by the build workflow. +### +# github@cisa.dhs.gov is a very generic email distribution, and it is +# unlikely that anyone on that distribution is familiar with the +# particulars of your repository. It is therefore *strongly* +# suggested that you use an email address here that is specific to the +# person or group that maintains this repository; for example: +# LABEL org.opencontainers.image.authors="vm-fusion-dev-group@trio.dhs.gov" +LABEL org.opencontainers.image.authors="github@cisa.dhs.gov" +LABEL org.opencontainers.image.vendor="Cybersecurity and Infrastructure Security Agency" + +### +# Unprivileged user setup variables +### +ARG CISA_UID=421 +ARG CISA_GID=${CISA_UID} +ARG CISA_USER="cisa" +ENV CISA_GROUP=${CISA_USER} +ENV CISA_HOME="/home/${CISA_USER}" +ENV VIRTUAL_ENV="${CISA_HOME}/.venv" + +### +# Create unprivileged user +### +RUN addgroup --system --gid ${CISA_GID} ${CISA_GROUP} \ + && adduser --system --uid ${CISA_UID} --ingroup ${CISA_GROUP} ${CISA_USER} ### -# Sym-link the Python binary in the venv to the system-wide Python and add the venv to -# the PATH. +# Copy in the Python virtual environment created in compile-stage, Sym-link the +# Python binary in the venv to the system-wide Python and add the venv to the PATH. # # Note that we sym-link the Python binary in the venv to the system-wide Python so that # any calls to `python3` will use our virtual environment. We are using short flags # because the ln binary in Alpine Linux does not support long flags. The -f instructs # ln to remove the existing file and the -s instructs ln to create a symbolic link. ### +COPY --from=compile-stage --chown=${CISA_USER}:${CISA_GROUP} ${VIRTUAL_ENV} ${VIRTUAL_ENV} RUN ln -fs "$(command -v python3)" "${VIRTUAL_ENV}"/bin/python3 ENV PATH="${VIRTUAL_ENV}/bin:$PATH" From 5dca6d92e268ce2cf1e75efae4e04703d9ec35b4 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 27 Feb 2024 19:13:33 -0500 Subject: [PATCH 307/326] Install core Python packages into the system Python environment Install the core Python packages (pip, setuptools, and wheel) into the system Python environment before installing pipenv. This keeps things consistent with our usual approach to Python environments. --- Dockerfile | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index ca0742e..96a5126 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,14 +16,21 @@ ENV PYTHON_SETUPTOOLS_VERSION=69.1.0 ENV PYTHON_WHEEL_VERSION=0.42.0 ### -# Install the specified version of pipenv; set up a Python virtual environment (venv); -# and install the specified versions of pip, setuptools, and wheel into the venv. +# Install the specified versions of pip, setuptools, and wheel into the system +# Python environment; install the specified version of pipenv into the system Python +# environment; set up a Python virtual environment (venv); and install the specified +# versions of pip, setuptools, and wheel into the venv. # # Note that we use the --no-cache-dir flag to avoid writing to a local # cache. This results in a smaller final image, at the cost of # slightly longer install times. ### -RUN python3 -m pip install --no-cache-dir --upgrade pipenv==${PYTHON_PIPENV_VERSION} \ +RUN python3 -m pip install --no-cache-dir --upgrade \ + pip==${PYTHON_PIP_VERSION} \ + setuptools==${PYTHON_SETUPTOOLS_VERSION} \ + wheel==${PYTHON_WHEEL_VERSION} \ + && python3 -m pip install --no-cache-dir --upgrade \ + pipenv==${PYTHON_PIPENV_VERSION} \ # Manueally create the virtual environment && python3 -m venv ${VIRTUAL_ENV} \ # Ensure the core Python packages are installed in the virtual environment From 79d8de9ae144f57326b4b6a620a55545328e8a38 Mon Sep 17 00:00:00 2001 From: Nick <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 28 Feb 2024 12:04:09 -0500 Subject: [PATCH 308/326] Fix outdated comment in the Dockerfile The comment references a command that is no longer being run. Co-authored-by: Shane Frasier --- Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 96a5126..38cf0a4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -44,9 +44,7 @@ RUN python3 -m pip install --no-cache-dir --upgrade \ # the virtual environment. # # Note that pipenv will install into a virtual environment if the VIRTUAL_ENV -# environment variable is set. We are using short flags because the rm binary -# in Alpine Linux does not support long flags. The -f instructs rm to remove -# files without prompting. +# environment variable is set. ### WORKDIR /tmp COPY src/Pipfile src/Pipfile.lock ./ From 440ba709fe36c8a86d9880995b46a56f6741229d Mon Sep 17 00:00:00 2001 From: Nick <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 28 Feb 2024 12:04:51 -0500 Subject: [PATCH 309/326] Fix typo in Dockerfile comment Co-authored-by: dav3r --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 38cf0a4..6a00b36 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,7 +31,7 @@ RUN python3 -m pip install --no-cache-dir --upgrade \ wheel==${PYTHON_WHEEL_VERSION} \ && python3 -m pip install --no-cache-dir --upgrade \ pipenv==${PYTHON_PIPENV_VERSION} \ - # Manueally create the virtual environment + # Manually create the virtual environment && python3 -m venv ${VIRTUAL_ENV} \ # Ensure the core Python packages are installed in the virtual environment && ${VIRTUAL_ENV}/bin/python3 -m pip install --no-cache-dir --upgrade \ From 78f4f78b2047cd29c5edac51fe0fd218ceb296cb Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 29 Feb 2024 13:17:59 -0500 Subject: [PATCH 310/326] Update image tag information in the README Change the tags used in the table to match the version of the project. Previously "1.2.3" was used as an example version but there is no reason not to use the real version of the image. --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 8d926e4..7ff4648 100644 --- a/README.md +++ b/README.md @@ -139,9 +139,9 @@ containerize. It is recommended that most users use a version tag (e.g. | Image:tag | Description | |-----------|-------------| -|`cisagov/example:1.2.3`| An exact release version. | -|`cisagov/example:1.2`| The most recent release matching the major and minor version numbers. | -|`cisagov/example:1`| The most recent release matching the major version number. | +|`cisagov/example:0.0.1`| An exact release version. | +|`cisagov/example:0.0`| The most recent release matching the major and minor version numbers. | +|`cisagov/example:0`| The most recent release matching the major version number. | |`cisagov/example:edge` | The most recent image built from a merge into the `develop` branch of this repository. | |`cisagov/example:nightly` | A nightly build of the `develop` branch of this repository. | |`cisagov/example:latest`| The most recent release image pushed to a container registry. Pulling an image using the `:latest` tag [should be avoided.](https://vsupalov.com/docker-latest-tag/) | From 1774267011a35e29623648a9fa631e499c3fd36f Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 29 Feb 2024 15:15:07 -0500 Subject: [PATCH 311/326] Add instructions for managing Python dependencies --- README.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/README.md b/README.md index 7ff4648..a74bba4 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,35 @@ environment variables. See the 1. Recreate and run the container by following the [previous instructions](#running-with-docker). +## Updating Python dependencies ## + +This image uses [Pipenv] to manage Python dependencies using a [Pipfile](https://github.com/pypa/pipfile). +Both updating dependencies and changing the [Pipenv] configuration in `src/Pipfile` +will result in a modified `src/Pipfile.lock` file that should be committed to the +repository. + +> [!WARNING] +> The `src/Pipfile.lock` as generated will fail `pre-commit` checks due to JSON formatting. + +### Updating dependencies ### + +If you want to update existing dependencies you would run the following command +in the `src/` subdirectory: + +```console +pipenv lock +``` + +### Modifying dependencies ### + +If you want to add or remove dependencies you would update the `src/Pipfile` file +and then update dependencies as you would above. + +> [!NOTE] +> You should only specify packages that are explicitly needed for your Docker +> configuration. Allow [Pipenv] to manage the dependencies of the specified +> packages. + ## Image tags ## The images of this container are tagged with [semantic @@ -254,3 +283,5 @@ dedication](https://creativecommons.org/publicdomain/zero/1.0/). All contributions to this project will be released under the CC0 dedication. By submitting a pull request, you are agreeing to comply with this waiver of copyright interest. + +[Pipenv]: https://pypi.org/project/pipenv/ From af716cc3d2f9dd7bcf4181856f8666cab7188cf2 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 28 Feb 2024 14:20:07 -0500 Subject: [PATCH 312/326] Bump setuptools from 69.1.0 to 69.1.1 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 6a00b36..4af43cc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ ENV VIRTUAL_ENV="${CISA_HOME}/.venv" # Versions of the Python packages installed directly ENV PYTHON_PIP_VERSION=24.0 ENV PYTHON_PIPENV_VERSION=2023.12.1 -ENV PYTHON_SETUPTOOLS_VERSION=69.1.0 +ENV PYTHON_SETUPTOOLS_VERSION=69.1.1 ENV PYTHON_WHEEL_VERSION=0.42.0 ### From a65ebdc8b38221690b006cea5b07d086c40a8539 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 28 Feb 2024 14:21:44 -0500 Subject: [PATCH 313/326] Bump Python from 3.12.0 to 3.12.2 --- Dockerfile | 4 ++-- src/Pipfile | 2 +- src/Pipfile.lock | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4af43cc..e78325e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # Official Docker images are in the form library/ while non-official # images are in the form /. -FROM docker.io/library/python:3.12.0-alpine3.18 as compile-stage +FROM docker.io/library/python:3.12.2-alpine3.18 as compile-stage ### # Unprivileged user variables @@ -53,7 +53,7 @@ RUN pipenv check --verbose \ # Official Docker images are in the form library/ while non-official # images are in the form /. -FROM docker.io/library/python:3.12.0-alpine3.18 as build-stage +FROM docker.io/library/python:3.12.2-alpine3.18 as build-stage ### # For a list of pre-defined annotation keys and value types see: diff --git a/src/Pipfile b/src/Pipfile index 56f2fc9..4df9a57 100644 --- a/src/Pipfile +++ b/src/Pipfile @@ -10,4 +10,4 @@ example = {file = "https://github.com/cisagov/skeleton-python-library/archive/v0 # This version should match the version of Python in the image [requires] -python_full_version = "3.12.0" +python_full_version = "3.12.2" diff --git a/src/Pipfile.lock b/src/Pipfile.lock index d39d053..95333e6 100644 --- a/src/Pipfile.lock +++ b/src/Pipfile.lock @@ -1,11 +1,11 @@ { "_meta": { "hash": { - "sha256": "654452851fea1eb2c8811649e5efe8873c8ff51f5c14dd27a4a8ebb5b15a27c4" + "sha256": "18956d92f029b8a57000e268dee96409725be2949018c193cedd3d1e9744d878" }, "pipfile-spec": 6, "requires": { - "python_full_version": "3.12.0" + "python_full_version": "3.12.2" }, "sources": [ { From 5d52367dc42d929a86546f1ae52ee339c86faf43 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 28 Feb 2024 14:22:25 -0500 Subject: [PATCH 314/326] Bump Alpine Linux from 3.18 to 3.19 --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index e78325e..836dbf7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # Official Docker images are in the form library/ while non-official # images are in the form /. -FROM docker.io/library/python:3.12.2-alpine3.18 as compile-stage +FROM docker.io/library/python:3.12.2-alpine3.19 as compile-stage ### # Unprivileged user variables @@ -53,7 +53,7 @@ RUN pipenv check --verbose \ # Official Docker images are in the form library/ while non-official # images are in the form /. -FROM docker.io/library/python:3.12.2-alpine3.18 as build-stage +FROM docker.io/library/python:3.12.2-alpine3.19 as build-stage ### # For a list of pre-defined annotation keys and value types see: From 56afca0d991c05010d22981ba424e631fd88cb12 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 28 Feb 2024 14:27:43 -0500 Subject: [PATCH 315/326] Bump cisagov/skeleton-python-library from 0.0.1 to 0.2.0 Update the Dockerfile and testing to accommodate changes in the new version. --- Dockerfile | 2 +- src/Pipfile | 2 +- src/Pipfile.lock | 19 +++++++++++++++++-- tests/container_test.py | 2 ++ 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 836dbf7..c40e756 100644 --- a/Dockerfile +++ b/Dockerfile @@ -108,4 +108,4 @@ USER ${CISA_USER}:${CISA_GROUP} EXPOSE 8080/TCP VOLUME ["/var/log"] ENTRYPOINT ["example"] -CMD ["--log-level", "DEBUG"] +CMD ["--log-level", "DEBUG", "8", "2"] diff --git a/src/Pipfile b/src/Pipfile index 4df9a57..6480562 100644 --- a/src/Pipfile +++ b/src/Pipfile @@ -6,7 +6,7 @@ name = "pypi" # List any Python dependencies for the image here [packages] # This should match the version of the image -example = {file = "https://github.com/cisagov/skeleton-python-library/archive/v0.0.1.tar.gz"} +example = {file = "https://github.com/cisagov/skeleton-python-library/archive/v0.2.0.tar.gz"} # This version should match the version of Python in the image [requires] diff --git a/src/Pipfile.lock b/src/Pipfile.lock index 95333e6..6afbec3 100644 --- a/src/Pipfile.lock +++ b/src/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "18956d92f029b8a57000e268dee96409725be2949018c193cedd3d1e9744d878" + "sha256": "ad65a437d348bca066601e0d05a14c0ee23e38b59d8d8e71558cb17c57259f6b" }, "pipfile-spec": 6, "requires": { @@ -16,6 +16,14 @@ ] }, "default": { + "contextlib2": { + "hashes": [ + "sha256:3fbdb64466afd23abaf6c977627b75b6139a5a3e8ce38405c5b413aed7a0471f", + "sha256:ab1e2bfe1d01d968e1b7e8d9023bc51ef3509bba217bb730cee3827e1ee82869" + ], + "markers": "python_version >= '3.6'", + "version": "==21.6.0" + }, "docopt": { "hashes": [ "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491" @@ -23,7 +31,14 @@ "version": "==0.6.2" }, "example": { - "file": "https://github.com/cisagov/skeleton-python-library/archive/v0.0.1.tar.gz" + "file": "https://github.com/cisagov/skeleton-python-library/archive/v0.2.0.tar.gz" + }, + "schema": { + "hashes": [ + "sha256:f06717112c61895cabc4707752b88716e8420a8819d71404501e114f91043197", + "sha256:f3ffdeeada09ec34bf40d7d79996d9f7175db93b7a5065de0faa7f41083c1e6c" + ], + "version": "==0.7.5" }, "setuptools": { "hashes": [ diff --git a/tests/container_test.py b/tests/container_test.py index 6451766..bddee4b 100644 --- a/tests/container_test.py +++ b/tests/container_test.py @@ -10,6 +10,7 @@ ENV_VAR = "ECHO_MESSAGE" ENV_VAR_VAL = "Hello World from docker compose!" READY_MESSAGE = "This is a debug message" +DIVISION_MESSAGE = "8 / 2 == 4.000000" SECRET_QUOTE = "Three may keep a secret, if two of them are dead." # nosec RELEASE_TAG = os.getenv("RELEASE_TAG") VERSION_FILE = "src/version.txt" @@ -52,6 +53,7 @@ def test_output(dockerc, main_container): # make sure container exited if running test isolated dockerc.wait(main_container.id) log_output = main_container.logs() + assert DIVISION_MESSAGE in log_output, "Division message not found in log output." assert SECRET_QUOTE in log_output, "Secret not found in log output." From ae0d4a886c28645fec794e30ee0c7c57eeffd46b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 4 Mar 2024 16:23:04 -0500 Subject: [PATCH 316/326] Bump version from 0.0.1 to 0.2.0 --- README.md | 18 +++++++++--------- src/version.txt | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index a74bba4..306552f 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ appropriate for Docker containers and the major languages that we use. To run the `cisagov/example` image via Docker: ```console -docker run cisagov/example:0.0.1 +docker run cisagov/example:0.2.0 ``` ### Running with Docker Compose ### @@ -37,7 +37,7 @@ docker run cisagov/example:0.0.1 services: example: - image: cisagov/example:0.0.1 + image: cisagov/example:0.2.0 volumes: - type: bind source: @@ -82,7 +82,7 @@ environment variables. See the services: example: - image: cisagov/example:0.0.1 + image: cisagov/example:0.2.0 volumes: - type: bind source: @@ -125,7 +125,7 @@ environment variables. See the 1. Pull the new image: ```console - docker pull cisagov/example:0.0.1 + docker pull cisagov/example:0.2.0 ``` 1. Recreate and run the container by following the [previous instructions](#running-with-docker). @@ -164,12 +164,12 @@ and then update dependencies as you would above. The images of this container are tagged with [semantic versions](https://semver.org) of the underlying example project that they containerize. It is recommended that most users use a version tag (e.g. -`:0.0.1`). +`:0.2.0`). | Image:tag | Description | |-----------|-------------| -|`cisagov/example:0.0.1`| An exact release version. | -|`cisagov/example:0.0`| The most recent release matching the major and minor version numbers. | +|`cisagov/example:0.2.0`| An exact release version. | +|`cisagov/example:0.2`| The most recent release matching the major and minor version numbers. | |`cisagov/example:0`| The most recent release matching the major version number. | |`cisagov/example:edge` | The most recent image built from a merge into the `develop` branch of this repository. | |`cisagov/example:nightly` | A nightly build of the `develop` branch of this repository. | @@ -225,7 +225,7 @@ Build the image locally using this git repository as the [build context](https:/ ```console docker build \ - --tag cisagov/example:0.0.1 \ + --tag cisagov/example:0.2.0 \ https://github.com/cisagov/example.git#develop ``` @@ -256,7 +256,7 @@ Docker: --file Dockerfile-x \ --platform linux/amd64 \ --output type=docker \ - --tag cisagov/example:0.0.1 . + --tag cisagov/example:0.2.0 . ``` ## New repositories from a skeleton ## diff --git a/src/version.txt b/src/version.txt index 8acdd82..0ea3a94 100644 --- a/src/version.txt +++ b/src/version.txt @@ -1 +1 @@ -0.0.1 +0.2.0 From 965da2cea1aebc7681459817c981c715449b43c0 Mon Sep 17 00:00:00 2001 From: Nick <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 13 Mar 2024 17:17:44 -0400 Subject: [PATCH 317/326] Correct usage of the term "symlink" Co-authored-by: dav3r --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index c40e756..8dc0ad5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -87,10 +87,10 @@ RUN addgroup --system --gid ${CISA_GID} ${CISA_GROUP} \ && adduser --system --uid ${CISA_UID} --ingroup ${CISA_GROUP} ${CISA_USER} ### -# Copy in the Python virtual environment created in compile-stage, Sym-link the +# Copy in the Python virtual environment created in compile-stage, symlink the # Python binary in the venv to the system-wide Python and add the venv to the PATH. # -# Note that we sym-link the Python binary in the venv to the system-wide Python so that +# Note that we symlink the Python binary in the venv to the system-wide Python so that # any calls to `python3` will use our virtual environment. We are using short flags # because the ln binary in Alpine Linux does not support long flags. The -f instructs # ln to remove the existing file and the -s instructs ln to create a symbolic link. From 570b7d5538da33d5379a4220a55cb4aa84757ad2 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 22 Nov 2024 19:16:35 -0500 Subject: [PATCH 318/326] Bump python from 3.12.2-alpine3.19 to 3.13.1-alpine3.20 The version of Python listed in the Pipfile is updated to match the new Docker image tag. --- Dockerfile | 4 ++-- src/Pipfile | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8dc0ad5..d234755 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # Official Docker images are in the form library/ while non-official # images are in the form /. -FROM docker.io/library/python:3.12.2-alpine3.19 as compile-stage +FROM docker.io/library/python:3.13.1-alpine3.20 as compile-stage ### # Unprivileged user variables @@ -53,7 +53,7 @@ RUN pipenv check --verbose \ # Official Docker images are in the form library/ while non-official # images are in the form /. -FROM docker.io/library/python:3.12.2-alpine3.19 as build-stage +FROM docker.io/library/python:3.13.1-alpine3.20 as build-stage ### # For a list of pre-defined annotation keys and value types see: diff --git a/src/Pipfile b/src/Pipfile index 6480562..fdd19e6 100644 --- a/src/Pipfile +++ b/src/Pipfile @@ -10,4 +10,4 @@ example = {file = "https://github.com/cisagov/skeleton-python-library/archive/v0 # This version should match the version of Python in the image [requires] -python_full_version = "3.12.2" +python_full_version = "3.13.1" From b3c5df532bb8f7dac5119574ce9b50bf9e52f1e1 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 22 Nov 2024 19:20:40 -0500 Subject: [PATCH 319/326] Bump Python packages installed in the Dockerfile - pip from 24.0 to 24.3.1 - pipenv from 2023.12.1 to 2024.4.0 - setuptools from 69.1.1 to 75.6.0 - wheel from 0.42.0 to 0.45.1 --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index d234755..444ee90 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,10 +10,10 @@ ENV CISA_HOME="/home/${CISA_USER}" ENV VIRTUAL_ENV="${CISA_HOME}/.venv" # Versions of the Python packages installed directly -ENV PYTHON_PIP_VERSION=24.0 -ENV PYTHON_PIPENV_VERSION=2023.12.1 -ENV PYTHON_SETUPTOOLS_VERSION=69.1.1 -ENV PYTHON_WHEEL_VERSION=0.42.0 +ENV PYTHON_PIP_VERSION=24.3.1 +ENV PYTHON_PIPENV_VERSION=2024.4.0 +ENV PYTHON_SETUPTOOLS_VERSION=75.6.0 +ENV PYTHON_WHEEL_VERSION=0.45.1 ### # Install the specified versions of pip, setuptools, and wheel into the system From c09b35263a6191aea522f9c67d770f54340156c1 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 22 Nov 2024 19:25:00 -0500 Subject: [PATCH 320/326] Update Python virtual environment dependencies Update the dependencies installed in the Python virtual environment by running `pipenv lock` in the `src/` directory. --- src/Pipfile.lock | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/src/Pipfile.lock b/src/Pipfile.lock index 6afbec3..408d508 100644 --- a/src/Pipfile.lock +++ b/src/Pipfile.lock @@ -1,11 +1,11 @@ { "_meta": { "hash": { - "sha256": "ad65a437d348bca066601e0d05a14c0ee23e38b59d8d8e71558cb17c57259f6b" + "sha256": "8a376df6f25cf8583d5da89da420c5e51660f33a081c1f85236643ef31601833" }, "pipfile-spec": 6, "requires": { - "python_full_version": "3.12.2" + "python_full_version": "3.13.1" }, "sources": [ { @@ -16,14 +16,6 @@ ] }, "default": { - "contextlib2": { - "hashes": [ - "sha256:3fbdb64466afd23abaf6c977627b75b6139a5a3e8ce38405c5b413aed7a0471f", - "sha256:ab1e2bfe1d01d968e1b7e8d9023bc51ef3509bba217bb730cee3827e1ee82869" - ], - "markers": "python_version >= '3.6'", - "version": "==21.6.0" - }, "docopt": { "hashes": [ "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491" @@ -35,18 +27,18 @@ }, "schema": { "hashes": [ - "sha256:f06717112c61895cabc4707752b88716e8420a8819d71404501e114f91043197", - "sha256:f3ffdeeada09ec34bf40d7d79996d9f7175db93b7a5065de0faa7f41083c1e6c" + "sha256:5d976a5b50f36e74e2157b47097b60002bd4d42e65425fcc9c9befadb4255dde", + "sha256:7da553abd2958a19dc2547c388cde53398b39196175a9be59ea1caf5ab0a1807" ], - "version": "==0.7.5" + "version": "==0.7.7" }, "setuptools": { "hashes": [ - "sha256:02fa291a0471b3a18b2b2481ed902af520c69e8ae0919c13da936542754b4c56", - "sha256:5c0806c7d9af348e6dd3777b4f4dbb42c7ad85b190104837488eab9a7c945cf8" + "sha256:8199222558df7c86216af4f84c30e9b34a61d8ba19366cc914424cdbd28252f6", + "sha256:ce74b49e8f7110f9bf04883b730f4765b774ef3ef28f722cce7c273d253aaf7d" ], - "markers": "python_version >= '3.8'", - "version": "==69.1.1" + "markers": "python_version >= '3.9'", + "version": "==75.6.0" } }, "develop": {} From a1ece8f523bb39bb0c559d67c6ce2558357adaac Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 6 Dec 2024 04:32:58 -0500 Subject: [PATCH 321/326] Add a missing Oxford comma to a comment in the Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 444ee90..b69a3f8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -88,7 +88,7 @@ RUN addgroup --system --gid ${CISA_GID} ${CISA_GROUP} \ ### # Copy in the Python virtual environment created in compile-stage, symlink the -# Python binary in the venv to the system-wide Python and add the venv to the PATH. +# Python binary in the venv to the system-wide Python, and add the venv to the PATH. # # Note that we symlink the Python binary in the venv to the system-wide Python so that # any calls to `python3` will use our virtual environment. We are using short flags From fd69f453785c6b660fd3c55b01112a9a7aa2a5b2 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 6 Dec 2024 10:26:13 -0500 Subject: [PATCH 322/326] Ensure FROM/AS keywords use the same casing This resolves the following warning from Docker when building the image: FromAsCasing: 'as' and 'FROM' keywords' casing do not match Co-authored-by: Shane Frasier --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index b69a3f8..ffcae61 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # Official Docker images are in the form library/ while non-official # images are in the form /. -FROM docker.io/library/python:3.13.1-alpine3.20 as compile-stage +FROM docker.io/library/python:3.13.1-alpine3.20 AS compile-stage ### # Unprivileged user variables @@ -53,7 +53,7 @@ RUN pipenv check --verbose \ # Official Docker images are in the form library/ while non-official # images are in the form /. -FROM docker.io/library/python:3.13.1-alpine3.20 as build-stage +FROM docker.io/library/python:3.13.1-alpine3.20 AS build-stage ### # For a list of pre-defined annotation keys and value types see: From 59c6cfc77491db1c500d313c32f5fc85bda18269 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Sun, 5 Jan 2025 07:15:43 -0500 Subject: [PATCH 323/326] Ensure the `bump-version` script can bump prereleases Currently there is a bug in the script's logic that prevents you from running `./bump-version prerelease` to bump _just_ the prerelease. This is due to faulty checking logic to ensure expected behavior when combining bump commands. --- bump-version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bump-version b/bump-version index 414552a..324fea9 100755 --- a/bump-version +++ b/bump-version @@ -125,7 +125,7 @@ if [ -n "$label" ] && [ "$with_prerelease" = false ] && [[ ! " ${commands_with_l invalid_option "Setting the label is only allowed for the following commands: ${commands_with_label[*]}" fi -if [ "$with_prerelease" = true ] && [[ ! " ${commands_with_prerelease[*]} " =~ [[:space:]]${bump_part}[[:space:]] ]]; then +if [ "$with_prerelease" = true ] && [ -n "$bump_part" ] && [[ ! " ${commands_with_prerelease[*]} " =~ [[:space:]]${bump_part}[[:space:]] ]]; then invalid_option "Changing the prerelease is only allowed in conjunction with the following commands: ${commands_with_prerelease[*]}" fi From bb998ddafd75f789eb723d086eb5fdab3226d2da Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Sun, 5 Jan 2025 06:53:13 -0500 Subject: [PATCH 324/326] Stop using an environment variable to store platforms Instead of using the CSV style in an environment variable we can instead use a newline delimited list directly for the `platforms` input. Since the environment variable is only used to provide a value for the `platforms` input there should be no issue with this change. --- .github/workflows/build.yml | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 612806d..f14bae0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -40,8 +40,6 @@ env: BUILDX_CACHE_DIR: ~/.cache/buildx IMAGE_NAME: cisagov/example PIP_CACHE_DIR: ~/.cache/pip - PLATFORMS: "linux/amd64,linux/arm/v6,linux/arm/v7,\ - linux/arm64,linux/ppc64le,linux/s390x" PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit RUN_TMATE: ${{ secrets.RUN_TMATE }} TERRAFORM_DOCS_REPO_BRANCH_NAME: improvement/support_atx_closed_markdown_headers @@ -452,12 +450,12 @@ jobs: uses: mxschmitt/action-tmate@v3 if: env.RUN_TMATE build-push-all: - # Builds the final set of images for each of the platforms listed in - # PLATFORMS environment variable. These images are tagged with the Docker - # tags calculated in the "prepare" job and pushed to Docker Hub and the - # GitHub Container Registry. The contents of README.md are pushed as the - # image's description to Docker Hub. This job is skipped when the - # triggering event is a pull request. + # Builds the final set of images for each of the platforms specified in the + # "platforms" input for the docker/build-push-action Action. These images + # are tagged with the Docker tags calculated in the "prepare" job and + # pushed to Docker Hub and the GitHub Container Registry. The contents of + # README.md are pushed as the image's description to Docker Hub. This job + # is skipped when the triggering event is a pull request. if: github.event_name != 'pull_request' name: Build and push all platforms needs: @@ -541,7 +539,13 @@ jobs: org.opencontainers.image.version=${{ needs.prepare.outputs.source_version }}" - platforms: ${{ env.PLATFORMS }} + platforms: | + linux/amd64 + linux/arm/v6 + linux/arm/v7 + linux/arm64 + linux/ppc64le + linux/s390x # Uncomment the following option if you are building an image for use # on Google Cloud Run or AWS Lambda. The current default image output # is unable to run on either. Please see the following issue for more From ec3b18edebef063f7534c4002d44a4521d8c9725 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 8 Jan 2025 14:21:10 -0500 Subject: [PATCH 325/326] Use the docker/metadata-action Action to generate metadata Switch to using the docker/metadata-action GitHub Action to generate Docker image metadata. This replaces manually specifying the labels and calculating appropriate tags. This approach will both be easier as well as eliminate some of the issues we've had with manually calculating the image tags. --- .github/dependabot.yml | 2 +- .github/workflows/build.yml | 158 +++++------------------------------- 2 files changed, 23 insertions(+), 137 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index d01227a..1128608 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -29,10 +29,10 @@ updates: - dependency-name: step-security/harden-runner # Managed by cisagov/skeleton-docker # - dependency-name: actions/download-artifact - # - dependency-name: actions/github-script # - dependency-name: actions/upload-artifact # - dependency-name: docker/build-push-action # - dependency-name: docker/login-action + # - dependency-name: docker/metadata-action # - dependency-name: docker/setup-buildx-action # - dependency-name: docker/setup-qemu-action # - dependency-name: github/codeql-action diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f14bae0..f15ba5e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -191,54 +191,13 @@ jobs: uses: mxschmitt/action-tmate@v3 if: env.RUN_TMATE prepare: - # Calculates and publishes outputs that are used by other jobs. - # - # Outputs: - # created: - # The current date-time in RFC3339 format. - # repometa: - # The json metadata describing this repository. - # source_version: - # The source version as reported by the `bump-version show` command. - # tags: - # A comma separated list of Docker tags to be applied to the images on - # Docker Hub. The tags will vary depending on: - # - The event that triggered the build. - # - The branch the build is based upon. - # - The git tag the build is based upon. - # - # When a build is based on a git tag of the form `v*.*.*` the image will - # be tagged on Docker Hub with multiple levels of version specificity. - # For example, a git tag of `v1.2.3+a` will generate Docker tags of - # `:1.2.3_a`, `:1.2.3`, `:1.2`, `:1`, and `:latest`. - # - # Builds targeting the default branch will be tagged with `:edge`. - # - # Builds from other branches will be tagged with the branch name. Solidi - # (`/` characters - commonly known as slashes) in branch names are - # replaced with hyphen-minuses (`-` characters) in the Docker tag. For - # more information about the solidus see these links: - # * https://www.compart.com/en/unicode/U+002F - # * https://en.wikipedia.org/wiki/Slash_(punctuation)#Encoding - # - # Builds triggered by a push event are tagged with a short hash in the - # form: sha-12345678 - # - # Builds triggered by a pull request are tagged with the pull request - # number in the form pr-123. - # - # Builds triggered using the GitHub GUI (workflow_dispatch) are tagged - # with the value specified by the user. - # - # Scheduled builds are tagged with `:nightly`. + # Generate Docker image metadata using the docker/metadata-action GitHub Action. name: Prepare build variables needs: - diagnostics outputs: - created: ${{ steps.prep.outputs.created }} - repometa: ${{ steps.repo.outputs.result }} - source_version: ${{ steps.prep.outputs.source_version }} - tags: ${{ steps.prep.outputs.tags }} + labels: ${{ steps.generate-metadata.outputs.labels }} + tags: ${{ steps.generate-metadata.outputs.tags }} permissions: # actions/checkout needs this to fetch code contents: read @@ -254,53 +213,24 @@ jobs: with: egress-policy: audit - uses: actions/checkout@v4 - - name: Gather repository metadata - id: repo - uses: actions/github-script@v7 + - id: generate-metadata + name: Generate Docker image metadata + uses: docker/metadata-action@v5 with: - script: | - const repo = await github.rest.repos.get(context.repo) - return repo.data - - name: Calculate output values - id: prep - run: | - VERSION=noop - SEMVER="^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-((0|[1-9][0-9]*|[0-9]*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9][0-9]*|[0-9]*[a-zA-Z-][0-9a-zA-Z-]*))*))?(\+([0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*))?$" - if [ "${{ github.event_name }}" = "schedule" ]; then - VERSION=nightly - elif [ "${{ github.event_name }}" = "workflow_dispatch" ]; then - VERSION=${{ github.event.inputs.image-tag }} - elif [[ $GITHUB_REF == refs/tags/* ]]; then - VERSION=${GITHUB_REF#refs/tags/} - elif [[ $GITHUB_REF == refs/heads/* ]]; then - VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g') - if [ "${{ github.event.repository.default_branch }}" = "$VERSION" ]; - then - VERSION=edge - fi - elif [[ $GITHUB_REF == refs/pull/* ]]; then - VERSION=pr-${{ github.event.number }} - fi - if [[ $VERSION =~ $SEMVER ]]; then - VERSION_NO_V=${VERSION#v} - MAJOR="${BASH_REMATCH[1]}" - MINOR="${BASH_REMATCH[2]}" - PATCH="${BASH_REMATCH[3]}" - TAGS="${IMAGE_NAME}:${VERSION_NO_V//+/_},${IMAGE_NAME}:${MAJOR}.${MINOR}.${PATCH},${IMAGE_NAME}:${MAJOR}.${MINOR},${IMAGE_NAME}:${MAJOR},${IMAGE_NAME}:latest" - else - TAGS="${IMAGE_NAME}:${VERSION}" - fi - if [ "${{ github.event_name }}" = "push" ]; then - TAGS="${TAGS},${IMAGE_NAME}:sha-${GITHUB_SHA::8}" - fi - for i in ${TAGS//,/ } - do - TAGS="${TAGS},ghcr.io/${i}" - done - echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT - echo "source_version=$(./bump-version show)" >> $GITHUB_OUTPUT - echo "tags=${TAGS}" >> $GITHUB_OUTPUT - echo tags=${TAGS} + images: | + ${{ env.IMAGE_NAME }} + ghcr.io/${{ env.IMAGE_NAME }} + tags: | + type=edge + type=raw,event=workflow_dispatch,value=${{ github.event.inputs.image-tag }} + type=ref,event=branch + type=ref,event=pr + type=ref,event=tag + type=schedule + type=semver,pattern={{major}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{version}} + type=sha - name: Setup tmate debug session uses: mxschmitt/action-tmate@v3 if: github.event.inputs.remote-shell == 'true' || env.RUN_TMATE @@ -350,29 +280,7 @@ jobs: cache-to: type=local,dest=${{ env.BUILDX_CACHE_DIR }} context: . file: ./Dockerfile - labels: "\ - org.opencontainers.image.created=${{ - needs.prepare.outputs.created }} - - org.opencontainers.image.description=${{ - fromJson(needs.prepare.outputs.repometa).description }} - - org.opencontainers.image.licenses=${{ - fromJson(needs.prepare.outputs.repometa).license.spdx_id }} - - org.opencontainers.image.revision=${{ github.sha }} - - org.opencontainers.image.source=${{ - fromJson(needs.prepare.outputs.repometa).clone_url }} - - org.opencontainers.image.title=${{ - fromJson(needs.prepare.outputs.repometa).name }} - - org.opencontainers.image.url=${{ - fromJson(needs.prepare.outputs.repometa).html_url }} - - org.opencontainers.image.version=${{ - needs.prepare.outputs.source_version }}" + labels: ${{ needs.prepare.outputs.labels }} outputs: type=docker,dest=dist/image.tar # Uncomment the following option if you are building an image for use # on Google Cloud Run or AWS Lambda. The current default image output @@ -516,29 +424,7 @@ jobs: cache-to: type=local,dest=${{ env.BUILDX_CACHE_DIR }} context: . file: ./Dockerfile-x - labels: "\ - org.opencontainers.image.created=${{ - needs.prepare.outputs.created }} - - org.opencontainers.image.description=${{ - fromJson(needs.prepare.outputs.repometa).description }} - - org.opencontainers.image.licenses=${{ - fromJson(needs.prepare.outputs.repometa).license.spdx_id }} - - org.opencontainers.image.revision=${{ github.sha }} - - org.opencontainers.image.source=${{ - fromJson(needs.prepare.outputs.repometa).clone_url }} - - org.opencontainers.image.title=${{ - fromJson(needs.prepare.outputs.repometa).name }} - - org.opencontainers.image.url=${{ - fromJson(needs.prepare.outputs.repometa).html_url }} - - org.opencontainers.image.version=${{ - needs.prepare.outputs.source_version }}" + labels: ${{ needs.prepare.outputs.labels }} platforms: | linux/amd64 linux/arm/v6 From 5698c1713675d8a650864a65eb9a05d7bc0a6593 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 10 Jan 2025 06:05:24 -0500 Subject: [PATCH 326/326] Only check the version label on releases Update the test configuration to only check the org.opencontainers.image.version label on releases. We must make this change because the docker/metadata-action will only populate the label with an appropriate value on a release. Otherwise it will be the branch that is sourced when building the image. --- tests/container_test.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/container_test.py b/tests/container_test.py index bddee4b..449c117 100644 --- a/tests/container_test.py +++ b/tests/container_test.py @@ -77,6 +77,9 @@ def test_log_version(dockerc, project_version, version_container): ), f"Container version output to log does not match project version file {VERSION_FILE}" +@pytest.mark.skipif( + RELEASE_TAG in [None, ""], reason="this is not a release (RELEASE_TAG not set)" +) def test_container_version_label_matches(project_version, version_container): """Verify the container version label is the correct version.""" assert (