Merge remote-tracking branch 'origin/dev' into pytest-cov-testing

.codecov.yml

@@ -1,9 +1,12 @@
 ---
 coverage:
   range: 70..100
-  round: down
+  round: nearest
   precision: 1
 
+ignore:
+  - "src/pudl/validate.py"
+
 codecov:
   token: 23a7ee04-6ac5-4d1b-9d36-86b0c50d40c5
   require_ci_to_pass: true

.gitattributes

@@ -1,7 +1,7 @@
 *.ipynb linguist-detectable=false
 *.html linguist-detectable=false
 eia861-transform.ipynb merge=ours
-environments/conda-*lock.yml merge=ours
+environments/conda-*lock.yml merge=ours linguist-generated=true
 *.csv text
 *.py text
 *.json text

.github/workflows/build-deploy-pudl.yml

@@ -12,6 +12,7 @@ env:
   GITHUB_REF: ${{ github.ref_name }} # This is changed to dev if running on a schedule
   GCE_INSTANCE: pudl-deployment-tag # This is changed to pudl-deployment-dev if running on a schedule
   GCE_INSTANCE_ZONE: ${{ secrets.GCE_INSTANCE_ZONE }}
+  GCS_OUTPUT_BUCKET: gs://nightly-build-outputs.catalyst.coop
 
 jobs:
   build_and_deploy_pudl:
@@ -27,13 +28,14 @@ jobs:
           echo "This action was triggered by a schedule." && echo "GCE_INSTANCE=pudl-deployment-dev" >> $GITHUB_ENV && echo "GITHUB_REF=dev" >> $GITHUB_ENV
 
       - name: Checkout Repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ env.GITHUB_REF }}
 
       - name: Get HEAD of the branch (main or dev)
         run: |
           echo "ACTION_SHA=$(git rev-parse HEAD)" >> $GITHUB_ENV
+          echo "SHORT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
 
       - name: Print action vars
         run: |
@@ -53,17 +55,17 @@ jobs:
             type=ref,event=tag
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2.5.0
+        uses: docker/setup-buildx-action@v3.0.0
 
       - name: Login to DockerHub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v2.1.0
+        uses: docker/login-action@v3.0.0
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Build image and push to Docker Hub
-        uses: docker/build-push-action@v4.0.0
+        uses: docker/build-push-action@v5.1.0
         with:
           context: .
           file: docker/Dockerfile
@@ -74,7 +76,7 @@ jobs:
           cache-to: type=gha,mode=max
 
       - id: "auth"
-        uses: "google-github-actions/auth@v1"
+        uses: "google-github-actions/auth@v2"
         with:
           workload_identity_provider: "projects/345950277072/locations/global/workloadIdentityPools/gh-actions-pool/providers/gh-actions-provider"
           service_account: "deploy-pudl-github-action@catalyst-cooperative-pudl.iam.gserviceaccount.com"
@@ -83,6 +85,11 @@ jobs:
       - name: Set up Cloud SDK
         uses: google-github-actions/setup-gcloud@v1
 
+      - name: Determine commit information
+        run: |-
+          echo "COMMIT_BRANCH=$(gitrev-parse --abbrev-ref HEAD)" >> $GITHUB_ENV
+          echo "COMMIT_TIME=$(git log -1 --format=%cd --date=format:%Y-%m-%d-%H%M)" >> $GITHUB_ENV
+
       # Deploy PUDL image to GCE
       - name: Deploy
         env:
@@ -119,6 +126,7 @@ jobs:
             --container-env DAGSTER_PG_DB="dagster-storage" \
             --container-env FLY_ACCESS_TOKEN=${{ secrets.FLY_ACCESS_TOKEN }} \
             --container-env PUDL_SETTINGS_YML="/home/mambauser/src/pudl/package_data/settings/etl_full.yml" \
+            --container-env PUDL_GCS_OUTPUT=${{ env.GCS_OUTPUT_BUCKET }}/${{ env.COMMIT_TIME }}-${{ env.SHORT_SHA }}-${{ env.COMMIT_BRANCH }}
 
       # Start the VM
       - name: Start the deploy-pudl-vm
@@ -129,6 +137,6 @@ jobs:
         uses: slackapi/[email protected]
         with:
           channel-id: "C03FHB9N0PQ"
-          slack-message: "build-deploy-pudl status: ${{ job.status }}\n${{ env.ACTION_SHA }}-${{ env.GITHUB_REF }}"
+          slack-message: "build-deploy-pudl status: ${{ job.status }}\n${{ env.COMMIT_TIME}}-${{ env.SHORT_SHA }}-${{ env.COMMIT_BRANCH }}"
         env:
           SLACK_BOT_TOKEN: ${{ secrets.PUDL_DEPLOY_SLACK_TOKEN }}

.github/workflows/docker-build-test.yml

@@ -13,7 +13,7 @@ jobs:
       id-token: write
     steps:
       - name: Checkout Repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Docker Metadata
         id: docker_metadata
@@ -24,10 +24,10 @@ jobs:
             latest=auto
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2.5.0
+        uses: docker/setup-buildx-action@v3.0.0
 
       - name: Build image but do not push to Docker Hub
-        uses: docker/build-push-action@v4.0.0
+        uses: docker/build-push-action@v5.1.0
         with:
           context: .
           file: docker/Dockerfile

.github/workflows/pytest.yml

@@ -169,7 +169,7 @@ jobs:
       - name: Set default GCP credentials
         id: gcloud-auth
         continue-on-error: true
-        uses: "google-github-actions/auth@v1"
+        uses: "google-github-actions/auth@v2"
         with:
           workload_identity_provider: "projects/345950277072/locations/global/workloadIdentityPools/gh-actions-pool/providers/gh-actions-provider"
           service_account: "tox-pytest-github-action@catalyst-cooperative-pudl.iam.gserviceaccount.com"
@@ -180,19 +180,19 @@ jobs:
         run: |
           coverage run ${{ env.COVERAGE_OPTIONS }} \
             src/pudl/ferc_to_sqlite/cli.py --clobber ${{ env.ETL_COMMANDLINE_OPTIONS }} ${{ env.ETL_CONFIG }}
-#      - name: Run pudl_etl
-#        env:
-#          COVERAGE_FILE: .coverage.pudl_etl
-#        run: |
-#          alembic upgrade head
-#          coverage run ${{ env.COVERAGE_OPTIONS }} \
-#            src/pudl/cli/etl.py ${{ env.ETL_COMMANDLINE_OPTIONS }} ${{ env.ETL_CONFIG }}
-#      - name: Run integration tests
-#        env:
-#          COVERAGE_FILE: .coverage.pytest
-#        run: |
-#          coverage run ${{ env.COVERAGE_OPTIONS }} \
-#           -m pytest -n auto --live-dbs test/integration
+      #      - name: Run pudl_etl
+      #        env:
+      #          COVERAGE_FILE: .coverage.pudl_etl
+      #        run: |
+      #          alembic upgrade head
+      #          coverage run ${{ env.COVERAGE_OPTIONS }} \
+      #            src/pudl/cli/etl.py ${{ env.ETL_COMMANDLINE_OPTIONS }} ${{ env.ETL_CONFIG }}
+      #      - name: Run integration tests
+      #        env:
+      #          COVERAGE_FILE: .coverage.pytest
+      #        run: |
+      #          coverage run ${{ env.COVERAGE_OPTIONS }} \
+      #           -m pytest -n auto --live-dbs test/integration
       - name: Checksum coverage files
         run: ls .coverage* | xargs md5sum | sort
       - name: Generate coverage

.github/workflows/run-etl.yml

@@ -13,7 +13,7 @@ jobs:
       id-token: write
     steps:
       - name: Checkout Repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Docker Metadata
         id: docker_metadata
         uses: docker/[email protected]
@@ -24,15 +24,15 @@ jobs:
             latest=auto
           tags: type=sha
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2.5.0
+        uses: docker/setup-buildx-action@v3.0.0
       - name: Login to DockerHub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v2.1.0
+        uses: docker/login-action@v3.0.0
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Build image and push to Docker Hub
-        uses: docker/build-push-action@v4.0.0
+        uses: docker/build-push-action@v5.1.0
         with:
           context: .
           file: docker/Dockerfile
@@ -48,7 +48,7 @@ jobs:
       contents: read
     steps:
       - id: gcloud-auth
-        uses: google-github-actions/auth@v1
+        uses: google-github-actions/auth@v2
         with:
           workload_identity_provider: "projects/345950277072/locations/global/workloadIdentityPools/gh-actions-pool/providers/gh-actions-provider"
 

.github/workflows/zenodo-cache-sync.yml

@@ -63,7 +63,7 @@ jobs:
 
       - name: Set default gcp credentials
         id: gcloud-auth
-        uses: "google-github-actions/auth@v1"
+        uses: "google-github-actions/auth@v2"
         with:
           workload_identity_provider: "projects/345950277072/locations/global/workloadIdentityPools/gh-actions-pool/providers/gh-actions-provider"
           service_account: "zenodo-cache-manager@catalyst-cooperative-pudl.iam.gserviceaccount.com"

Makefile

@@ -1,8 +1,6 @@
-covargs := --append --source=src/pudl
 gcs_cache_path := --gcs-cache-path=gs://zenodo-cache.catalyst.coop
-pytest_covargs := --cov-append --cov=src/pudl --cov-report=xml
-coverage_report := coverage report --sort=cover
-pytest_args := --durations 20 ${pytest_covargs} ${gcs_cache_path}
+covargs := --append
+pytest_args := --durations 20 ${gcs_cache_path}
 etl_fast_yml := src/pudl/package_data/settings/etl_fast.yml
 etl_full_yml := src/pudl/package_data/settings/etl_full.yml
 
@@ -96,10 +94,7 @@ ferc:
 	rm -f ${PUDL_OUTPUT}/ferc*.sqlite
 	rm -f ${PUDL_OUTPUT}/ferc*_xbrl_datapackage.json
 	rm -f ${PUDL_OUTPUT}/ferc*_xbrl_taxonomy_metadata.json
-	coverage run ${covargs} -- \
-		src/pudl/ferc_to_sqlite/cli.py \
-		${gcs_cache_path} \
-		${etl_full_yml}
+	coverage run ${covargs} -- src/pudl/ferc_to_sqlite/cli.py ${gcs_cache_path} ${etl_full_yml}
 
 # Remove the existing PUDL DB if it exists.
 # Create a new empty DB using alembic.
@@ -108,7 +103,7 @@ ferc:
 pudl:
 	rm -f ${PUDL_OUTPUT}/pudl.sqlite
 	alembic upgrade head
-	coverage run ${covargs} -- src/pudl/cli/etl.py ${gcs_cache_path} ${etl_full_yml}
+	coverage run ${covargs} -- src/pudl/etl/cli.py ${gcs_cache_path} ${etl_full_yml}
 
 ########################################################################################
 # Targets that are coordinated by pytest -- mostly they're actual tests.
@@ -125,13 +120,13 @@ pytest-integration:
 coverage-erase:
 	coverage erase
 
-.PHONY: pytest-coverage
-pytest-coverage: coverage-erase docs-build pytest-ci
-	${coverage_report}
-
 .PHONY: pytest-ci
 pytest-ci: pytest-unit pytest-integration
 
+.PHONY: pytest-coverage
+pytest-coverage: coverage-erase docs-build pytest-ci
+	coverage report
+
 .PHONY: pytest-integration-full
 pytest-integration-full:
 	pytest ${pytest_args} -n auto --live-dbs --etl-settings ${etl_full_yml} test/integration
@@ -151,7 +146,7 @@ nuke: coverage-erase docs-build pytest-unit ferc pudl
 	pudl_check_fks
 	pytest ${pytest_args} -n auto --live-dbs --etl-settings ${etl_full_yml} test/integration
 	pytest ${pytest_args} -n auto --live-dbs test/validate
-	${coverage_report}
+	coverage report
 
 # Check that designated Jupyter notebooks can be run against the current DB
 .PHONY: pytest-jupyter

devtools/sqlite_to_duckdb.py

@@ -1,3 +1,4 @@
+#! /usr/bin/env python
 """A naive script for converting SQLite to DuckDB."""
 import logging
 from pathlib import Path

docker/Dockerfile

@@ -48,10 +48,7 @@ ENV LD_LIBRARY_PATH=${CONDA_PREFIX}/lib
 # We need information from .git to get version with setuptools_scm so we mount that
 # directory without copying it into the image.
 RUN --mount=type=bind,source=.git,target=${PUDL_REPO}/.git \
-    ${CONDA_RUN} pip install --no-cache-dir --no-deps --editable . && \
-    # Run the PUDL setup script so we know where to read and write data
-    ${CONDA_RUN} pudl_setup
-
+    ${CONDA_RUN} pip install --no-cache-dir --no-deps --editable .
 
 # Install awscli2
 # Change back to root because the install script needs access to /usr/local/aws-cli