Improve flexibility for publishing options

.github/workflows/build-deploy-pudl.yml

@@ -83,6 +83,11 @@ jobs:
       - name: Set up Cloud SDK
         uses: google-github-actions/setup-gcloud@v1
 
+      - name: Determine commit information
+        run: |-
+          echo "COMMIT_BRANCH=$(gitrev-parse --abbrev-ref HEAD)" >> $GITHUB_ENV
+          echo "COMMIT_TIME=$(git log -1 --format=%cd --date=format:%Y-%m-%d-%H%M)" >> $GITHUB_ENV
+
       # Deploy PUDL image to GCE
       - name: Deploy
         run: |-
@@ -110,6 +115,7 @@ jobs:
             --container-env AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
             --container-env AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
             --container-env AWS_DEFAULT_REGION=${{ secrets.AWS_DEFAULT_REGION }} \
+            --container-env PUDL_GCS_OUTPUT=gs://nightly-build-outputs.catalyst.coop/${{ env.COMMIT_TIME }}-${{ env.ACTION_SHA}-${{ env.COMMIT_BRANCH }}
 
       # Start the VM
       - name: Start the deploy-pudl-vm

docker/gcp_pudl_etl.sh

@@ -2,6 +2,8 @@
 # This script runs the entire ETL and validation tests in a docker container on a Google Compute Engine instance.
 # This script won't work locally because it needs adequate GCP permissions.
 
+: "${PUDL_GCS_OUTPUT:=gs://nightly-build-outputs.catalyst.coop/$ACTION_SHA-$GITHUB_REF}"
+
 set -x
 
 function send_slack_msg() {
@@ -41,7 +43,6 @@ function run_pudl_etl() {
 
 function shutdown_vm() {
     # Copy the outputs to the GCS bucket
-    gsutil -m cp -r $PUDL_OUTPUT "gs://nightly-build-outputs.catalyst.coop/$ACTION_SHA-$GITHUB_REF"
 
     upload_file_to_slack $LOGFILE "pudl_etl logs for $ACTION_SHA-$GITHUB_REF:"
 
@@ -54,6 +55,11 @@ function shutdown_vm() {
     curl -X POST -H "Content-Length: 0" -H "Authorization: Bearer ${ACCESS_TOKEN}" https://compute.googleapis.com/compute/v1/projects/catalyst-cooperative-pudl/zones/$GCE_INSTANCE_ZONE/instances/$GCE_INSTANCE/stop
 }
 
+function copy_outputs_to_gcs() {
+    echo "Copying outputs to GCP bucket $PUDL_GCS_OUTPUT"
+    gsutil -m cp -r $PUDL_OUTPUT ${PUDL_GCS_OUTPUT}
+}
+
 function copy_outputs_to_distribution_bucket() {
     echo "Copying outputs to GCP distribution bucket"
     gsutil -m -u $GCP_BILLING_PROJECT cp -r "$PUDL_OUTPUT/*" "gs://pudl.catalyst.coop/$GITHUB_REF"
@@ -89,8 +95,12 @@ run_pudl_etl 2>&1 | tee $LOGFILE
 # Notify slack if the etl succeeded.
 if [[ ${PIPESTATUS[0]} == 0 ]]; then
     notify_slack "success"
+    copy_outputs_to_gcs
 
     # Dump outputs to s3 bucket if branch is dev or build was triggered by a tag
+    # TODO: this behavior should be controlled by on/off switch here and this logic
+    # should be moved to the triggering github action. Having it here feels
+    # fragmented.
     if [ $GITHUB_ACTION_TRIGGER = "push" ] || [ $GITHUB_REF = "dev" ]; then
         copy_outputs_to_distribution_bucket
     fi