wrap up first pass, add ACLs to prod env

.envs/prod.env → .envs/prod/.env

@@ -1,3 +1,3 @@
-export DBT_DATASET=pypi_vulnerabilities
+export DBT_DATASET=pypi_vulnerabilities_us
 export DBT_LOCATION=US
 export DBT_PROJECT=pypi-vulnerabilities

.envs/prod/dataset_acl.json

@@ -0,0 +1,20 @@
+{
+    "access": [
+        {
+            "role": "READER",
+            "specialGroup": "projectReaders"
+        },
+        {
+            "role": "WRITER",
+            "specialGroup": "projectWriters"
+        },
+        {
+            "role": "OWNER",
+            "specialGroup": "projectOwners"
+        },
+        {
+            "role": "roles/bigquery.dataViewer",
+            "specialGroup": "allAuthenticatedUsers"
+        }
+    ]
+}

.github/actions/dbt_build/action.yml

@@ -12,7 +12,7 @@ runs:
       shell: bash
       run: |
           source .venv/bin/activate
-          source .envs/${{ inputs.env }}.env
+          source .envs/${{ inputs.env }}/.env
           rm -rf logs
           dbt clean
           dbt deps

.github/workflows/deploy.yml

@@ -21,9 +21,16 @@ jobs:
       - uses: google-github-actions/setup-gcloud@v2
         with:
           version: '>= 363.0.0'
-      - run: |
+      - name: ensure prod dataset exists
+        run: |
           source .venv/bin/activate
-          source .envs/prod.env
+          source .envs/prod/.env
+          dbt run-operation ensure_target_dataset_exists
+          bq update --source .envs/prod/dataset_acl.json
+      - name: load prod safety db
+        run: |
+          source .venv/bin/activate
+          source .envs/prod/.env
           python etl/safety_db_to_jsonl.py
           python etl/safety_jsonl_to_bq.py
       - uses: ./.github/actions/dbt_build

.gitignore

@@ -1,19 +1,19 @@
 # Python virtualenv files
-.venv/
+/.venv/
 
 # User's environment settings
-.env
+/.env
 
 # DBT logs
-logs/
+/logs/
 
 # DBT target dir
-target/
+/target/
 
 # DBT packages
-dbt_packages/
-package-lock.yml 
+/dbt_packages/
+/package-lock.yml 
 
 # files that we don't want committed
-uncommitted/*
-!uncommitted/README.md
+/uncommitted/*
+!/uncommitted/README.md

dbt_project.yml

@@ -31,6 +31,7 @@ clean-targets:         # directories to be removed by `dbt clean`
 models:
   +labels:
     stability: stable
+    data_classification: public
   +grant_access_to: []
   +persist_docs:
     # push any model/column descriptions to the target database

macros/ensure_target_dataset_exists.sql

@@ -9,7 +9,8 @@
     CREATE SCHEMA IF NOT EXISTS `{{ project_id }}`.`{{ dataset_name }}`
     OPTIONS (
         description = 'Exploring vulnerable PyPI downloads. Managed by https://github.com/brabster/pypi_vulnerabilities',
-        location = '{{ dataset_location }}'
+        location = '{{ dataset_location }}',
+        labels = [('data_classification': 'public')]
     )
 
 {% endmacro %}

models/overview.md

@@ -1,3 +1,10 @@
 {% docs __overview__ %}
 BigQuery-based investigation into PyPI package downloads and vulnerabilities.
-{% enddocs %}
+
+- Derived from public domain information
+- Operating in BigQuery sandbox - 60 day expiry on tables
+- More info in [README.md on GitHub](https://github.com/brabster/pypi_vulnerabilities)
+- Star project {{ env_var('DBT_PROJECT') }} in BigQuery console
+
+{% enddocs %}
+