Add consistency with US data package

.github/workflows/pull_request.yaml

@@ -0,0 +1,75 @@
+# This workflow is used to check various parts of the
+# run prior to merging:
+
+# - The code's formatting
+# - Versioning - note that this checks, but does not update
+# - That the code builds successfully
+# - That any automated tests pass
+name: Pull request
+
+on:
+  pull_request:
+    branches: [main]
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    name: Lint
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.12
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install black
+      - name: Check formatting
+        run: black . -l 79 --check
+  check-version:
+    name: Check version
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # Fetch all history for all tags and branches
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+          ref: ${{ github.event.pull_request.head.ref }}
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.12
+      - name: Build changelog
+        run: pip install "yaml-changelog>=0.1.7" && make changelog
+      - name: Preview changelog update
+        run: ".github/get-changelog-diff.sh"
+      - name: Check version number has been properly updated
+        run: ".github/is-version-number-acceptable.sh"
+  test:
+    name: Build and test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # Fetch all history for all tags and branches
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.12
+      - name: Install package
+        run: make install
+      - name: Download data inputs
+        run: make download
+        env:
+          POLICYENGINE_UK_DATA_GITHUB_TOKEN: ${{ secrets.POLICYENGINE_UK_DATA_GITHUB_TOKEN }}
+      - name: Build datasets
+        run: make data
+        env:
+          LITE_MODE: true
+      - name: Run tests
+        run: pytest
+      - name: Test documentation builds
+        run: make documentation

.github/workflows/ci_cd.yaml → .github/workflows/push.yaml

@@ -1,60 +1,16 @@
-name: CI/CD
+# After successful versioning, this script runs various 
+# parts of the push process
+name: Push
 
 on:
-  pull_request:
-    branches: [main]
-  push:
-    branches: [main]
+  workflow_run:
+    workflows: ["Update versioning"]
+    types: [completed]
 
 jobs:
-  publish-to-pypi:
-    name: Publish to PyPI
-    runs-on: ubuntu-latest
-    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0 # Fetch all history for all tags and branches
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: 3.12
-      - name: Install package
-        run: pip install -e .
-      - name: Build package
-        run: python -m build
-      - name: Publish to PyPI
-        uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          user: __token__
-          password: ${{ secrets.PYPI }}
-          skip-existing: true
-  publish-docs:
-    name: Publish documentation
-    runs-on: ubuntu-latest
-    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0 # Fetch all history for all tags and branches
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: 3.12
-      - name: Install package
-        run: pip install -e ".[dev]"
-      - name: Build Jupyter Book
-        run: make documentation
-      - name: Deploy documentation
-        uses: JamesIves/github-pages-deploy-action@releases/v4
-        with:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          BRANCH: gh-pages
-          FOLDER: docs/_build/html
   lint:
     runs-on: ubuntu-latest
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
     name: Lint
     steps:
       - uses: actions/checkout@v4
@@ -68,10 +24,10 @@ jobs:
           pip install black
       - name: Check formatting
         run: black . -l 79 --check
-
   test:
     name: Build and test
     runs-on: ubuntu-latest
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -82,59 +38,66 @@ jobs:
         with:
           python-version: 3.12
       - name: Install package
-        run: pip install -e ".[dev]"
+        run: make install
       - name: Download data inputs
         run: make download
         env:
           POLICYENGINE_UK_DATA_GITHUB_TOKEN: ${{ secrets.POLICYENGINE_UK_DATA_GITHUB_TOKEN }}
       - name: Build datasets
         run: make data
+        env:
+          LITE_MODE: true
       - name: Run tests
         run: pytest
       - name: Test documentation builds
         run: make documentation
-
+      - name: Build Jupyter Book
+        run: make documentation
+      - name: Deploy documentation
+        uses: JamesIves/github-pages-deploy-action@releases/v4
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          BRANCH: gh-pages
+          FOLDER: docs/_build/html
+  publish-to-pypi:
+    name: Publish to PyPI
+    runs-on: ubuntu-latest
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # Fetch all history for all tags and branches
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.12
+      - name: Install package
+        run: make install
+      - name: Build package
+        run: python -m build
+      - name: Publish a git tag
+        run: ".github/publish-git-tag.sh || true"
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          user: __token__
+          password: ${{ secrets.PYPI }}
+          skip-existing: true
   docker:
-    name: Publish docker image
+    name: Docker
     runs-on: ubuntu-latest
-    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
       - name: Log in to the Container registry
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{github.actor}}
           password: ${{secrets.GITHUB_TOKEN}}
       - name: Build container
         run: docker build . -f docker/policyengine_uk_data.Dockerfile -t ghcr.io/policyengine/policyengine-uk-data:latest
       - name: Push container
-        run: docker push ghcr.io/policyengine/policyengine-uk-data:latest
-
-  upload:
-    name: Upload data 
-    runs-on: ubuntu-latest
-    needs: [lint, test]
-    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0 # Fetch all history for all tags and branches
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: 3.12
-      - name: Install package
-        run: pip install -e ".[dev]"
-      - name: Download data inputs
-        run: make download
-        env:
-          POLICYENGINE_UK_DATA_GITHUB_TOKEN: ${{ secrets.POLICYENGINE_UK_DATA_GITHUB_TOKEN }}
-      - name: Build datasets
-        run: make data
-      - name: Upload data
-        run: make upload
-        env:
-          POLICYENGINE_UK_DATA_GITHUB_TOKEN: ${{ secrets.POLICYENGINE_UK_DATA_GITHUB_TOKEN }}
+        run: docker push ghcr.io/policyengine/policyengine-uk-data:latest

.github/workflows/update_versioning.yaml

@@ -0,0 +1,42 @@
+# This builds and pushes the changelog, then
+# automatically runs push_2 using the new version number
+# to allow for proper pushing to PyPI.
+
+# This script must run first and complete to allow for 
+# proper versioning.
+
+name: Update versioning
+
+on:
+  push:
+    branches: [main]
+
+jobs:
+  check-version:
+    name: Check version
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # Fetch all history for all tags and branches
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+          ref: ${{ github.event.pull_request.head.ref }}
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.12
+      - name: Build changelog
+        run: pip install "yaml-changelog>=0.1.7" && make changelog
+      - name: Preview changelog update
+        run: ".github/get-changelog-diff.sh"
+      - name: Check version number has been properly updated
+        run: ".github/is-version-number-acceptable.sh"
+      - name: Update changelog
+        uses: EndBug/add-and-commit@v9
+        with:
+          add: "."
+          committer_name: Github Actions[bot]
+          author_name: Github Actions[bot]
+          message: Update PolicyEngine US data
+          github_token: ${{ secrets.POLICYENGINE_US_DATA_GITHUB_TOKEN }}

Makefile

@@ -7,6 +7,7 @@ test:
 	pytest
 
 install:
+	pip install policyengine-uk==2.1.1
 	pip install -e ".[dev]"
 
 download:

changelog_entry.yaml

@@ -0,0 +1,5 @@
+- bump: minor
+  changes:
+    added:
+    - 5% dropout in calibration.
+    - Code quality improvements.