Manual dagster job #10
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Manual dagster job" | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| command: | |
| required: true | |
| description: "Command to run from the 'dagster_utils' library" | |
| version: | |
| required: false | |
| default: "latest" | |
| description: "Version of the 'dagster_utils' library to use. Defaults to 'latest'" | |
| configuration_location: | |
| required: true | |
| description: "Location of the configuration file that you want to run, relative to the 'jobs' folder (e.g. ./conf/MYCONF.yml)" | |
| jobs: | |
| generate_job_spec: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v2 | |
| - name: Set up Python 3.9 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.9" | |
| - name: Virtual environment | |
| working-directory: ./jobs | |
| run: | | |
| pip install --upgrade pip | |
| python -m venv .gavenv | |
| shell: bash | |
| - name: Install dependencies | |
| working-directory: ./jobs | |
| run: | | |
| . .gavenv/bin/activate | |
| pip3 install -r requirements.txt | |
| shell: bash | |
| - name: Parse kubernetes spec | |
| working-directory: ./jobs | |
| run: | | |
| . .gavenv/bin/activate | |
| python jobs_cli.py parse \ | |
| ${{ github.event.inputs.configuration_location }} \ | |
| ./job_spec.yml \ | |
| europe-west4-docker.pkg.dev/jasper-ginn-dagster/utilities-areg-euw4-jgdag-prd/dagster_scripts \ | |
| ${{ github.event.inputs.version }} \ | |
| ${{ github.event.inputs.command }} \ | |
| ${{ github.run_id }} \ | |
| "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| - name: Echo parsed kubernetes spec | |
| run: cat ./jobs/job_spec.yml | |
| - name: Upload kubernetes job spec | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: job-spec | |
| path: ./jobs/job_spec.yml | |
| deploy_job: | |
| runs-on: ubuntu-latest | |
| needs: generate_job_spec | |
| if: always() && (needs.generate_job_spec.result == 'success') | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v2 | |
| - name: Download kubernetes job spec | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: job-spec | |
| path: ./jobs | |
| - id: 'authenticate_gcp' | |
| uses: 'google-github-actions/auth@v2' | |
| with: | |
| credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}' | |
| - name: 'Set up Cloud SDK' | |
| uses: 'google-github-actions/setup-gcloud@v2' | |
| - name: 'Install GKE auth plugin' | |
| run: gcloud components install gke-gcloud-auth-plugin | |
| - uses: azure/setup-kubectl@v3 | |
| with: | |
| version: 'v1.24.0' | |
| id: 'install_kubectl' | |
| - name: 'Configure GCP project' | |
| run: gcloud config set project jasper-ginn-dagster | |
| - name: 'Configure kubectl' | |
| run: | | |
| USE_GKE_GCLOUD_AUTH_PLUGIN=true gcloud container clusters get-credentials \ | |
| clus-kube-euw4-jgdag-prd \ | |
| --region europe-west4 \ | |
| --project jasper-ginn-dagster | |
| - name: 'Deploy job' | |
| run: | | |
| kubectl apply -f ./jobs/job_spec.yml -n dagster-prd |