build: add pants setup

.github/workflows/pipeline.yaml

@@ -27,6 +27,12 @@ jobs:
       name: Setup just
     - name: Install dependencies
       run: just setup
+    - name: Initialize Pants
+      uses: pantsbuild/actions/init-pants@main
+      with:
+        # cache0 makes it easy to bust the cache if needed
+        gha-cache-key: cache0-py310
+        named-caches-hash: ${{ hashFiles('lockfiles/*.json', '**/something-else.lock') }}
     - uses: actions/cache@v3
       with:
         path: ~/.cache/pre-commit/

.gitignore

@@ -11,3 +11,6 @@
 
 # Python
 __pycache__
+
+# Secrets
+.secrets

.justfile

@@ -20,3 +20,11 @@ webserver:
   set -eo pipefail
   DAGSTER_WEBSERVER_POD_NAME=$(kubectl get pods --namespace dagster-prd -l "app.kubernetes.io/name=dagster,app.kubernetes.io/instance=dagster,component=dagster-webserver" | cut -d' ' -f 1 | sed -n '2p')
   kubectl --namespace dagster-prd port-forward $DAGSTER_WEBSERVER_POD_NAME 8080:80
+
+docker_login:
+  #!/usr/bin/env bash
+  set -e
+  ARTIFACT_SA=$(gcloud secrets versions access latest \
+    --secret=ARTIFACT_WRITER_SA_JSON_KEY_B64 \
+    | python -m base64 -d)
+  docker login -u _json_key -p "$ARTIFACT_SA" europe-west4-docker.pkg.dev

deployment/values.yaml

@@ -0,0 +1,161 @@
+---
+global:
+  postgresqlSecretName: ""
+  dagsterHome: ""
+  serviceAccountName: ""
+  celeryConfigSecretName: ""
+
+dagsterHome: "/opt/dagster/dagster_home"
+postgresqlSecretName: "dagster-postgresql-secret"
+celeryConfigSecretName: "dagster-celery-config-secret"
+
+####################################################################################################
+# User Code Deployments: Configuration for user code containers to be loaded via GRPC server. For
+# each item in the "deployments" list, a K8s Deployment and K8s Service will be created to run the
+# GRPC server that Dagster communicates with to get repository information and the current
+# image information. These deployments can be updated independently of the webserver, and Dagster
+# will pull the current image for all execution. When using a distributed executor (such as
+# Celery-K8s) for pipeline run, the current image will be queried once and used for all
+# solid executions for that pipeline run. In order to guarantee that all solid executions within a
+# pipeline execution use the same image, we recommend using a unique tag (ie not "latest").
+#
+# All user code will be invoked within the images.
+####################################################################################################
+deployments:
+  - name: "k8s-example-user-code-1"
+    image:
+      # When a tag is not supplied, it will default as the Helm chart version.
+      repository: "docker.io/dagster/user-code-example"
+      tag: ~
+      # Change with caution! If you're using a fixed tag for pipeline run images, changing the
+      # image pull policy to anything other than "Always" will use a cached/stale image, which is
+      # almost certainly not what you want.
+      pullPolicy: Always
+    # Arguments to `dagster api grpc`.
+    # Ex: "dagster api grpc -m dagster_test.test_project.test_jobs.repo -a define_demo_execution_repo"
+    # would translate to:
+    # dagsterApiGrpcArgs:
+    #   - "-m"
+    #   - "dagster_test.test_project.test_jobs.repo"
+    #   - "-a"
+    #   - "define_demo_execution_repo"
+    # The `dagsterApiGrpcArgs` key can also be replaced with `codeServerArgs` to use a new
+    # experimental `dagster code-server start` command instead of `dagster api grpc`, which takes
+    # identical arguments but can reload its definitions from within the Dagster UI without
+    # needing to restart the user code deployment pod.
+    dagsterApiGrpcArgs:
+      - "-f"
+      - "/example_project/example_repo/repo.py"
+    port: 3030
+
+    # Whether or not to include configuration specified for this user code deployment in the pods
+    # launched for runs from that deployment
+    includeConfigInLaunchedRuns:
+      enabled: true
+
+    # Additional volumes that should be included in the Deployment's Pod. See:
+    # https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#volume-v1-core
+    #
+    # Example:
+    #
+    # volumes:
+    #   - name: my-volume
+    #     configMap: my-config-map
+    volumes: []
+
+    # Additional volume mounts that should be included in the container in the deployment's pod. See:
+    # See: https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#volumemount-v1-core
+    #
+    # Example:
+    #
+    # volumeMounts:
+    #   - name: test-volume
+    #     mountPath: /opt/dagster/test_folder
+    #     subPath: test_file.yaml
+    volumeMounts: []
+
+    # Additional environment variables to set.
+    # These will be directly applied to the daemon container. See
+    # https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/
+    #
+    # Example:
+    #
+    # env:
+    # - name: ENV_ONE
+    #   value: "one"
+    # - name: ENV_TWO
+    #   value: "two"
+    env: []
+
+    # Additional environment variables can be retrieved and set from ConfigMaps. See:
+    # https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#configure-all-key-value-pairs-in-a-configmap-as-container-environment-variables
+    #
+    # Example:
+    #
+    # envConfigMaps:
+    #   - name: config-map
+    envConfigMaps: []
+
+    # Additional environment variables can be retrieved and set from Secrets. See:
+    # https://kubernetes.io/docs/concepts/configuration/secret/#use-case-as-container-environment-variables
+    #
+    # Example:
+    #
+    # envSecrets:
+    #   - name: secret
+    envSecrets: []
+
+    annotations: {}
+    nodeSelector: {}
+    affinity: {}
+    tolerations: []
+    podSecurityContext: {}
+    securityContext: {}
+    resources: {}
+    labels: {}
+
+    # Override the default K8s scheduler
+    # schedulerName: ~
+
+    # Readiness probe detects when the pod is ready to serve requests.
+    # https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes
+    readinessProbe:
+      # Readiness probes are enabled by default.
+      enabled: true
+      # If `readinessProbe` has no `exec` field, then the following default will be used:
+      # exec:
+      #   command: ["dagster", "api", "grpc-health-check", "-p", "{{ $deployment.port }}"]
+      periodSeconds: 20
+      timeoutSeconds: 10
+      successThreshold: 1
+      failureThreshold: 3
+
+    # As of 0.14.0, liveness probes are disabled by default. If you want to enable them, it's recommended to also
+    # enable startup probes.
+    livenessProbe: {}
+    startupProbe:
+      enabled: false
+
+    service:
+      annotations: {}
+
+# Specify secrets to run containers based on images in private registries. See:
+# https://kubernetes.io/docs/concepts/containers/images/#referring-to-an-imagepullsecrets-on-a-pod
+imagePullSecrets: []
+
+serviceAccount:
+  create: true
+
+  # Specifies the name for the service account to reference in the chart. Note that setting
+  # the global service account name will override this field.
+  name: ""
+
+  annotations: {}
+
+# Whether to bind a role to the service account that allows it to create Kubernetes jobs
+rbacEnabled: true
+
+####################################################################################################
+# Extra Manifests: (Optional) Create additional k8s resources within this chart
+####################################################################################################
+extraManifests: []

requirements.txt

@@ -1 +1,2 @@
 pre-commit==3.4.0
+pyinvoke