diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index f3e2d36..24631d0 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -27,6 +27,12 @@ jobs: name: Setup just - name: Install dependencies run: just setup + - name: Initialize Pants + uses: pantsbuild/actions/init-pants@main + with: + # cache0 makes it easy to bust the cache if needed + gha-cache-key: cache0-py310 + named-caches-hash: ${{ hashFiles('lockfiles/*.json', '**/something-else.lock') }} - uses: actions/cache@v3 with: path: ~/.cache/pre-commit/ diff --git a/.gitignore b/.gitignore index 75bd22c..7bda2ec 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,6 @@ # Python __pycache__ + +# Secrets +.secrets diff --git a/.justfile b/.justfile index a489d3d..edd9a25 100644 --- a/.justfile +++ b/.justfile @@ -20,3 +20,11 @@ webserver: set -eo pipefail DAGSTER_WEBSERVER_POD_NAME=$(kubectl get pods --namespace dagster-prd -l "app.kubernetes.io/name=dagster,app.kubernetes.io/instance=dagster,component=dagster-webserver" | cut -d' ' -f 1 | sed -n '2p') kubectl --namespace dagster-prd port-forward $DAGSTER_WEBSERVER_POD_NAME 8080:80 + +docker_login: + #!/usr/bin/env bash + set -e + ARTIFACT_SA=$(gcloud secrets versions access latest \ + --secret=ARTIFACT_WRITER_SA_JSON_KEY_B64 \ + | python -m base64 -d) + docker login -u _json_key -p "$ARTIFACT_SA" europe-west4-docker.pkg.dev diff --git a/deployment/values.yaml b/deployment/values.yaml new file mode 100644 index 0000000..c95c2c5 --- /dev/null +++ b/deployment/values.yaml @@ -0,0 +1,161 @@ +--- +global: + postgresqlSecretName: "" + dagsterHome: "" + serviceAccountName: "" + celeryConfigSecretName: "" + +dagsterHome: "/opt/dagster/dagster_home" +postgresqlSecretName: "dagster-postgresql-secret" +celeryConfigSecretName: "dagster-celery-config-secret" + +#################################################################################################### +# User Code Deployments: Configuration for user code containers to be loaded via GRPC server. For +# each item in the "deployments" list, a K8s Deployment and K8s Service will be created to run the +# GRPC server that Dagster communicates with to get repository information and the current +# image information. These deployments can be updated independently of the webserver, and Dagster +# will pull the current image for all execution. When using a distributed executor (such as +# Celery-K8s) for pipeline run, the current image will be queried once and used for all +# solid executions for that pipeline run. In order to guarantee that all solid executions within a +# pipeline execution use the same image, we recommend using a unique tag (ie not "latest"). +# +# All user code will be invoked within the images. +#################################################################################################### +deployments: + - name: "k8s-example-user-code-1" + image: + # When a tag is not supplied, it will default as the Helm chart version. + repository: "docker.io/dagster/user-code-example" + tag: ~ + # Change with caution! If you're using a fixed tag for pipeline run images, changing the + # image pull policy to anything other than "Always" will use a cached/stale image, which is + # almost certainly not what you want. + pullPolicy: Always + # Arguments to `dagster api grpc`. + # Ex: "dagster api grpc -m dagster_test.test_project.test_jobs.repo -a define_demo_execution_repo" + # would translate to: + # dagsterApiGrpcArgs: + # - "-m" + # - "dagster_test.test_project.test_jobs.repo" + # - "-a" + # - "define_demo_execution_repo" + # The `dagsterApiGrpcArgs` key can also be replaced with `codeServerArgs` to use a new + # experimental `dagster code-server start` command instead of `dagster api grpc`, which takes + # identical arguments but can reload its definitions from within the Dagster UI without + # needing to restart the user code deployment pod. + dagsterApiGrpcArgs: + - "-f" + - "/example_project/example_repo/repo.py" + port: 3030 + + # Whether or not to include configuration specified for this user code deployment in the pods + # launched for runs from that deployment + includeConfigInLaunchedRuns: + enabled: true + + # Additional volumes that should be included in the Deployment's Pod. See: + # https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#volume-v1-core + # + # Example: + # + # volumes: + # - name: my-volume + # configMap: my-config-map + volumes: [] + + # Additional volume mounts that should be included in the container in the deployment's pod. See: + # See: https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#volumemount-v1-core + # + # Example: + # + # volumeMounts: + # - name: test-volume + # mountPath: /opt/dagster/test_folder + # subPath: test_file.yaml + volumeMounts: [] + + # Additional environment variables to set. + # These will be directly applied to the daemon container. See + # https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/ + # + # Example: + # + # env: + # - name: ENV_ONE + # value: "one" + # - name: ENV_TWO + # value: "two" + env: [] + + # Additional environment variables can be retrieved and set from ConfigMaps. See: + # https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#configure-all-key-value-pairs-in-a-configmap-as-container-environment-variables + # + # Example: + # + # envConfigMaps: + # - name: config-map + envConfigMaps: [] + + # Additional environment variables can be retrieved and set from Secrets. See: + # https://kubernetes.io/docs/concepts/configuration/secret/#use-case-as-container-environment-variables + # + # Example: + # + # envSecrets: + # - name: secret + envSecrets: [] + + annotations: {} + nodeSelector: {} + affinity: {} + tolerations: [] + podSecurityContext: {} + securityContext: {} + resources: {} + labels: {} + + # Override the default K8s scheduler + # schedulerName: ~ + + # Readiness probe detects when the pod is ready to serve requests. + # https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes + readinessProbe: + # Readiness probes are enabled by default. + enabled: true + # If `readinessProbe` has no `exec` field, then the following default will be used: + # exec: + # command: ["dagster", "api", "grpc-health-check", "-p", "{{ $deployment.port }}"] + periodSeconds: 20 + timeoutSeconds: 10 + successThreshold: 1 + failureThreshold: 3 + + # As of 0.14.0, liveness probes are disabled by default. If you want to enable them, it's recommended to also + # enable startup probes. + livenessProbe: {} + startupProbe: + enabled: false + + service: + annotations: {} + +# Specify secrets to run containers based on images in private registries. See: +# https://kubernetes.io/docs/concepts/containers/images/#referring-to-an-imagepullsecrets-on-a-pod +imagePullSecrets: [] + +serviceAccount: + create: true + + # Specifies the name for the service account to reference in the chart. Note that setting + # the global service account name will override this field. + name: "" + + annotations: {} + +# Whether to bind a role to the service account that allows it to create Kubernetes jobs +rbacEnabled: true + +#################################################################################################### +# Extra Manifests: (Optional) Create additional k8s resources within this chart +#################################################################################################### +extraManifests: [] diff --git a/requirements.txt b/requirements.txt index 42517ef..60a163b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ pre-commit==3.4.0 +pyinvoke