update workflows to support dbt-spark in the monorepo

.github/workflows/_generate-changelog.yml

@@ -35,6 +35,7 @@ on:
                 -   "dbt-adapters"
                 -   "dbt-athena"
                 -   "dbt-athena-community"
+                -   "dbt-spark"
             merge:
                 description: "Choose whether to merge the changelog branch"
                 type: boolean

.github/workflows/_integration-tests.yml

@@ -52,6 +52,17 @@ permissions:
     id-token: write
     contents: read
 
+env:
+    DBT_INVOCATION_ENV: ${{ vars.DBT_INVOCATION_ENV }}
+    DD_CIVISIBILITY_AGENTLESS_ENABLED: ${{ vars.DD_CIVISIBILITY_AGENTLESS_ENABLED }}
+    DD_API_KEY: ${{ secrets.DD_API_KEY }}
+    DD_SITE: ${{ vars.DD_SITE }}
+    DD_ENV: ${{ vars.DD_ENV }}
+    DD_SERVICE: ${{ github.event.repository.name }}  # this can change per run because of forks
+    DBT_TEST_USER_1: ${{ vars.DBT_TEST_USER_1 }}
+    DBT_TEST_USER_2: ${{ vars.DBT_TEST_USER_2 }}
+    DBT_TEST_USER_3: ${{ vars.DBT_TEST_USER_3 }}
+
 jobs:
     integration-tests-athena:
         if: ${{ inputs.package == 'dbt-athena' || inputs.package == 'dbt-athena-community' }}
@@ -83,3 +94,38 @@ jobs:
                 aws-region: ${{ vars.DBT_TEST_ATHENA_REGION_NAME }}
         -   run: hatch run integration-tests
             working-directory: ./${{ inputs.package }}
+
+    integration-tests-spark:
+        if: ${{ inputs.package == 'dbt-spark' }}
+        runs-on: ${{ inputs.os }}
+        environment:
+            name: "dbt-spark"
+        env:
+            DBT_DATABRICKS_CLUSTER_NAME: ${{ secrets.DBT_DATABRICKS_CLUSTER_NAME }}
+            DBT_DATABRICKS_HOST_NAME: ${{ secrets.DBT_DATABRICKS_HOST_NAME }}
+            DBT_DATABRICKS_ENDPOINT: ${{ secrets.DBT_DATABRICKS_ENDPOINT }}
+            DBT_DATABRICKS_TOKEN: ${{ secrets.DBT_DATABRICKS_TOKEN }}
+            DBT_DATABRICKS_USER: ${{ secrets.DBT_DATABRICKS_USER }}
+        strategy:
+            fail-fast: false
+            matrix:
+                profile:
+                -   "apache_spark"
+                -   "spark_session"
+                -   "spark_http_odbc"
+                -   "databricks_sql_endpoint"
+                -   "databricks_cluster"
+                -   "databricks_http_cluster"
+        steps:
+        -   uses: actions/checkout@v4
+            with:
+                ref: ${{ inputs.branch }}
+                repository: ${{ inputs.repository }}
+        -   uses: actions/setup-python@v5
+            with:
+                python-version: ${{ inputs.python-version }}
+        -   uses: pypa/hatch@install
+        -   run: hatch run pip install -r dagger/requirements.txt
+            working-directory: ./${{ inputs.package }}
+        -   run: hatch run integration-tests --profile ${{ matrix.profile }}
+            working-directory: ./${{ inputs.package }}

.github/workflows/_publish-internal.yml

@@ -23,6 +23,7 @@ on:
                 options:
                 -   "dbt-adapters"
                 -   "dbt-athena"
+                -   "dbt-spark"
             deploy-to:
                 description: "Choose whether to publish to test or prod"
                 type: environment

.github/workflows/_publish-pypi.yml

@@ -24,6 +24,7 @@ on:
                 -   "dbt-adapters"
                 -   "dbt-athena"
                 -   "dbt-athena-community"
+                -   "dbt-spark"
             deploy-to:
                 description: "Choose whether to publish to test or prod"
                 type: environment

.github/workflows/_unit-tests.yml

@@ -32,6 +32,7 @@ on:
                 -   "dbt-adapters"
                 -   "dbt-athena"
                 -   "dbt-athena-community"
+                -   "dbt-spark"
             branch:
                 description: "Choose the branch to test"
                 type: string

.github/workflows/_verify-build.yml

@@ -33,6 +33,7 @@ on:
                 -   "dbt-tests-adapter"
                 -   "dbt-athena"
                 -   "dbt-athena-community"
+                -   "dbt-spark"
             branch:
                 description: "Choose the branch to build"
                 type: string