forced some integration tests to run in series for athena, during Git…

…Hub workflow
dbt-labs · Jan 17, 2025 · 705e11c · 705e11c
1 parent c1f9b89
commit 705e11c
Show file tree

Hide file tree

Showing 4 changed files with 29 additions and 4 deletions.
diff --git a/.github/workflows/_integration-tests.yml b/.github/workflows/_integration-tests.yml
@@ -94,7 +94,8 @@ jobs:
             with:
                 role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.ASSUMABLE_ROLE_NAME }}
                 aws-region: ${{ vars.DBT_TEST_ATHENA_REGION_NAME }}
-        -   run: hatch run integration-tests
+        # run flaky tests command if run-flaky-tests flag is true. Otherwise, run the other tests
+        -   run: ${{ inputs.run-flaky-tests && "hatch run integration-tests -m flaky -n1 --ddtrace" ||  hatch run integration-tests -m "not flaky" }}
             working-directory: ./${{ inputs.package }}
 
     integration-tests-bigquery:

diff --git a/.github/workflows/pull-request-checks.yml b/.github/workflows/pull-request-checks.yml
@@ -65,7 +65,7 @@ jobs:
             os: ${{ matrix.os }}
             python-version: ${{ matrix.python-version }}
 
-    integration-tests:
+    integration-tests-not-flaky:
         uses: ./.github/workflows/_integration-tests.yml
         strategy:
             fail-fast: false
@@ -83,8 +83,29 @@ jobs:
             repository: ${{ github.event.pull_request.head.repo.full_name }}
             os: ${{ matrix.os }}
             python-version: ${{ matrix.python-version }}
+            run-flaky-tests: false
+        secrets: inherit
+    # Special case here for Athena, since we have two identical packages -- dbt-athena and dbt-athena-community
+    # We don't want certain tests within this adapter to run in parallel
+    integration-tests-flaky-athena:
+        uses: ./.github/workflows/_integration-tests.yml
+        strategy:
+            fail-fast: false
+            max-parallel: 1
+            matrix:
+                package:
+                    - "dbt-athena"
+                    - "dbt-athena-community"
+                os: [ ubuntu-22.04 ]
+                python-version: [ "3.9", "3.10", "3.11", "3.12" ]
+        with:
+            package: ${{ matrix.package }}
+            branch: ${{ github.event.pull_request.head.ref }}
+            repository: ${{ github.event.pull_request.head.repo.full_name }}
+            os: ${{ matrix.os }}
+            python-version: ${{ matrix.python-version }}
+            run-flaky-tests: true
         secrets: inherit
-
     # This job does nothing and is only used for branch protection
     results:
         name: "Pull request checks"  # keep this name, branch protection references it

diff --git a/dbt-athena/pyproject.toml b/dbt-athena/pyproject.toml
@@ -52,4 +52,6 @@ filterwarnings = [
     "ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning",
     "ignore:unclosed file .*:ResourceWarning",
 ]
-addopts = "-v --color=yes -n auto"
+markers = [
+    "flaky: marks tests as flaky so they run one at a time (de-select with `-m 'not flaky'`)"
+]
diff --git a/dbt-athena/tests/functional/adapter/test_retries_iceberg.py b/dbt-athena/tests/functional/adapter/test_retries_iceberg.py
@@ -68,6 +68,7 @@ def get_models__source(unique_tag: str):
 seeds__expected_target_post = "id,status\n" + "\n".join([f"{i},{i}" for i in range(PARALLELISM)])
 
 
+@pytest.mark.flaky
 class TestIcebergRetriesDisabled:
     @pytest.fixture(scope="class")
     def unique_tag(self):