From 613fa58ff9d1f06877ad8790a145d91f5913f862 Mon Sep 17 00:00:00 2001 From: colin-rogers-dbt <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Tue, 20 Feb 2024 07:53:05 -0800 Subject: [PATCH 1/9] fix spark cluster start mechanism and add extra dev requirements (#986) --- dagger/run_dbt_spark_tests.py | 18 +++++++++++++++--- dev-requirements.txt | 3 +++ tests/functional/conftest.py | 28 ++++++++++++++++++---------- 3 files changed, 36 insertions(+), 13 deletions(-) diff --git a/dagger/run_dbt_spark_tests.py b/dagger/run_dbt_spark_tests.py index 436cb1e92..15f9cf2c2 100644 --- a/dagger/run_dbt_spark_tests.py +++ b/dagger/run_dbt_spark_tests.py @@ -112,15 +112,27 @@ async def test_spark(test_args): .with_exec(["./scripts/install_os_reqs.sh"]) # install dbt-spark + python deps .with_directory("/src", req_files) - .with_directory("src/dbt", dbt_spark_dir) - .with_directory("src/tests", test_dir) - .with_workdir("/src") .with_exec(["pip", "install", "-U", "pip"]) + .with_workdir("/src") .with_exec(["pip", "install", "-r", "requirements.txt"]) .with_exec(["pip", "install", "-r", "dev-requirements.txt"]) + ) + + # install local dbt-spark changes + tst_container = ( + tst_container.with_workdir("/") + .with_directory("src/dbt", dbt_spark_dir) + .with_workdir("/src") .with_exec(["pip", "install", "-e", "."]) ) + # install local test changes + tst_container = ( + tst_container.with_workdir("/") + .with_directory("src/tests", test_dir) + .with_workdir("/src") + ) + if test_profile == "apache_spark": spark_ctr, spark_host = get_spark_container(client) tst_container = tst_container.with_service_binding(alias=spark_host, service=spark_ctr) diff --git a/dev-requirements.txt b/dev-requirements.txt index 28a626fc3..8f674d84b 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,5 +1,8 @@ # install latest changes in dbt-core # TODO: how to automate switching from develop to version branches? +git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core +git+https://github.com/dbt-labs/dbt-common.git +git+https://github.com/dbt-labs/dbt-adapters.git git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter # if version 1.x or greater -> pin to major version diff --git a/tests/functional/conftest.py b/tests/functional/conftest.py index c1a0397bd..476ffb474 100644 --- a/tests/functional/conftest.py +++ b/tests/functional/conftest.py @@ -1,19 +1,27 @@ -from multiprocessing import Lock - +import time import pytest -_db_start_lock = Lock() -_DB_CLUSTER_STARTED = False + +def _wait_for_databricks_cluster(project): + """ + It takes roughly 3min for the cluster to start, to be safe we'll wait for 5min + """ + for _ in range(60): + try: + project.run_sql("SELECT 1", fetch=True) + return + except Exception: + time.sleep(10) + + raise Exception("Databricks cluster did not start in time") # Running this should prevent tests from needing to be retried because the Databricks cluster isn't available @pytest.fixture(scope="class", autouse=True) def start_databricks_cluster(project, request): - global _DB_CLUSTER_STARTED profile_type = request.config.getoption("--profile") - with _db_start_lock: - if "databricks" in profile_type and not _DB_CLUSTER_STARTED: - print("Starting Databricks cluster") - project.run_sql("SELECT 1") - _DB_CLUSTER_STARTED = True + if "databricks" in profile_type: + _wait_for_databricks_cluster(project) + + yield 1 From ef91425004d58948532af5176be9d18af41d0b87 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Wed, 21 Feb 2024 11:56:09 -0500 Subject: [PATCH 2/9] Add functional tests for unit testing (#976) --- .../unreleased/Features-20240220-195925.yaml | 6 ++++ dbt/include/spark/macros/adapters.sql | 1 + dbt/include/spark/macros/utils/safe_cast.sql | 8 +++++ .../adapter/unit_testing/test_unit_testing.py | 34 +++++++++++++++++++ 4 files changed, 49 insertions(+) create mode 100644 .changes/unreleased/Features-20240220-195925.yaml create mode 100644 dbt/include/spark/macros/utils/safe_cast.sql create mode 100644 tests/functional/adapter/unit_testing/test_unit_testing.py diff --git a/.changes/unreleased/Features-20240220-195925.yaml b/.changes/unreleased/Features-20240220-195925.yaml new file mode 100644 index 000000000..c5d86ab7c --- /dev/null +++ b/.changes/unreleased/Features-20240220-195925.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Implement spark__safe_cast and add functional tests for unit testing +time: 2024-02-20T19:59:25.907821-05:00 +custom: + Author: michelleark + Issue: "987" diff --git a/dbt/include/spark/macros/adapters.sql b/dbt/include/spark/macros/adapters.sql index bf9f63cf9..a6404a2de 100644 --- a/dbt/include/spark/macros/adapters.sql +++ b/dbt/include/spark/macros/adapters.sql @@ -387,6 +387,7 @@ "identifier": tmp_identifier }) -%} + {%- set tmp_relation = tmp_relation.include(database=false, schema=false) -%} {% do return(tmp_relation) %} {% endmacro %} diff --git a/dbt/include/spark/macros/utils/safe_cast.sql b/dbt/include/spark/macros/utils/safe_cast.sql new file mode 100644 index 000000000..3ce5820a8 --- /dev/null +++ b/dbt/include/spark/macros/utils/safe_cast.sql @@ -0,0 +1,8 @@ +{% macro spark__safe_cast(field, type) %} +{%- set field_clean = field.strip('"').strip("'") if (cast_from_string_unsupported_for(type) and field is string) else field -%} +cast({{field_clean}} as {{type}}) +{% endmacro %} + +{% macro cast_from_string_unsupported_for(type) %} + {{ return(type.lower().startswith('struct') or type.lower().startswith('array') or type.lower().startswith('map')) }} +{% endmacro %} diff --git a/tests/functional/adapter/unit_testing/test_unit_testing.py b/tests/functional/adapter/unit_testing/test_unit_testing.py new file mode 100644 index 000000000..b70c581d1 --- /dev/null +++ b/tests/functional/adapter/unit_testing/test_unit_testing.py @@ -0,0 +1,34 @@ +import pytest + +from dbt.tests.adapter.unit_testing.test_types import BaseUnitTestingTypes +from dbt.tests.adapter.unit_testing.test_case_insensitivity import BaseUnitTestCaseInsensivity +from dbt.tests.adapter.unit_testing.test_invalid_input import BaseUnitTestInvalidInput + + +class TestSparkUnitTestingTypes(BaseUnitTestingTypes): + @pytest.fixture + def data_types(self): + # sql_value, yaml_value + return [ + ["1", "1"], + ["2.0", "2.0"], + ["'12345'", "12345"], + ["'string'", "string"], + ["true", "true"], + ["date '2011-11-11'", "2011-11-11"], + ["timestamp '2013-11-03 00:00:00-0'", "2013-11-03 00:00:00-0"], + ["array(1, 2, 3)", "'array(1, 2, 3)'"], + [ + "map('10', 't', '15', 'f', '20', NULL)", + """'map("10", "t", "15", "f", "20", NULL)'""", + ], + ['named_struct("a", 1, "b", 2, "c", 3)', """'named_struct("a", 1, "b", 2, "c", 3)'"""], + ] + + +class TestSparkUnitTestCaseInsensitivity(BaseUnitTestCaseInsensivity): + pass + + +class TestSparkUnitTestInvalidInput(BaseUnitTestInvalidInput): + pass From 5d600086746d75781838ed71aa266f18c1bb37f1 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Tue, 27 Feb 2024 14:27:34 -0500 Subject: [PATCH 3/9] remove actions that sync github and jira (#977) The failing tests are from the retired CircleCI checks that were implemented in GHA. --- .github/workflows/jira-creation.yml | 28 -------------------------- .github/workflows/jira-label.yml | 28 -------------------------- .github/workflows/jira-transition.yml | 29 --------------------------- 3 files changed, 85 deletions(-) delete mode 100644 .github/workflows/jira-creation.yml delete mode 100644 .github/workflows/jira-label.yml delete mode 100644 .github/workflows/jira-transition.yml diff --git a/.github/workflows/jira-creation.yml b/.github/workflows/jira-creation.yml deleted file mode 100644 index 2611a8bdd..000000000 --- a/.github/workflows/jira-creation.yml +++ /dev/null @@ -1,28 +0,0 @@ -# **what?** -# Mirrors issues into Jira. Includes the information: title, -# GitHub Issue ID and URL - -# **why?** -# Jira is our tool for tracking and we need to see these issues in there - -# **when?** -# On issue creation or when an issue is labeled `Jira` - -name: Jira Issue Creation - -on: - issues: - types: [opened, labeled] - -permissions: - issues: write - -jobs: - call-label-action: - uses: dbt-labs/actions/.github/workflows/jira-creation.yml@main - with: - project_key: ADAP - secrets: - JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }} - JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }} - JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }} diff --git a/.github/workflows/jira-label.yml b/.github/workflows/jira-label.yml deleted file mode 100644 index 1637cbe38..000000000 --- a/.github/workflows/jira-label.yml +++ /dev/null @@ -1,28 +0,0 @@ -# **what?** -# Calls mirroring Jira label Action. Includes adding a new label -# to an existing issue or removing a label as well - -# **why?** -# Jira is our tool for tracking and we need to see these labels in there - -# **when?** -# On labels being added or removed from issues - -name: Jira Label Mirroring - -on: - issues: - types: [labeled, unlabeled] - -permissions: - issues: read - -jobs: - call-label-action: - uses: dbt-labs/actions/.github/workflows/jira-label.yml@main - with: - project_key: ADAP - secrets: - JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }} - JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }} - JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }} diff --git a/.github/workflows/jira-transition.yml b/.github/workflows/jira-transition.yml deleted file mode 100644 index 99158a15f..000000000 --- a/.github/workflows/jira-transition.yml +++ /dev/null @@ -1,29 +0,0 @@ -# **what?** -# Transition a Jira issue to a new state -# Only supports these GitHub Issue transitions: -# closed, deleted, reopened - -# **why?** -# Jira needs to be kept up-to-date - -# **when?** -# On issue closing, deletion, reopened - -name: Jira Issue Transition - -on: - issues: - types: [closed, deleted, reopened] - -# no special access is needed -permissions: read-all - -jobs: - call-label-action: - uses: dbt-labs/actions/.github/workflows/jira-transition.yml@main - with: - project_key: ADAP - secrets: - JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }} - JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }} - JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }} From 5246f8207799ccd8d35cf0693fc0eb931b2aa3f9 Mon Sep 17 00:00:00 2001 From: colin-rogers-dbt <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Tue, 27 Feb 2024 18:41:13 -0800 Subject: [PATCH 4/9] update dbt-common dependency to <2.0 (#992) * update dbt-common dependency to <2.0 * update dbt-adapters dependency to <2.0 --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 2d6e00e53..f87cc3213 100644 --- a/setup.py +++ b/setup.py @@ -64,8 +64,8 @@ def _get_plugin_version_dict(): include_package_data=True, install_requires=[ "sqlparams>=3.0.0", - "dbt-common<1.0", - "dbt-adapters~=0.1.0a1", + "dbt-common<2.0", + "dbt-adapters<2.0", ], extras_require={ "ODBC": odbc_extras, From c56b9ce02000bd4e7a88f9fd0ac913a3f47ff2f3 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Thu, 29 Feb 2024 18:24:24 -0500 Subject: [PATCH 5/9] include a pre-release in the dbt-adapters pin to allow pre-releases to be installed on main (#993) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f87cc3213..7342c8660 100644 --- a/setup.py +++ b/setup.py @@ -65,7 +65,7 @@ def _get_plugin_version_dict(): install_requires=[ "sqlparams>=3.0.0", "dbt-common<2.0", - "dbt-adapters<2.0", + "dbt-adapters>=0.1.0a1,<2.0", ], extras_require={ "ODBC": odbc_extras, From 0aee01e7d36754736840008a8fb29c1450752c83 Mon Sep 17 00:00:00 2001 From: colin-rogers-dbt <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Fri, 1 Mar 2024 15:27:35 -0800 Subject: [PATCH 6/9] update install_requires to allow for pre-release common/adapters (#995) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7342c8660..5f6290d1b 100644 --- a/setup.py +++ b/setup.py @@ -64,7 +64,7 @@ def _get_plugin_version_dict(): include_package_data=True, install_requires=[ "sqlparams>=3.0.0", - "dbt-common<2.0", + "dbt-common>=0.1.0a1,<2.0", "dbt-adapters>=0.1.0a1,<2.0", ], extras_require={ From 480355936d44353859f5bdbf80863803495c047f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 1 Mar 2024 16:03:21 -0800 Subject: [PATCH 7/9] [create-pull-request] automated change (#994) Co-authored-by: Github Build Bot Co-authored-by: Mike Alfare <13974384+mikealfare@users.noreply.github.com> --- .bumpversion.cfg | 2 +- .changes/1.8.0-b1.md | 39 ++++++++++++++++++ .../Dependencies-20231009-220732.yaml | 0 .../Dependencies-20231010-222853.yaml | 0 .../Dependencies-20231010-222910.yaml | 0 .../Dependencies-20231013-223750.yaml | 0 .../Dependencies-20231027-230251.yaml | 0 .../Dependencies-20231027-230254.yaml | 0 .../Dependencies-20231027-230301.yaml | 0 .../Dependencies-20231108-222326.yaml | 0 .../Dependencies-20231110-224056.yaml | 0 .../Dependencies-20231113-224111.yaml | 0 .../Dependencies-20231127-220733.yaml | 0 .../Dependencies-20231127-220737.yaml | 0 .../Dependencies-20231127-220741.yaml | 0 .../Dependencies-20231204-224210.yaml | 0 .../Dependencies-20231212-223929.yaml | 0 .../Features-20240220-195925.yaml | 0 .../Fixes-20231107-134141.yaml | 0 .../Fixes-20231221-081949.yaml | 0 .../Under the Hood-20230929-161218.yaml | 0 .../Under the Hood-20231119-132050.yaml | 0 .../Under the Hood-20231214-134728.yaml | 0 .../Under the Hood-20240111-114806.yaml | 0 CHANGELOG.md | 41 +++++++++++++++++++ dbt/adapters/spark/__version__.py | 2 +- setup.py | 2 +- 27 files changed, 83 insertions(+), 3 deletions(-) create mode 100644 .changes/1.8.0-b1.md rename .changes/{unreleased => 1.8.0}/Dependencies-20231009-220732.yaml (100%) rename .changes/{unreleased => 1.8.0}/Dependencies-20231010-222853.yaml (100%) rename .changes/{unreleased => 1.8.0}/Dependencies-20231010-222910.yaml (100%) rename .changes/{unreleased => 1.8.0}/Dependencies-20231013-223750.yaml (100%) rename .changes/{unreleased => 1.8.0}/Dependencies-20231027-230251.yaml (100%) rename .changes/{unreleased => 1.8.0}/Dependencies-20231027-230254.yaml (100%) rename .changes/{unreleased => 1.8.0}/Dependencies-20231027-230301.yaml (100%) rename .changes/{unreleased => 1.8.0}/Dependencies-20231108-222326.yaml (100%) rename .changes/{unreleased => 1.8.0}/Dependencies-20231110-224056.yaml (100%) rename .changes/{unreleased => 1.8.0}/Dependencies-20231113-224111.yaml (100%) rename .changes/{unreleased => 1.8.0}/Dependencies-20231127-220733.yaml (100%) rename .changes/{unreleased => 1.8.0}/Dependencies-20231127-220737.yaml (100%) rename .changes/{unreleased => 1.8.0}/Dependencies-20231127-220741.yaml (100%) rename .changes/{unreleased => 1.8.0}/Dependencies-20231204-224210.yaml (100%) rename .changes/{unreleased => 1.8.0}/Dependencies-20231212-223929.yaml (100%) rename .changes/{unreleased => 1.8.0}/Features-20240220-195925.yaml (100%) rename .changes/{unreleased => 1.8.0}/Fixes-20231107-134141.yaml (100%) rename .changes/{unreleased => 1.8.0}/Fixes-20231221-081949.yaml (100%) rename .changes/{unreleased => 1.8.0}/Under the Hood-20230929-161218.yaml (100%) rename .changes/{unreleased => 1.8.0}/Under the Hood-20231119-132050.yaml (100%) rename .changes/{unreleased => 1.8.0}/Under the Hood-20231214-134728.yaml (100%) rename .changes/{unreleased => 1.8.0}/Under the Hood-20240111-114806.yaml (100%) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 486768676..595914b21 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.8.0a1 +current_version = 1.8.0b1 parse = (?P[\d]+) # major version number \.(?P[\d]+) # minor version number \.(?P[\d]+) # patch version number diff --git a/.changes/1.8.0-b1.md b/.changes/1.8.0-b1.md new file mode 100644 index 000000000..4f4091a91 --- /dev/null +++ b/.changes/1.8.0-b1.md @@ -0,0 +1,39 @@ +## dbt-spark 1.8.0-b1 - March 01, 2024 + +### Features + +- Implement spark__safe_cast and add functional tests for unit testing ([#987](https://github.com/dbt-labs/dbt-spark/issues/987)) + +### Fixes + +- Support new agate Integer type and empty seed test ([#935](https://github.com/dbt-labs/dbt-spark/issues/935)) +- Fix hardcoded file format for python models ([#803](https://github.com/dbt-labs/dbt-spark/issues/803)) + +### Under the Hood + +- Add GitHub action for integration testing and use dagger-io to run tests. Remove CircleCI workflow. ([#719](https://github.com/dbt-labs/dbt-spark/issues/719)) +- Add tests for --empty flag ([#949](https://github.com/dbt-labs/dbt-spark/issues/949)) +- Remove unused `invalid_insert_overwrite_delta_msg` message ([#962](https://github.com/dbt-labs/dbt-spark/issues/962)) +- Update import paths and list_relations to support decoupling adapters/core ([#972](https://github.com/dbt-labs/dbt-spark/issues/972)) + +### Dependencies + +- Update pre-commit-hooks requirement from ~=4.4 to ~=4.5 ([#903](https://github.com/dbt-labs/dbt-spark/pull/903)) +- Bump mypy from 1.5.1 to 1.6.0 ([#904](https://github.com/dbt-labs/dbt-spark/pull/904)) +- Update pyodbc requirement from ~=4.0.39 to ~=5.0.0 ([#905](https://github.com/dbt-labs/dbt-spark/pull/905)) +- Update pre-commit requirement from ~=3.4 to ~=3.5 ([#914](https://github.com/dbt-labs/dbt-spark/pull/914)) +- Update pyodbc requirement from ~=5.0.0 to ~=5.0.1 ([#925](https://github.com/dbt-labs/dbt-spark/pull/925)) +- Bump mypy from 1.6.0 to 1.6.1 ([#926](https://github.com/dbt-labs/dbt-spark/pull/926)) +- Update black requirement from ~=23.9 to ~=23.10 ([#927](https://github.com/dbt-labs/dbt-spark/pull/927)) +- Update black requirement from ~=23.10 to ~=23.11 ([#942](https://github.com/dbt-labs/dbt-spark/pull/942)) +- Bump mypy from 1.6.1 to 1.7.0 ([#946](https://github.com/dbt-labs/dbt-spark/pull/946)) +- Update pytest-xdist requirement from ~=3.3 to ~=3.4 ([#947](https://github.com/dbt-labs/dbt-spark/pull/947)) +- Update pytest-xdist requirement from ~=3.4 to ~=3.5 ([#951](https://github.com/dbt-labs/dbt-spark/pull/951)) +- Update wheel requirement from ~=0.41 to ~=0.42 ([#952](https://github.com/dbt-labs/dbt-spark/pull/952)) +- Bump mypy from 1.7.0 to 1.7.1 ([#953](https://github.com/dbt-labs/dbt-spark/pull/953)) +- Update freezegun requirement from ~=1.2 to ~=1.3 ([#956](https://github.com/dbt-labs/dbt-spark/pull/956)) +- Update black requirement from ~=23.11 to ~=23.12 ([#959](https://github.com/dbt-labs/dbt-spark/pull/959)) + +### Contributors +- [@JCZuurmond,](https://github.com/JCZuurmond,) ([#719](https://github.com/dbt-labs/dbt-spark/issues/719)) +- [@ben-schreiber](https://github.com/ben-schreiber) ([#803](https://github.com/dbt-labs/dbt-spark/issues/803)) diff --git a/.changes/unreleased/Dependencies-20231009-220732.yaml b/.changes/1.8.0/Dependencies-20231009-220732.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20231009-220732.yaml rename to .changes/1.8.0/Dependencies-20231009-220732.yaml diff --git a/.changes/unreleased/Dependencies-20231010-222853.yaml b/.changes/1.8.0/Dependencies-20231010-222853.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20231010-222853.yaml rename to .changes/1.8.0/Dependencies-20231010-222853.yaml diff --git a/.changes/unreleased/Dependencies-20231010-222910.yaml b/.changes/1.8.0/Dependencies-20231010-222910.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20231010-222910.yaml rename to .changes/1.8.0/Dependencies-20231010-222910.yaml diff --git a/.changes/unreleased/Dependencies-20231013-223750.yaml b/.changes/1.8.0/Dependencies-20231013-223750.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20231013-223750.yaml rename to .changes/1.8.0/Dependencies-20231013-223750.yaml diff --git a/.changes/unreleased/Dependencies-20231027-230251.yaml b/.changes/1.8.0/Dependencies-20231027-230251.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20231027-230251.yaml rename to .changes/1.8.0/Dependencies-20231027-230251.yaml diff --git a/.changes/unreleased/Dependencies-20231027-230254.yaml b/.changes/1.8.0/Dependencies-20231027-230254.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20231027-230254.yaml rename to .changes/1.8.0/Dependencies-20231027-230254.yaml diff --git a/.changes/unreleased/Dependencies-20231027-230301.yaml b/.changes/1.8.0/Dependencies-20231027-230301.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20231027-230301.yaml rename to .changes/1.8.0/Dependencies-20231027-230301.yaml diff --git a/.changes/unreleased/Dependencies-20231108-222326.yaml b/.changes/1.8.0/Dependencies-20231108-222326.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20231108-222326.yaml rename to .changes/1.8.0/Dependencies-20231108-222326.yaml diff --git a/.changes/unreleased/Dependencies-20231110-224056.yaml b/.changes/1.8.0/Dependencies-20231110-224056.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20231110-224056.yaml rename to .changes/1.8.0/Dependencies-20231110-224056.yaml diff --git a/.changes/unreleased/Dependencies-20231113-224111.yaml b/.changes/1.8.0/Dependencies-20231113-224111.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20231113-224111.yaml rename to .changes/1.8.0/Dependencies-20231113-224111.yaml diff --git a/.changes/unreleased/Dependencies-20231127-220733.yaml b/.changes/1.8.0/Dependencies-20231127-220733.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20231127-220733.yaml rename to .changes/1.8.0/Dependencies-20231127-220733.yaml diff --git a/.changes/unreleased/Dependencies-20231127-220737.yaml b/.changes/1.8.0/Dependencies-20231127-220737.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20231127-220737.yaml rename to .changes/1.8.0/Dependencies-20231127-220737.yaml diff --git a/.changes/unreleased/Dependencies-20231127-220741.yaml b/.changes/1.8.0/Dependencies-20231127-220741.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20231127-220741.yaml rename to .changes/1.8.0/Dependencies-20231127-220741.yaml diff --git a/.changes/unreleased/Dependencies-20231204-224210.yaml b/.changes/1.8.0/Dependencies-20231204-224210.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20231204-224210.yaml rename to .changes/1.8.0/Dependencies-20231204-224210.yaml diff --git a/.changes/unreleased/Dependencies-20231212-223929.yaml b/.changes/1.8.0/Dependencies-20231212-223929.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20231212-223929.yaml rename to .changes/1.8.0/Dependencies-20231212-223929.yaml diff --git a/.changes/unreleased/Features-20240220-195925.yaml b/.changes/1.8.0/Features-20240220-195925.yaml similarity index 100% rename from .changes/unreleased/Features-20240220-195925.yaml rename to .changes/1.8.0/Features-20240220-195925.yaml diff --git a/.changes/unreleased/Fixes-20231107-134141.yaml b/.changes/1.8.0/Fixes-20231107-134141.yaml similarity index 100% rename from .changes/unreleased/Fixes-20231107-134141.yaml rename to .changes/1.8.0/Fixes-20231107-134141.yaml diff --git a/.changes/unreleased/Fixes-20231221-081949.yaml b/.changes/1.8.0/Fixes-20231221-081949.yaml similarity index 100% rename from .changes/unreleased/Fixes-20231221-081949.yaml rename to .changes/1.8.0/Fixes-20231221-081949.yaml diff --git a/.changes/unreleased/Under the Hood-20230929-161218.yaml b/.changes/1.8.0/Under the Hood-20230929-161218.yaml similarity index 100% rename from .changes/unreleased/Under the Hood-20230929-161218.yaml rename to .changes/1.8.0/Under the Hood-20230929-161218.yaml diff --git a/.changes/unreleased/Under the Hood-20231119-132050.yaml b/.changes/1.8.0/Under the Hood-20231119-132050.yaml similarity index 100% rename from .changes/unreleased/Under the Hood-20231119-132050.yaml rename to .changes/1.8.0/Under the Hood-20231119-132050.yaml diff --git a/.changes/unreleased/Under the Hood-20231214-134728.yaml b/.changes/1.8.0/Under the Hood-20231214-134728.yaml similarity index 100% rename from .changes/unreleased/Under the Hood-20231214-134728.yaml rename to .changes/1.8.0/Under the Hood-20231214-134728.yaml diff --git a/.changes/unreleased/Under the Hood-20240111-114806.yaml b/.changes/1.8.0/Under the Hood-20240111-114806.yaml similarity index 100% rename from .changes/unreleased/Under the Hood-20240111-114806.yaml rename to .changes/1.8.0/Under the Hood-20240111-114806.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 902db37fc..d65c50be4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,47 @@ - "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version. - Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-spark/blob/main/CONTRIBUTING.md#adding-changelog-entry) +## dbt-spark 1.8.0-b1 - March 01, 2024 + +### Features + +- Implement spark__safe_cast and add functional tests for unit testing ([#987](https://github.com/dbt-labs/dbt-spark/issues/987)) + +### Fixes + +- Support new agate Integer type and empty seed test ([#935](https://github.com/dbt-labs/dbt-spark/issues/935)) +- Fix hardcoded file format for python models ([#803](https://github.com/dbt-labs/dbt-spark/issues/803)) + +### Under the Hood + +- Add GitHub action for integration testing and use dagger-io to run tests. Remove CircleCI workflow. ([#719](https://github.com/dbt-labs/dbt-spark/issues/719)) +- Add tests for --empty flag ([#949](https://github.com/dbt-labs/dbt-spark/issues/949)) +- Remove unused `invalid_insert_overwrite_delta_msg` message ([#962](https://github.com/dbt-labs/dbt-spark/issues/962)) +- Update import paths and list_relations to support decoupling adapters/core ([#972](https://github.com/dbt-labs/dbt-spark/issues/972)) + +### Dependencies + +- Update pre-commit-hooks requirement from ~=4.4 to ~=4.5 ([#903](https://github.com/dbt-labs/dbt-spark/pull/903)) +- Bump mypy from 1.5.1 to 1.6.0 ([#904](https://github.com/dbt-labs/dbt-spark/pull/904)) +- Update pyodbc requirement from ~=4.0.39 to ~=5.0.0 ([#905](https://github.com/dbt-labs/dbt-spark/pull/905)) +- Update pre-commit requirement from ~=3.4 to ~=3.5 ([#914](https://github.com/dbt-labs/dbt-spark/pull/914)) +- Update pyodbc requirement from ~=5.0.0 to ~=5.0.1 ([#925](https://github.com/dbt-labs/dbt-spark/pull/925)) +- Bump mypy from 1.6.0 to 1.6.1 ([#926](https://github.com/dbt-labs/dbt-spark/pull/926)) +- Update black requirement from ~=23.9 to ~=23.10 ([#927](https://github.com/dbt-labs/dbt-spark/pull/927)) +- Update black requirement from ~=23.10 to ~=23.11 ([#942](https://github.com/dbt-labs/dbt-spark/pull/942)) +- Bump mypy from 1.6.1 to 1.7.0 ([#946](https://github.com/dbt-labs/dbt-spark/pull/946)) +- Update pytest-xdist requirement from ~=3.3 to ~=3.4 ([#947](https://github.com/dbt-labs/dbt-spark/pull/947)) +- Update pytest-xdist requirement from ~=3.4 to ~=3.5 ([#951](https://github.com/dbt-labs/dbt-spark/pull/951)) +- Update wheel requirement from ~=0.41 to ~=0.42 ([#952](https://github.com/dbt-labs/dbt-spark/pull/952)) +- Bump mypy from 1.7.0 to 1.7.1 ([#953](https://github.com/dbt-labs/dbt-spark/pull/953)) +- Update freezegun requirement from ~=1.2 to ~=1.3 ([#956](https://github.com/dbt-labs/dbt-spark/pull/956)) +- Update black requirement from ~=23.11 to ~=23.12 ([#959](https://github.com/dbt-labs/dbt-spark/pull/959)) + +### Contributors +- [@JCZuurmond,](https://github.com/JCZuurmond,) ([#719](https://github.com/dbt-labs/dbt-spark/issues/719)) +- [@ben-schreiber](https://github.com/ben-schreiber) ([#803](https://github.com/dbt-labs/dbt-spark/issues/803)) + + ## Previous Releases For information on prior major and minor releases, see their changelogs: - [1.6](https://github.com/dbt-labs/dbt-spark/blob/1.6.latest/CHANGELOG.md) diff --git a/dbt/adapters/spark/__version__.py b/dbt/adapters/spark/__version__.py index f15b401d1..6496f3e22 100644 --- a/dbt/adapters/spark/__version__.py +++ b/dbt/adapters/spark/__version__.py @@ -1 +1 @@ -version = "1.8.0a1" +version = "1.8.0b1" diff --git a/setup.py b/setup.py index 5f6290d1b..067ca41b1 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ def _get_plugin_version_dict(): package_name = "dbt-spark" -package_version = "1.8.0a1" +package_version = "1.8.0b1" description = """The Apache Spark adapter plugin for dbt""" odbc_extras = ["pyodbc~=4.0.39"] From d568d21736059aa06fd760672f681010e4ae252c Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Fri, 1 Mar 2024 19:36:56 -0500 Subject: [PATCH 8/9] Vendor release-prep from centralized dbt-release, replace tox testing with spark repo testing (#997) * vendor release-prep from centralized dbt-release, replace tox testing with spark repo testing * replace dbt --version --- .github/workflows/main.yml | 6 +- .github/workflows/release-prep.yml | 650 +++++++++++++++++++++++++++++ .github/workflows/release.yml | 4 +- 3 files changed, 655 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/release-prep.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 20f3f88f4..68911710f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -64,7 +64,7 @@ jobs: mypy --version python -m pip install -r requirements.txt python -m pip install -r dev-requirements.txt - dbt --version + python -c "import dbt.adapters.spark" - name: Run pre-commit hooks run: pre-commit run --all-files --show-diff-on-failure @@ -200,10 +200,10 @@ jobs: find ./dist/*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ - name: Check wheel distributions run: | - dbt --version + python -c "import dbt.adapters.spark" - name: Install source distributions run: | find ./dist/*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ - name: Check source distributions run: | - dbt --version + python -c "import dbt.adapters.spark" diff --git a/.github/workflows/release-prep.yml b/.github/workflows/release-prep.yml new file mode 100644 index 000000000..1a6e450c4 --- /dev/null +++ b/.github/workflows/release-prep.yml @@ -0,0 +1,650 @@ +# **what?** +# Perform the version bump, generate the changelog and run tests. +# +# Inputs: +# sha: The commit to attach to this release +# version_number: The release version number (i.e. 1.0.0b1, 1.2.3rc2, 1.0.0) +# target_branch: The branch that we will release from +# env_setup_script_path: Path to the environment setup script +# test_run: Test run (The temp branch will be used for release) +# nightly_release: Identifier that this is nightly release +# +# Outputs: +# final_sha: The sha that will actually be released. This can differ from the +# input sha if adding a version bump and/or changelog +# changelog_path: Path to the changelog file (ex .changes/1.2.3-rc1.md) +# +# Branching strategy: +# - During execution workflow execution the temp branch will be generated. +# - For normal runs the temp branch will be removed once changes were merged to target branch; +# - For test runs we will keep temp branch and will use it for release; +# Naming strategy: +# - For normal runs: prep-release/${{ inputs.version_number }}_$GITHUB_RUN_ID +# - For test runs: prep-release/test-run/${{ inputs.version_number }}_$GITHUB_RUN_ID +# - For nightly releases: prep-release/nightly-release/${{ inputs.version_number }}_$GITHUB_RUN_ID +# +# **why?** +# Reusable and consistent GitHub release process. +# +# **when?** +# Call when ready to kick off a build and release +# +# Validation Checks +# +# 1. Bump the version if it has not been bumped +# 2. Generate the changelog (via changie) if there is no markdown file for this version +# + +name: Version Bump and Changelog Generation + +on: + workflow_call: + inputs: + sha: + required: true + type: string + version_number: + required: true + type: string + target_branch: + required: true + type: string + env_setup_script_path: + required: false + type: string + default: "" + test_run: + required: false + default: true + type: boolean + nightly_release: + type: boolean + default: false + required: false + outputs: + final_sha: + description: The new commit that includes the changelog and version bump. + value: ${{ jobs.determine-release-sha.outputs.final_sha }} + changelog_path: + description: The path to the changelog for this version + value: ${{ jobs.audit-changelog.outputs.changelog_path }} + secrets: + FISHTOWN_BOT_PAT: + description: "Token to commit/merge changes into branches" + required: true + IT_TEAM_MEMBERSHIP: + description: "Token that can view org level teams" + required: true + +permissions: + contents: write + +defaults: + run: + shell: bash + +env: + PYTHON_TARGET_VERSION: 3.8 + NOTIFICATION_PREFIX: "[Release Preparation]" + +jobs: + log-inputs: + runs-on: ubuntu-latest + + steps: + - name: "[DEBUG] Print Variables" + run: | + # WORKFLOW INPUTS + echo The last commit sha in the release: ${{ inputs.sha }} + echo The release version number: ${{ inputs.version_number }} + echo The branch that we will release from: ${{ inputs.target_branch }} + echo Path to the environment setup script: ${{ inputs.env_setup_script_path }} + echo Test run: ${{ inputs.test_run }} + echo Nightly release: ${{ inputs.nightly_release }} + # ENVIRONMENT VARIABLES + echo Python target version: ${{ env.PYTHON_TARGET_VERSION }} + echo Notification prefix: ${{ env.NOTIFICATION_PREFIX }} + + audit-changelog: + runs-on: ubuntu-latest + + outputs: + changelog_path: ${{ steps.set_path.outputs.changelog_path }} + exists: ${{ steps.set_existence.outputs.exists }} + base_version: ${{ steps.semver.outputs.base-version }} + prerelease: ${{ steps.semver.outputs.pre-release }} + is_prerelease: ${{ steps.semver.outputs.is-pre-release }} + + steps: + - name: "Checkout ${{ github.repository }} Commit ${{ inputs.sha }}" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.sha }} + + - name: "Audit Version And Parse Into Parts" + id: semver + uses: dbt-labs/actions/parse-semver@v1.1.0 + with: + version: ${{ inputs.version_number }} + + - name: "Set Changelog Path" + id: set_path + run: | + path=".changes/" + if [[ ${{ steps.semver.outputs.is-pre-release }} -eq 1 ]] + then + path+="${{ steps.semver.outputs.base-version }}-${{ steps.semver.outputs.pre-release }}.md" + else + path+="${{ steps.semver.outputs.base-version }}.md" + fi + # Send notification + echo "changelog_path=$path" >> $GITHUB_OUTPUT + title="Changelog path" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$changelog_path" + + - name: "Set Changelog Existence For Subsequent Jobs" + id: set_existence + run: | + does_exist=false + if test -f ${{ steps.set_path.outputs.changelog_path }} + then + does_exist=true + fi + echo "exists=$does_exist">> $GITHUB_OUTPUT + + - name: "[Notification] Set Changelog Existence For Subsequent Jobs" + run: | + title="Changelog exists" + if [[ ${{ steps.set_existence.outputs.exists }} == true ]] + then + message="Changelog file ${{ steps.set_path.outputs.changelog_path }} already exists" + else + message="Changelog file ${{ steps.set_path.outputs.changelog_path }} doesn't exist" + fi + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + - name: "Spark safety check" + if: ${{ contains(github.repository, 'dbt-labs/dbt-spark') }} + run: | + if [[ ${{ steps.set_existence.outputs.exists }} != true ]] + then + title="Spark version-bump.yml check" + message="dbt-spark needs version-bump.yml run before running the release. The changelog is not up to date." + echo "::error title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + exit 1 + fi + + - name: "[DEBUG] Print Outputs" + run: | + echo changelog_path: ${{ steps.set_path.outputs.changelog_path }} + echo exists: ${{ steps.set_existence.outputs.exists }} + echo base_version: ${{ steps.semver.outputs.base-version }} + echo prerelease: ${{ steps.semver.outputs.pre-release }} + echo is_prerelease: ${{ steps.semver.outputs.is-pre-release }} + + audit-version-in-code: + runs-on: ubuntu-latest + + outputs: + up_to_date: ${{ steps.version-check.outputs.up_to_date }} + + steps: + - name: "Checkout ${{ github.repository }} Commit ${{ inputs.sha }}" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.sha }} + + - name: "Check Current Version In Code" + id: version-check + run: | + is_updated=false + if grep -Fxq "current_version = ${{ inputs.version_number }}" .bumpversion.cfg + then + is_updated=true + fi + echo "up_to_date=$is_updated" >> $GITHUB_OUTPUT + + - name: "[Notification] Check Current Version In Code" + run: | + title="Version check" + if [[ ${{ steps.version-check.outputs.up_to_date }} == true ]] + then + message="The version in the codebase is equal to the provided version" + else + message="The version in the codebase differs from the provided version" + fi + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + - name: "Spark safety check" + if: ${{ contains(github.repository, 'dbt-labs/dbt-spark') }} + run: | + if [[ ${{ steps.version-check.outputs.up_to_date }} != true ]] + then + title="Spark version-bump.yml check" + message="dbt-spark needs version-bump.yml run before running the release. The version bump is not up to date." + echo "::error title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + exit 1 + fi + + - name: "[DEBUG] Print Outputs" + run: | + echo up_to_date: ${{ steps.version-check.outputs.up_to_date }} + + skip-generate-changelog: + runs-on: ubuntu-latest + needs: [audit-changelog] + if: needs.audit-changelog.outputs.exists == 'true' + + steps: + - name: "Changelog Exists, Skip Generating New Changelog" + run: | + # Send notification + title="Skip changelog generation" + message="A changelog file already exists at ${{ needs.audit-changelog.outputs.changelog_path }}, skipping generating changelog" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + skip-version-bump: + runs-on: ubuntu-latest + needs: [audit-version-in-code] + if: needs.audit-version-in-code.outputs.up_to_date == 'true' + + steps: + - name: "Version Already Bumped" + run: | + # Send notification + title="Skip version bump" + message="The version has already been bumped to ${{ inputs.version_number }}, skipping version bump" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + create-temp-branch: + runs-on: ubuntu-latest + needs: [audit-changelog, audit-version-in-code] + if: needs.audit-changelog.outputs.exists == 'false' || needs.audit-version-in-code.outputs.up_to_date == 'false' + + outputs: + branch_name: ${{ steps.variables.outputs.branch_name }} + + steps: + - name: "Checkout ${{ github.repository }} Commit ${{ inputs.sha }}" + uses: actions/checkout@v4 + with: + ref: ${{ inputs.sha }} + + - name: "Generate Branch Name" + id: variables + run: | + name="prep-release/" + if [[ ${{ inputs.nightly_release }} == true ]] + then + name+="nightly-release/" + elif [[ ${{ inputs.test_run }} == true ]] + then + name+="test-run/" + fi + name+="${{ inputs.version_number }}_$GITHUB_RUN_ID" + echo "branch_name=$name" >> $GITHUB_OUTPUT + + - name: "Create Branch - ${{ steps.variables.outputs.branch_name }}" + run: | + git checkout -b ${{ steps.variables.outputs.branch_name }} + git push -u origin ${{ steps.variables.outputs.branch_name }} + + - name: "[Notification] Temp branch created" + run: | + # Send notification + title="Temp branch generated" + message="The ${{ steps.variables.outputs.branch_name }} branch created" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + - name: "[DEBUG] Print Outputs" + run: | + echo branch_name ${{ steps.variables.outputs.branch_name }} + + generate-changelog-bump-version: + runs-on: ubuntu-latest + needs: [audit-changelog, audit-version-in-code, create-temp-branch] + + steps: + - name: "Checkout ${{ github.repository }} Branch ${{ needs.create-temp-branch.outputs.branch_name }}" + uses: actions/checkout@v4 + with: + ref: ${{ needs.create-temp-branch.outputs.branch_name }} + + - name: "Install Spark Dependencies" + if: ${{ contains(github.repository, 'dbt-labs/dbt-spark') }} + run: | + sudo apt-get update + sudo apt-get install libsasl2-dev + + - name: "Add Homebrew To PATH" + run: | + echo "/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin" >> $GITHUB_PATH + + - name: "Install Homebrew Packages" + run: | + brew install pre-commit + brew tap miniscruff/changie https://github.com/miniscruff/changie + brew install changie + + - name: "Set json File Name" + id: json_file + run: | + echo "name=output_$GITHUB_RUN_ID.json" >> $GITHUB_OUTPUT + + - name: "Get Core Team Membership" + run: | + gh api -H "Accept: application/vnd.github+json" orgs/dbt-labs/teams/core-group/members > ${{ steps.json_file.outputs.name }} + env: + GH_TOKEN: ${{ secrets.IT_TEAM_MEMBERSHIP }} + + - name: "Set Core Team Membership for Changie Contributors exclusion" + id: set_team_membership + run: | + team_list=$(jq -r '.[].login' ${{ steps.json_file.outputs.name }}) + echo $team_list + team_list_single=$(echo $team_list | tr '\n' ' ') + echo "CHANGIE_CORE_TEAM=$team_list_single" >> $GITHUB_ENV + + - name: "Delete the json File" + run: | + rm ${{ steps.json_file.outputs.name }} + + - name: "Generate Release Changelog" + if: needs.audit-changelog.outputs.exists == 'false' + run: | + if [[ ${{ needs.audit-changelog.outputs.is_prerelease }} -eq 1 ]] + then + changie batch ${{ needs.audit-changelog.outputs.base_version }} --move-dir '${{ needs.audit-changelog.outputs.base_version }}' --prerelease ${{ needs.audit-changelog.outputs.prerelease }} + elif [[ -d ".changes/${{ needs.audit-changelog.outputs.base_version }}" ]] + then + changie batch ${{ needs.audit-changelog.outputs.base_version }} --include '${{ needs.audit-changelog.outputs.base_version }}' --remove-prereleases + else # releasing a final patch with no prereleases + changie batch ${{ needs.audit-changelog.outputs.base_version }} + fi + changie merge + git status + + - name: "Check Changelog Created Successfully" + if: needs.audit-changelog.outputs.exists == 'false' + run: | + title="Changelog" + if [[ -f ${{ needs.audit-changelog.outputs.changelog_path }} ]] + then + message="Changelog file created successfully" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + else + message="Changelog failed to generate" + echo "::error title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + exit 1 + fi + + - name: "Set up Python - ${{ env.PYTHON_TARGET_VERSION }}" + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_TARGET_VERSION }} + + - name: "Install Python Dependencies" + if: needs.audit-version-in-code.outputs.up_to_date == 'false' + run: | + python3 -m venv env + source env/bin/activate + python -m pip install --upgrade pip + + - name: "Bump Version To ${{ inputs.version_number }}" + if: needs.audit-version-in-code.outputs.up_to_date == 'false' + # note: bumpversion is no longer supported, it actually points to bump2version now + run: | + source env/bin/activate + if [ -f "editable-requirements.txt" ] + then + python -m pip install -r dev-requirements.txt -r editable-requirements.txt + else + python -m pip install -r dev-requirements.txt + fi + env/bin/bumpversion --allow-dirty --new-version ${{ inputs.version_number }} major + git status + + - name: "[Notification] Bump Version To ${{ inputs.version_number }}" + if: needs.audit-version-in-code.outputs.up_to_date == 'false' + run: | + title="Version bump" + message="Version successfully bumped in codebase to ${{ inputs.version_number }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + # this step will fail on whitespace errors but also correct them + - name: "Remove Trailing Whitespace Via Pre-commit" + continue-on-error: true + run: | + pre-commit run trailing-whitespace --files .bumpversion.cfg CHANGELOG.md .changes/* + git status + + # this step will fail on newline errors but also correct them + - name: "Removing Extra Newlines Via Pre-commit" + continue-on-error: true + run: | + pre-commit run end-of-file-fixer --files .bumpversion.cfg CHANGELOG.md .changes/* + git status + + - name: "Commit & Push Changes" + run: | + #Data for commit + user="Github Build Bot" + email="buildbot@fishtownanalytics.com" + commit_message="Bumping version to ${{ inputs.version_number }} and generate changelog" + #Commit changes to branch + git config user.name "$user" + git config user.email "$email" + git pull + git add . + git commit -m "$commit_message" + git push + + run-unit-tests: + name: unit test / python ${{ matrix.python-version }} + + runs-on: ubuntu-latest + timeout-minutes: 10 + + strategy: + fail-fast: false + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11"] + + steps: + - name: Check out the repository + uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install python dependencies + run: | + sudo apt-get update + sudo apt-get install libsasl2-dev + python -m pip install --user --upgrade pip + python -m pip --version + python -m pip install -r requirements.txt + python -m pip install -r dev-requirements.txt + python -m pip install -e . + + - name: Run unit tests + run: python -m pytest --color=yes --csv unit_results.csv -v tests/unit + + run-integration-tests: + name: ${{ matrix.test }} + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + test: + - "apache_spark" + - "spark_session" + - "databricks_sql_endpoint" + - "databricks_cluster" + - "databricks_http_cluster" + + env: + DBT_INVOCATION_ENV: github-actions + DD_CIVISIBILITY_AGENTLESS_ENABLED: true + DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} + DD_SITE: datadoghq.com + DD_ENV: ci + DD_SERVICE: ${{ github.event.repository.name }} + DBT_DATABRICKS_CLUSTER_NAME: ${{ secrets.DBT_DATABRICKS_CLUSTER_NAME }} + DBT_DATABRICKS_HOST_NAME: ${{ secrets.DBT_DATABRICKS_HOST_NAME }} + DBT_DATABRICKS_ENDPOINT: ${{ secrets.DBT_DATABRICKS_ENDPOINT }} + DBT_DATABRICKS_TOKEN: ${{ secrets.DBT_DATABRICKS_TOKEN }} + DBT_DATABRICKS_USER: ${{ secrets.DBT_DATABRICKS_USERNAME }} + DBT_TEST_USER_1: "buildbot+dbt_test_user_1@dbtlabs.com" + DBT_TEST_USER_2: "buildbot+dbt_test_user_2@dbtlabs.com" + DBT_TEST_USER_3: "buildbot+dbt_test_user_3@dbtlabs.com" + + steps: + - name: Check out the repository + if: github.event_name != 'pull_request_target' + uses: actions/checkout@v3 + with: + persist-credentials: false + + # explicitly checkout the branch for the PR, + # this is necessary for the `pull_request` event + - name: Check out the repository (PR) + if: github.event_name == 'pull_request_target' + uses: actions/checkout@v3 + with: + persist-credentials: false + ref: ${{ github.event.pull_request.head.sha }} + + # the python version used here is not what is used in the tests themselves + - name: Set up Python for dagger + uses: actions/setup-python@v4 + with: + python-version: "3.11" + + - name: Install python dependencies + run: | + python -m pip install --user --upgrade pip + python -m pip --version + python -m pip install -r dagger/requirements.txt + + - name: Run tests for ${{ matrix.test }} + run: python dagger/run_dbt_spark_tests.py --profile ${{ matrix.test }} + + merge-changes-into-target-branch: + runs-on: ubuntu-latest + needs: [run-unit-tests, run-integration-tests, create-temp-branch, audit-version-in-code, audit-changelog] + if: | + !failure() && !cancelled() && + inputs.test_run == false && + ( + needs.audit-changelog.outputs.exists == 'false' || + needs.audit-version-in-code.outputs.up_to_date == 'false' + ) + + steps: + - name: "[Debug] Print Variables" + run: | + echo target_branch: ${{ inputs.target_branch }} + echo branch_name: ${{ needs.create-temp-branch.outputs.branch_name }} + echo inputs.test_run: ${{ inputs.test_run }} + echo needs.audit-changelog.outputs.exists: ${{ needs.audit-changelog.outputs.exists }} + echo needs.audit-version-in-code.outputs.up_to_date: ${{ needs.audit-version-in-code.outputs.up_to_date }} + + - name: "Checkout Repo ${{ github.repository }}" + uses: actions/checkout@v4 + + - name: "Merge Changes Into ${{ inputs.target_branch }}" + uses: everlytic/branch-merge@1.1.5 + with: + source_ref: ${{ needs.create-temp-branch.outputs.branch_name }} + target_branch: ${{ inputs.target_branch }} + github_token: ${{ secrets.FISHTOWN_BOT_PAT }} + commit_message_template: "[Automated] Merged {source_ref} into target {target_branch} during release process" + + - name: "[Notification] Changes Merged into ${{ inputs.target_branch }}" + run: | + title="Changelog and Version Bump Branch Merge" + message="The ${{ needs.create-temp-branch.outputs.branch_name }} branch was merged into ${{ inputs.target_branch }}" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + determine-release-sha: + runs-on: ubuntu-latest + needs: + [ + create-temp-branch, + merge-changes-into-target-branch, + audit-changelog, + audit-version-in-code, + ] + # always run this job, regardless of if the dependant jobs were skipped + if: ${{ !failure() && !cancelled() }} + + # Get the sha that will be released. If the changelog already exists on the input sha and the version has already been bumped, + # then it is what we will release. Otherwise we generated a changelog and did the version bump in this workflow and there is a + # new sha to use from the merge we just did. Grab that here instead. + outputs: + final_sha: ${{ steps.resolve_commit_sha.outputs.release_sha }} + + steps: + - name: "[Debug] Print Variables" + run: | + echo target_branch: ${{ inputs.target_branch }} + echo new_branch: ${{ needs.create-temp-branch.outputs.branch_name }} + echo changelog_exists: ${{ needs.audit-changelog.outputs.exists }} + echo up_to_date: ${{ needs.audit-version-in-code.outputs.up_to_date }} + + - name: "Resolve Branch To Checkout" + id: resolve_branch + run: | + branch="" + if [[ ${{ inputs.test_run == true }} ]] + then + branch=${{ needs.create-temp-branch.outputs.branch_name }} + else + branch=${{ inputs.target_branch }} + fi + echo "target_branch=$branch" >> $GITHUB_OUTPUT + + - name: "[Notification] Resolve Branch To Checkout" + run: | + title="Branch pick" + message="The ${{ steps.resolve_branch.outputs.target_branch }} branch will be used for release" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + - name: "Checkout Resolved Branch - ${{ steps.resolve_branch.outputs.target_branch }}" + uses: actions/checkout@v4 + with: + ref: ${{ steps.resolve_branch.outputs.target_branch }} + + - name: "[Debug] Log Branch" + run: git status + + - name: "Resolve Commit SHA For Release" + id: resolve_commit_sha + run: | + commit_sha="" + if [[ ${{ needs.audit-changelog.outputs.exists }} == false ]] || [[ ${{ needs.audit-version-in-code.outputs.up_to_date }} == false ]] + then + commit_sha=$(git rev-parse HEAD) + else + commit_sha=${{ inputs.sha }} + fi + echo "release_sha=$commit_sha" >> $GITHUB_OUTPUT + + - name: "[Notification] Resolve Commit SHA For Release" + run: | + title="Release commit pick" + message="The ${{ steps.resolve_commit_sha.outputs.release_sha }} commit will be used for release" + echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" + + - name: "Remove Temp Branch - ${{ needs.create-temp-branch.outputs.branch_name }}" + if: ${{ inputs.test_run == false && needs.create-temp-branch.outputs.branch_name != '' }} + run: | + git push origin -d ${{ needs.create-temp-branch.outputs.branch_name }} + + - name: "[Debug] Print Outputs" + run: | + echo release_sha: ${{ steps.resolve_commit_sha.outputs.release_sha }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index df30809a9..9b2774f17 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -44,7 +44,7 @@ on: package_test_command: description: "Package test command" type: string - default: "dbt --version" + default: "python -c \"import dbt.adapters.spark\"" required: true env_setup_script_path: description: "Environment setup script path" @@ -89,7 +89,7 @@ jobs: audit-version-and-changelog: name: Bump package version, Generate changelog - uses: dbt-labs/dbt-release/.github/workflows/release-prep.yml@main + uses: dbt-labs/dbt-spark/.github/workflows/release-prep.yml@main with: sha: ${{ inputs.sha }} From a2c487111d57c9331620613ec83813ff6f0d7d96 Mon Sep 17 00:00:00 2001 From: colin-rogers-dbt <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Fri, 1 Mar 2024 17:53:49 -0800 Subject: [PATCH 9/9] add tox file so release workflows succeed (#996) * add tox file so release workflows succeed * add empty tox file --- tox.ini | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 tox.ini diff --git a/tox.ini b/tox.ini new file mode 100644 index 000000000..62bb9c5b0 --- /dev/null +++ b/tox.ini @@ -0,0 +1,3 @@ +[tox] +skipsdist = True +envlist = unit, flake8, integration-spark-thrift