From 6094b5b1c9d046f6c5188cbf3c52a3835908956c Mon Sep 17 00:00:00 2001 From: noyaarie Date: Tue, 7 Jan 2025 14:56:05 +0200 Subject: [PATCH 1/4] replace old flags with new --- integration_tests/tests/dbt_project.py | 2 +- .../tests/test_dbt_artifacts/test_columns.py | 14 +++++++------- macros/edr/dbt_artifacts/upload_dbt_artifacts.sql | 2 +- macros/edr/dbt_artifacts/upload_dbt_columns.sql | 2 +- macros/edr/system/system_utils/get_config_var.sql | 3 +-- 5 files changed, 11 insertions(+), 12 deletions(-) diff --git a/integration_tests/tests/dbt_project.py b/integration_tests/tests/dbt_project.py index 5fd5168c2..67fa9d976 100644 --- a/integration_tests/tests/dbt_project.py +++ b/integration_tests/tests/dbt_project.py @@ -18,7 +18,7 @@ _DEFAULT_VARS = { "disable_dbt_invocation_autoupload": True, "disable_dbt_artifacts_autoupload": True, - "disable_dbt_columns_autoupload": True, + "columns_upload_strategy": 'none', "disable_run_results": True, "disable_freshness_results": True, "debug_logs": True, diff --git a/integration_tests/tests/test_dbt_artifacts/test_columns.py b/integration_tests/tests/test_dbt_artifacts/test_columns.py index 9c4ebeff7..1a444147f 100644 --- a/integration_tests/tests/test_dbt_artifacts/test_columns.py +++ b/integration_tests/tests/test_dbt_artifacts/test_columns.py @@ -28,17 +28,17 @@ @Parametrization.autodetect_parameters() @Parametrization.case( name="default", - only_with_description=None, + columns_upload_strategy=None, expected_columns=["with_description"], ) @Parametrization.case( name="only_with_description", - only_with_description=True, + columns_upload_strategy='enriched_only', expected_columns=["with_description"], ) @Parametrization.case( name="all", - only_with_description=False, + columns_upload_strategy='all', expected_columns=[ "with_description", "without_description", @@ -48,13 +48,13 @@ ) def test_flatten_table_columns( dbt_project: DbtProject, - only_with_description: Optional[bool], + columns_upload_strategy: Optional[str], expected_columns: List[str], ) -> None: - if only_with_description is not None: + if columns_upload_strategy is not None: dbt_project.dbt_runner.vars[ - "upload_only_columns_with_descriptions" - ] = only_with_description + "columns_upload_strategy" + ] = columns_upload_strategy flattened_columns = json.loads( dbt_project.dbt_runner.run_operation( "elementary.flatten_table_columns", macro_args={"table_node": TABLE_NODE} diff --git a/macros/edr/dbt_artifacts/upload_dbt_artifacts.sql b/macros/edr/dbt_artifacts/upload_dbt_artifacts.sql index 19fa0382f..a9b7070dc 100644 --- a/macros/edr/dbt_artifacts/upload_dbt_artifacts.sql +++ b/macros/edr/dbt_artifacts/upload_dbt_artifacts.sql @@ -11,7 +11,7 @@ } %} - {% if not elementary.get_config_var('disable_dbt_columns_autoupload') %} + {% if elementary.get_config_var('columns_upload_strategy') != 'none' %} {% do model_upload_func_map.update({"dbt_columns": elementary.upload_dbt_columns}) %} {% endif %} diff --git a/macros/edr/dbt_artifacts/upload_dbt_columns.sql b/macros/edr/dbt_artifacts/upload_dbt_columns.sql index 667fd6ebe..6bcbd40f1 100644 --- a/macros/edr/dbt_artifacts/upload_dbt_columns.sql +++ b/macros/edr/dbt_artifacts/upload_dbt_columns.sql @@ -35,7 +35,7 @@ {% set flattened_columns = [] %} {% for column_node in column_nodes.values() %} - {% if not elementary.get_config_var('upload_only_columns_with_descriptions') or column_node.get('description') %} + {% if elementary.get_config_var('columns_upload_strategy') == 'all' or column_node.get('description') %} {% set flat_column = elementary.flatten_column(table_node, column_node) %} {% do flattened_columns.append(flat_column) %} {% endif %} diff --git a/macros/edr/system/system_utils/get_config_var.sql b/macros/edr/system/system_utils/get_config_var.sql index b90f580d8..9373319d6 100644 --- a/macros/edr/system/system_utils/get_config_var.sql +++ b/macros/edr/system/system_utils/get_config_var.sql @@ -32,8 +32,7 @@ 'disable_freshness_results': false, 'disable_tests_results': false, 'disable_dbt_artifacts_autoupload': false, - 'disable_dbt_columns_autoupload': false, - 'upload_only_columns_with_descriptions': true, + 'columns_upload_strategy': 'enriched_only', 'disable_dbt_invocation_autoupload': false, 'disable_skipped_model_alerts': true, 'disable_skipped_test_alerts': true, From b2ba1d6e616e21c220f46cdc673ef251d3a6311a Mon Sep 17 00:00:00 2001 From: noyaarie Date: Tue, 7 Jan 2025 15:09:52 +0200 Subject: [PATCH 2/4] change condition - meta & tags --- macros/edr/dbt_artifacts/upload_dbt_columns.sql | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/macros/edr/dbt_artifacts/upload_dbt_columns.sql b/macros/edr/dbt_artifacts/upload_dbt_columns.sql index 6bcbd40f1..1152b5cd9 100644 --- a/macros/edr/dbt_artifacts/upload_dbt_columns.sql +++ b/macros/edr/dbt_artifacts/upload_dbt_columns.sql @@ -35,7 +35,17 @@ {% set flattened_columns = [] %} {% for column_node in column_nodes.values() %} - {% if elementary.get_config_var('columns_upload_strategy') == 'all' or column_node.get('description') %} + {% set config_dict = elementary.safe_get_with_default(column_node, 'config', {}) %} + {% set config_meta_dict = elementary.safe_get_with_default(config_dict, 'meta') %} + {% set meta_dict = elementary.safe_get_with_default(column_node, 'meta') %} + {% set has_meta = config_meta_dict or meta_dict %} + + {% set config_tags = elementary.safe_get_with_default(config_dict, 'tags') %} + {% set global_tags = elementary.safe_get_with_default(column_node, 'tags') %} + {% set meta_tags = elementary.safe_get_with_default(meta_dict, 'tags') %} + {% set has_tags = config_tags or global_tags or meta_tags %} + + {% if elementary.get_config_var('columns_upload_strategy') == 'all' or column_node.get('description') or has_meta or has_tags %} {% set flat_column = elementary.flatten_column(table_node, column_node) %} {% do flattened_columns.append(flat_column) %} {% endif %} From 07209da0ba32ab15a4e04668c885ba5b85098631 Mon Sep 17 00:00:00 2001 From: noyaarie Date: Tue, 7 Jan 2025 15:22:20 +0200 Subject: [PATCH 3/4] pre-commit --- integration_tests/tests/dbt_project.py | 2 +- .../tests/test_dbt_artifacts/test_columns.py | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/integration_tests/tests/dbt_project.py b/integration_tests/tests/dbt_project.py index 67fa9d976..bb2f026b3 100644 --- a/integration_tests/tests/dbt_project.py +++ b/integration_tests/tests/dbt_project.py @@ -18,7 +18,7 @@ _DEFAULT_VARS = { "disable_dbt_invocation_autoupload": True, "disable_dbt_artifacts_autoupload": True, - "columns_upload_strategy": 'none', + "columns_upload_strategy": "none", "disable_run_results": True, "disable_freshness_results": True, "debug_logs": True, diff --git a/integration_tests/tests/test_dbt_artifacts/test_columns.py b/integration_tests/tests/test_dbt_artifacts/test_columns.py index 1a444147f..41238bef4 100644 --- a/integration_tests/tests/test_dbt_artifacts/test_columns.py +++ b/integration_tests/tests/test_dbt_artifacts/test_columns.py @@ -33,12 +33,12 @@ ) @Parametrization.case( name="only_with_description", - columns_upload_strategy='enriched_only', + columns_upload_strategy="enriched_only", expected_columns=["with_description"], ) @Parametrization.case( name="all", - columns_upload_strategy='all', + columns_upload_strategy="all", expected_columns=[ "with_description", "without_description", @@ -52,9 +52,7 @@ def test_flatten_table_columns( expected_columns: List[str], ) -> None: if columns_upload_strategy is not None: - dbt_project.dbt_runner.vars[ - "columns_upload_strategy" - ] = columns_upload_strategy + dbt_project.dbt_runner.vars["columns_upload_strategy"] = columns_upload_strategy flattened_columns = json.loads( dbt_project.dbt_runner.run_operation( "elementary.flatten_table_columns", macro_args={"table_node": TABLE_NODE} From 273a508143bc6ac783152a0396a2351ee9c02964 Mon Sep 17 00:00:00 2001 From: noyaarie Date: Tue, 7 Jan 2025 16:42:41 +0200 Subject: [PATCH 4/4] fix upload columns --- macros/edr/dbt_artifacts/upload_dbt_columns.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/macros/edr/dbt_artifacts/upload_dbt_columns.sql b/macros/edr/dbt_artifacts/upload_dbt_columns.sql index 1152b5cd9..c87bb4c04 100644 --- a/macros/edr/dbt_artifacts/upload_dbt_columns.sql +++ b/macros/edr/dbt_artifacts/upload_dbt_columns.sql @@ -37,8 +37,8 @@ {% for column_node in column_nodes.values() %} {% set config_dict = elementary.safe_get_with_default(column_node, 'config', {}) %} {% set config_meta_dict = elementary.safe_get_with_default(config_dict, 'meta') %} - {% set meta_dict = elementary.safe_get_with_default(column_node, 'meta') %} - {% set has_meta = config_meta_dict or meta_dict %} + {% set meta_dict = elementary.safe_get_with_default(column_node, 'meta', {}) %} + {% set has_meta = config_meta_dict or meta_dict | length > 0 %} {% set config_tags = elementary.safe_get_with_default(config_dict, 'tags') %} {% set global_tags = elementary.safe_get_with_default(column_node, 'tags') %}