Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Ele 3941 collect column info #772

Merged
merged 4 commits into from
Jan 8, 2025
Merged

Ele 3941 collect column info #772

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
6094b5b
replace old flags with new
NoyaArie Jan 7, 2025
b2ba1d6
change condition - meta & tags
NoyaArie Jan 7, 2025
07209da
pre-commit
NoyaArie Jan 7, 2025
273a508
fix upload columns
NoyaArie Jan 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion integration_tests/tests/dbt_project.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
_DEFAULT_VARS = {
"disable_dbt_invocation_autoupload": True,
"disable_dbt_artifacts_autoupload": True,
"disable_dbt_columns_autoupload": True,
"columns_upload_strategy": "none",
"disable_run_results": True,
"disable_freshness_results": True,
"debug_logs": True,
Expand Down
14 changes: 6 additions & 8 deletions integration_tests/tests/test_dbt_artifacts/test_columns.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,17 +28,17 @@
@Parametrization.autodetect_parameters()
@Parametrization.case(
name="default",
only_with_description=None,
columns_upload_strategy=None,
expected_columns=["with_description"],
)
@Parametrization.case(
name="only_with_description",
only_with_description=True,
columns_upload_strategy="enriched_only",
expected_columns=["with_description"],
)
@Parametrization.case(
name="all",
only_with_description=False,
columns_upload_strategy="all",
expected_columns=[
"with_description",
"without_description",
Expand All @@ -48,13 +48,11 @@
)
def test_flatten_table_columns(
dbt_project: DbtProject,
only_with_description: Optional[bool],
columns_upload_strategy: Optional[str],
expected_columns: List[str],
) -> None:
if only_with_description is not None:
dbt_project.dbt_runner.vars[
"upload_only_columns_with_descriptions"
] = only_with_description
if columns_upload_strategy is not None:
dbt_project.dbt_runner.vars["columns_upload_strategy"] = columns_upload_strategy
flattened_columns = json.loads(
dbt_project.dbt_runner.run_operation(
"elementary.flatten_table_columns", macro_args={"table_node": TABLE_NODE}
Expand Down
2 changes: 1 addition & 1 deletion macros/edr/dbt_artifacts/upload_dbt_artifacts.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
}
%}

{% if not elementary.get_config_var('disable_dbt_columns_autoupload') %}
{% if elementary.get_config_var('columns_upload_strategy') != 'none' %}
{% do model_upload_func_map.update({"dbt_columns": elementary.upload_dbt_columns}) %}
{% endif %}

Expand Down
12 changes: 11 additions & 1 deletion macros/edr/dbt_artifacts/upload_dbt_columns.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,17 @@

{% set flattened_columns = [] %}
{% for column_node in column_nodes.values() %}
{% if not elementary.get_config_var('upload_only_columns_with_descriptions') or column_node.get('description') %}
{% set config_dict = elementary.safe_get_with_default(column_node, 'config', {}) %}
{% set config_meta_dict = elementary.safe_get_with_default(config_dict, 'meta') %}
{% set meta_dict = elementary.safe_get_with_default(column_node, 'meta', {}) %}
{% set has_meta = config_meta_dict or meta_dict | length > 0 %}

{% set config_tags = elementary.safe_get_with_default(config_dict, 'tags') %}
{% set global_tags = elementary.safe_get_with_default(column_node, 'tags') %}
{% set meta_tags = elementary.safe_get_with_default(meta_dict, 'tags') %}
{% set has_tags = config_tags or global_tags or meta_tags %}

{% if elementary.get_config_var('columns_upload_strategy') == 'all' or column_node.get('description') or has_meta or has_tags %}
{% set flat_column = elementary.flatten_column(table_node, column_node) %}
{% do flattened_columns.append(flat_column) %}
{% endif %}
Expand Down
3 changes: 1 addition & 2 deletions macros/edr/system/system_utils/get_config_var.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,7 @@
'disable_freshness_results': false,
'disable_tests_results': false,
'disable_dbt_artifacts_autoupload': false,
'disable_dbt_columns_autoupload': false,
'upload_only_columns_with_descriptions': true,
'columns_upload_strategy': 'enriched_only',
'disable_dbt_invocation_autoupload': false,
'disable_skipped_model_alerts': true,
'disable_skipped_test_alerts': true,
Expand Down
Loading