Edit tests

dbt_project.yml

@@ -32,7 +32,7 @@ vars:
   future_proof_date: '9999-12-31'
 
 on-run-end:
-  - "{% if target.name == 'prod' %}{{ dbt_artifacts.upload_results(results) }}{% endif %}"
+  - "{{ centralize_test_failures(results) if results !=[]}} "
 
 
 # Configuring models
@@ -46,6 +46,7 @@ models:
         +group: finance
       marketing:
         +group: marketing
+
       operations:
         +group: operations
     intermediate:
@@ -64,8 +65,4 @@ models:
     staging:
       +materialized: ephemeral
 
-tests:
-  rapid_onboarding_exemplar:
-    _samples:
-      staging:
-        +store_failures: "{{ true if target.name == 'prod' else false }}"
+

macros/centralize_test_failures.sql

@@ -0,0 +1,56 @@
+{% macro centralize_test_failures(results) %}
+  {# --add "{{ centralize_test_failures(results) }}" to an on-run-end: block in dbt_project.yml #}
+  {# --run with dbt build --store-failures. The next v.1.0.X release of dbt will include post run hooks for dbt test! #}
+
+  {%- set test_results = [] -%}
+  {%- for result in results -%}
+    {%- if result.node.resource_type == 'test' and result.status != 'skipped' and (
+          result.node.config.get('store_failures') or flags.STORE_FAILURES
+      )
+    -%}
+      {%- do test_results.append(result) -%}
+    {%- endif -%}
+  {%- endfor -%}
+  {%if test_results != []%}
+  {%- set central_tbl -%} {{ target.schema }}.test_failure_central {%- endset -%}
+  {%- set history_tbl -%} {{ target.schema }}.test_failure_history {%- endset -%}
+
+  {{ log("Centralizing test failures in " + central_tbl, info = true) if execute }}
+
+  create or replace table {{ central_tbl }} as (
+
+  {% for result in test_results %}
+
+    select
+      '{{ result.node.name }}' as test_name,
+      '{{ result.node.unique_id }}' as model_name,
+      object_construct_keep_null(*) as test_failures_json,
+      current_timestamp as _timestamp
+
+    from {{ result.node.relation_name }}
+
+    {{ "union all" if not loop.last }}
+
+  {% endfor %}
+
+  );
+
+  -- only run centralization in higher environments
+  {% if target.name != 'default' %}
+      create table if not exists {{ history_tbl }} as (
+        select 
+          {{ dbt_utils.surrogate_key(["test_name", "test_failures_json", "_timestamp"]) }} as sk_id, 
+          * 
+        from {{ central_tbl }}
+        where false
+      );
+
+    insert into {{ history_tbl }} 
+      select 
+       {{ dbt_utils.surrogate_key(["test_name", "test_failures_json", "_timestamp"]) }} as sk_id, 
+       * 
+      from {{ central_tbl }}
+    ;
+  {% endif %}
+{%endif%}
+{% endmacro %}

models/aggregates/_agg__models.yml

@@ -14,12 +14,24 @@ models:
 
   - name: agg_regions_segments
     description: Aggregated model by region and segment.
+    tests:
+      - dbt_datamocktool.unit_test:
+          config: 
+            tags: unit_test
+          input_mapping:
+            ref('dim_customers'): ref('dmt__dim_customers')
+            ref('fct_orders'): ref('dmt__fct_orders')
+          expected_output: ref('dmt__agg_regions_segment')
+          depends_on:
+            - ref('fct_orders')
+            - ref('dim_customers')
     columns:
       - name: region
         description: One of the five global regions.
         tests:
           - accepted_values:
               values: ['MIDDLE EAST','AFRICA','EUROPE','ASIA','AMERICA']
+
 
       - name: market_segment
         description: One of the five market segments.

models/marts/finance/fct_order_items.sql

@@ -1,6 +1,6 @@
 {{
     config(
-        tags = ['finance']
+        tags = ['finance', 'test_tag']
     )
 }}
 

models/marts/marketing/_marketing__models.yml

@@ -13,8 +13,7 @@ models:
       - name: customer_id
         description: Primary id on the customers table
         tests:
-          - unique
-          - not_null
+          - unique_and_not_null
       - name: region
         description: region name
         tests:

models/marts/marketing/dim_customers.sql

@@ -1,8 +1,4 @@
-{{
-    config(
-        transient = false
-    )
-}}
+
 
 with customer as (
 
@@ -42,5 +38,13 @@ select
     *
 from
     final
-order by
-    customer_id
+-- union all 
+-- select 
+--     1 as customer_id, 
+--     'Customer#000000001' as name, 
+--     null as address,
+--     null as nation, 
+--     null as region, 
+--     null as phone_number, 
+--     null as account_balance, 
+--     null as market_segment

models/staging/tpch/_tpch__models.yml

@@ -1,14 +1,20 @@
 version: 2
 
 models:
+  - name: pre_stg_tpch__customers
+    description: staging layer for customers data
+    tests: 
+      - audit_helper_compare_queries:
+          table_b: source('tpch', 'customer')
+          primary_key: c_custkey
+          except_columns: c_address
+          config: 
+            tags: audit_helper
   - name: stg_tpch__customers
     description: staging layer for customers data
     columns:
       - name: customer_id
         description: primary id of the model
-        tests:
-          - unique
-          - not_null
       - name: name
         description: customer id
       - name: address

models/staging/tpch/pre_stg_tpch__customers.sql

@@ -0,0 +1,31 @@
+{{
+    config(
+        materialized='table'
+    )
+}}
+with source as (
+
+    select * from {{ source('tpch', 'customer') }}
+
+),
+
+renamed as (
+
+    select
+        -- ids
+        c_custkey, -- as customer_id,
+        c_nationkey,-- as nation_id,
+
+        -- descriptions
+        c_name,-- as name,
+        c_address,-- as address,
+        c_phone,-- as phone_number,
+        c_acctbal,-- as account_balance,
+        c_mktsegment,-- as market_segment,
+        c_comment-- as comment
+
+    from source
+
+)
+
+select * from renamed

models/staging/tpch/stg_tpch__customers.sql

@@ -1,6 +1,11 @@
+{{
+    config(
+        materialized='table'
+    )
+}}
 with source as (
 
-    select * from {{ source('tpch', 'customer') }}
+    select * from {{ ref('pre_stg_tpch__customers') }}
 
 ),
 

packages.yml

@@ -11,3 +11,5 @@ packages:
     version: 0.6.0
   - package: calogica/dbt_expectations
     version: 0.8.5
+  - package: mjirv/dbt_datamocktool
+    version: [">=0.3.2"]

seeds/dmt__agg_regions_segment.csv

@@ -0,0 +1,2 @@
+REGION,MARKET_SEGMENT,TOTAL_SALES
+ASIA,HOUSEHOLD,100

seeds/dmt__dim_customers.csv

@@ -0,0 +1,2 @@
+customer_id,name,address,nation,region,phone_number,account_balance,market_segment
+5,Customer#000000005,9s7fFRiJFCHIjz4S31ekp3pTH1O8WcQyb3Z0,INDIA,ASIA,18-739-887-4246,10000,HOUSEHOLD

seeds/dmt__fct_orders.csv

@@ -0,0 +1,2 @@
+order_id,order_date,customer_id,order_status_code,priority_code,clerk_name,ship_priority,order_count,gross_item_sales_amount,item_discount_amount,item_tax_amount,net_item_sales_amount
+1,07/01/23,5,O,2-HIGH,Clerk#000000001,0,1,107.5,0,7.5,100

selectors.yml

@@ -7,4 +7,54 @@ selectors:
         - 'seeds/*'
         - intersection:
           - 'config.materialized:snapshot'
-          - 'tag:daily'
+          - 'tag:daily'
+
+  - name: selector_example_test_tags
+    description: this is an example of a job that has tests we want to exclude 
+    definition: 
+      union: 
+        - '+stg_tpch__customers+'
+        - exclude: 
+            - 'tag:audit_helper'
+            - 'tag:unit_test'
+
+
+  - name: selector_example
+    description: this is an example of a job
+    definition: 
+      union: 
+        - intersection:
+          - method: tag
+            value: finance
+          - union: 
+            - method: path
+              value: models/marts/marketing
+            - method: path
+              value: models/marts/finance
+          - exclude: 
+              - method: tag
+                value: test_tag
+
+
+  # - name: selector_example
+  #   description: this is an example of a job
+  #   definition: 
+  #     union: 
+  #       - method: path
+  #         value: models/marts/marketing
+  #       - method: path
+  #         value: models/marts/finance
+  #       - intersection:
+  #         - method: tag
+  #           value: finance          
+  #       - exclude: 
+  #           - method: tag
+  #             value: test_tag
+
+  - name: selector_example2
+    description: this is an example of a job
+    definition: "models/marts/finance,tag:finance"
+
+        # - exclude: 
+        #     - method: tag
+        #       value: test_tag

tests/_samples/audit_helper_compare_all_columns__stg_tpch__orders.sql

@@ -2,7 +2,8 @@
 
 {{
     config(
-        severity = 'warn'
+        severity = 'warn',
+        tags='audit_helper'
     )
 }}
 

tests/generic/audit_helper_compare_queries.sql

@@ -0,0 +1,51 @@
+{% test audit_helper_compare_queries(model, table_b, primary_key, except_columns=none) %}
+{{config(post_hook= "{{centralize_test_failures(results)}}", store_failures = true)}}
+
+{% set column_names = dbt_utils.get_filtered_columns_in_relation(from=model, except=exclude_columns) %}
+
+{% set column_selection %}
+
+  {% for column_name in column_names %} 
+    {{ adapter.quote(column_name) }}{% if not loop.last %},{% endif %} 
+  {% endfor %}
+
+{% endset %}
+
+with a as (
+    select 
+       {{ column_selection }}
+    from {{ model }}
+), 
+
+b as (
+    select 
+        {{ column_selection }}
+    from {{ table_b }}
+),
+
+a_except_b as (
+    select * from a
+    except
+    select * from b
+),
+
+b_except_a as (
+    select * from b
+    except
+    select * from a
+),
+
+union_all as (
+    select 
+    *
+    from a_except_b
+    union all 
+    select 
+    *
+    from b_except_a
+)
+
+select * from union_all
+order by {{ primary_key }}
+
+{%endtest%}

tests/generic/unique_and_not_null.sql

@@ -1,5 +1,5 @@
 {% test unique_and_not_null(model, column_name) %}
-
+{{config(store_failures=true)}}
     with null_records as (
         select {{ column_name }}
         from {{ model }}