master merge for 0.4.1 release

.git-blame-ignore-revs

@@ -0,0 +1,2 @@
+# introduce formatting with black
+c3ddbaa6e61c44a3809e625c802cb4c7632934a3

.github/workflows/lint.yml

@@ -13,11 +13,19 @@ jobs:
     uses: ./.github/workflows/get_docs_changes.yml
 
   run_lint:
-    name: Runs mypy, flake and bandit
+    name: Lint
     needs: get_docs_changes
     if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ["ubuntu-latest", "macos-latest", "windows-latest"]
+        python-version: ["3.8.x", "3.9.x", "3.10.x", "3.11.x"]
 
-    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    runs-on: ${{ matrix.os }}    
 
     steps:
 
@@ -27,34 +35,44 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.10.x"
+          python-version: ${{ matrix.python-version }}
 
       - name: Install Poetry
         uses: snok/install-poetry@v1
         with:
           virtualenvs-create: true
           virtualenvs-in-project: true
-          installer-parallel: true
+          installer-parallel: true         
 
       - name: Load cached venv
         id: cached-poetry-dependencies
         uses: actions/cache@v3
         with:
           path: .venv
-          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
+          key: venv-${{ matrix.os }}-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}
 
       - name: Install dependencies
         # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-        run: poetry install --no-interaction --all-extras --with airflow
-
-      # - name: Install self
-      #   run: poetry install --no-interaction
+        run: poetry install --all-extras --with airflow,providers,pipeline,sentry-sdk
 
-      - name: Run lint
-        run: make lint
+      - name: Run make lint
+        run: |
+          export PATH=$PATH:"/c/Program Files/usr/bin" # needed for Windows        
+          make lint
 
       # - name: print envs
       #   run: |
       #     echo "The GitHub Actor's username is: $GITHUB_ACTOR"
       #     echo "The GitHub repo owner is: $GITHUB_REPOSITORY_OWNER"
       #     echo "The GitHub repo is: $GITHUB_REPOSITORY"
+
+  matrix_job_required_check:
+    name: Lint results
+    needs: run_lint
+    runs-on: ubuntu-latest
+    if: always()
+    steps:
+      - name: Check matrix job results
+        if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
+        run: |
+          echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1      

.github/workflows/test_airflow.yml

@@ -41,7 +41,7 @@ jobs:
           key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-airflow-runner
 
       - name: Install dependencies
-        run: poetry install --no-interaction --with airflow -E duckdb -E parquet
+        run: poetry install --no-interaction --with airflow --with pipeline -E duckdb -E parquet --with sentry-sdk
 
       - run: |
           poetry run pytest tests/helpers/airflow_tests

.github/workflows/test_common.yml

@@ -55,40 +55,67 @@ jobs:
           virtualenvs-in-project: true
           installer-parallel: true
 
-      - name: Load cached venv
-        id: cached-poetry-dependencies
-        uses: actions/cache@v3
-        with:
-          # path: ${{ steps.pip-cache.outputs.dir }}
-          path: .venv
-          key: venv-${{ matrix.os }}-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}
+      # NOTE: do not cache. we want to have a clean state each run and we upgrade depdendencies later
+      # - name: Load cached venv
+      #   id: cached-poetry-dependencies
+      #   uses: actions/cache@v3
+      #   with:
+      #     # path: ${{ steps.pip-cache.outputs.dir }}
+      #     path: .venv
+      #     key: venv-${{ matrix.os }}-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}
+
+      - name: Install dependencies
+        run: poetry install --no-interaction --with sentry-sdk
+
+      - run: |
+          poetry run pytest tests/common tests/normalize tests/reflection tests/sources tests/load/test_dummy_client.py tests/extract/test_extract.py tests/extract/test_sources.py tests/pipeline/test_pipeline_state.py
+        if: runner.os != 'Windows'
+        name: Run common tests with minimum dependencies Linux/MAC
+      - run: |
+          poetry run pytest tests/common tests/normalize tests/reflection tests/sources tests/load/test_dummy_client.py tests/extract/test_extract.py tests/extract/test_sources.py tests/pipeline/test_pipeline_state.py -m "not forked"
+        if: runner.os == 'Windows'
+        name: Run common tests with minimum dependencies Windows
+        shell: cmd
 
-      - name: Install dependencies + sentry
-        run: poetry install --no-interaction  -E parquet -E pydantic && pip install sentry-sdk
+      - name: Install duckdb dependencies
+        run: poetry install --no-interaction -E duckdb --with sentry-sdk
 
       - run: |
-          poetry run pytest tests/common tests/normalize tests/reflection tests/sources
+          poetry run pytest tests/pipeline/test_pipeline.py
         if: runner.os != 'Windows'
-        name: Run tests Linux/MAC
+        name: Run pipeline smoke tests with minimum deps Linux/MAC
       - run: |
-          poetry run pytest tests/common tests/normalize tests/reflection tests/sources -m "not forked"
+          poetry run pytest tests/pipeline/test_pipeline.py
         if: runner.os == 'Windows'
-        name: Run tests Windows
+        name: Run smoke tests with minimum deps Windows
         shell: cmd
 
-      - name: Install extra dependencies
-        run: poetry install --no-interaction -E duckdb -E cli -E parquet -E pydantic
+      - name: Install pipeline dependencies
+        run: poetry install --no-interaction -E duckdb -E cli -E parquet --with sentry-sdk --with pipeline
 
       - run: |
-          poetry run pytest tests/extract tests/pipeline tests/cli/common
+          poetry run pytest tests/extract tests/pipeline tests/libs tests/cli/common tests/destinations
         if: runner.os != 'Windows'
-        name: Run extra tests Linux/MAC
+        name: Run extract and pipeline tests Linux/MAC
       - run: |
-          poetry run pytest tests/extract tests/pipeline tests/cli/common
+          poetry run pytest tests/extract tests/pipeline tests/libs tests/cli/common tests/destinations
         if: runner.os == 'Windows'
-        name: Run extra tests Windows
+        name: Run extract tests Windows
         shell: cmd
 
+      # - name: Install Pydantic 1.0
+      #   run: pip install "pydantic<2"
+
+      # - run: |
+      #     poetry run pytest tests/libs
+      #   if: runner.os != 'Windows'
+      #   name: Run extract and pipeline tests Linux/MAC
+      # - run: |
+      #     poetry run pytest tests/libs
+      #   if: runner.os == 'Windows'
+      #   name: Run extract tests Windows
+      #   shell: cmd
+
   matrix_job_required_check:
     name: Common tests
     needs: run_common

.github/workflows/test_dbt_runner.yml

@@ -68,7 +68,7 @@ jobs:
 
       - name: Install dependencies
         # install dlt with postgres support
-        run: poetry install --no-interaction -E postgres -E dbt
+        run: poetry install --no-interaction -E postgres -E dbt --with sentry-sdk
 
       - run: |
           poetry run pytest tests/helpers/dbt_tests -k '(not venv)'

.github/workflows/test_destination_athena.yml

@@ -9,9 +9,9 @@ on:
   workflow_dispatch:
 
 env:
-  DESTINATION__FILESYSTEM__CREDENTIALS__AWS_ACCESS_KEY_ID: AKIAT4QMVMC4J46G55G4
+  DESTINATION__FILESYSTEM__CREDENTIALS__AWS_ACCESS_KEY_ID: AKIAT4QMVMC4LGORLZOK
   DESTINATION__FILESYSTEM__CREDENTIALS__AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-  DESTINATION__ATHENA__CREDENTIALS__AWS_ACCESS_KEY_ID: AKIAT4QMVMC4J46G55G4
+  DESTINATION__ATHENA__CREDENTIALS__AWS_ACCESS_KEY_ID: AKIAT4QMVMC4LGORLZOK
   DESTINATION__ATHENA__CREDENTIALS__AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
   DESTINATION__ATHENA__CREDENTIALS__REGION_NAME: eu-central-1
   DESTINATION__ATHENA__QUERY_RESULT_BUCKET: s3://dlt-athena-output
@@ -70,7 +70,7 @@ jobs:
 
       - name: Install dependencies
         # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-        run: poetry install --no-interaction -E athena
+        run: poetry install --no-interaction -E athena --with sentry-sdk --with pipeline
 
       - run: |
           poetry run pytest tests/load

.github/workflows/test_destination_athena_iceberg.yml

@@ -9,9 +9,9 @@ on:
   workflow_dispatch:
 
 env:
-  DESTINATION__FILESYSTEM__CREDENTIALS__AWS_ACCESS_KEY_ID: AKIAT4QMVMC4J46G55G4
+  DESTINATION__FILESYSTEM__CREDENTIALS__AWS_ACCESS_KEY_ID: AKIAT4QMVMC4LGORLZOK
   DESTINATION__FILESYSTEM__CREDENTIALS__AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-  DESTINATION__ATHENA__CREDENTIALS__AWS_ACCESS_KEY_ID: AKIAT4QMVMC4J46G55G4
+  DESTINATION__ATHENA__CREDENTIALS__AWS_ACCESS_KEY_ID: AKIAT4QMVMC4LGORLZOK
   DESTINATION__ATHENA__CREDENTIALS__AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
   DESTINATION__ATHENA__CREDENTIALS__REGION_NAME: eu-central-1
   DESTINATION__ATHENA__QUERY_RESULT_BUCKET: s3://dlt-athena-output
@@ -70,7 +70,7 @@ jobs:
 
       - name: Install dependencies
         # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-        run: poetry install --no-interaction -E athena
+        run: poetry install --no-interaction -E  --with sentry-sdk --with pipeline
 
       - run: |
           poetry run pytest tests/load

.github/workflows/test_destination_bigquery.yml

@@ -18,7 +18,7 @@ env:
   CREDENTIALS__REFRESH_TOKEN: ${{ secrets.CREDENTIALS__REFRESH_TOKEN }}
 
   # needed for bigquery staging tests
-  # DESTINATION__FILESYSTEM__CREDENTIALS__AWS_ACCESS_KEY_ID: AKIAT4QMVMC4J46G55G4
+  # DESTINATION__FILESYSTEM__CREDENTIALS__AWS_ACCESS_KEY_ID: AKIAT4QMVMC4LGORLZOK
   # DESTINATION__FILESYSTEM__CREDENTIALS__AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
 
   RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
@@ -79,7 +79,7 @@ jobs:
 
       - name: Install dependencies
         # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-        run: poetry install --no-interaction -E bigquery --with providers -E parquet
+        run: poetry install --no-interaction -E bigquery --with providers -E parquet --with sentry-sdk --with pipeline
 
       # - name: Install self
       #   run: poetry install --no-interaction

.github/workflows/test_destination_mssql.yml

@@ -65,7 +65,7 @@ jobs:
           key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-gcp
 
       - name: Install dependencies
-        run: poetry install --no-interaction -E mssql -E s3 -E gs -E az -E parquet
+        run: poetry install --no-interaction -E mssql -E s3 -E gs -E az -E parquet --with sentry-sdk --with pipeline
 
       - run: |
           poetry run pytest tests/load --ignore tests/load/pipeline/test_dbt_helper.py

.github/workflows/test_destination_qdrant.yml

@@ -59,7 +59,7 @@ jobs:
           key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-gcp
 
       - name: Install dependencies
-        run: poetry install --no-interaction -E qdrant -E parquet
+        run: poetry install --no-interaction -E qdrant -E parquet --with sentry-sdk --with pipeline
       - run: |
           poetry run pytest tests/load/
         if: runner.os != 'Windows'

.github/workflows/test_destination_snowflake.yml

@@ -13,7 +13,7 @@ env:
   CREDENTIALS__PASSWORD: ${{ secrets.PG_PASSWORD }}
 
   # needed for snowflake staging tests
-  DESTINATION__FILESYSTEM__CREDENTIALS__AWS_ACCESS_KEY_ID: AKIAT4QMVMC4J46G55G4
+  DESTINATION__FILESYSTEM__CREDENTIALS__AWS_ACCESS_KEY_ID: AKIAT4QMVMC4LGORLZOK
   DESTINATION__FILESYSTEM__CREDENTIALS__AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
   DESTINATION__FILESYSTEM__CREDENTIALS__PROJECT_ID: chat-analytics-rasa-ci
   DESTINATION__FILESYSTEM__CREDENTIALS__CLIENT_EMAIL: chat-analytics-loader@chat-analytics-rasa-ci.iam.gserviceaccount.com
@@ -71,7 +71,7 @@ jobs:
           key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-gcp
 
       - name: Install dependencies
-        run: poetry install --no-interaction -E snowflake -E s3 -E gs -E az
+        run: poetry install --no-interaction -E snowflake -E s3 -E gs -E az -E parquet --with sentry-sdk --with pipeline
 
       - run: |
           poetry run pytest tests/load

.github/workflows/test_destination_synapse.yml

@@ -5,9 +5,9 @@ on:
     branches:
       - master
       - devel
-      
+
   workflow_dispatch:
-  
+
 env:
   DESTINATION__SYNAPSE__CREDENTIALS: ${{ secrets.SYNAPSE_CREDENTIALS }}
   DESTINATION__SYNAPSE__CREDENTIALS__PASSWORD: ${{ secrets.SYNAPSE_PASSWORD }}
@@ -42,7 +42,7 @@ jobs:
     runs-on: ${{ matrix.os }}
 
     steps:
-      
+
       - name: Check out
         uses: actions/checkout@master
 
@@ -70,7 +70,7 @@ jobs:
           key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-gcp
 
       - name: Install dependencies
-        run: poetry install --no-interaction -E synapse -E s3 -E gs -E az
+        run: poetry install --no-interaction -E synapse -E s3 -E gs -E az --with sentry-sdk --with pipeline
 
       - run: |
           poetry run pytest tests/load --ignore tests/load/pipeline/test_dbt_helper.py

.github/workflows/test_destination_weaviate.yml

@@ -61,7 +61,7 @@ jobs:
           key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-gcp
 
       - name: Install dependencies
-        run: poetry install --no-interaction -E weaviate -E parquet
+        run: poetry install --no-interaction -E weaviate -E parquet --with sentry-sdk --with pipeline
       - run: |
           poetry run pytest tests/load/
         if: runner.os != 'Windows'

.github/workflows/test_destinations.yml

@@ -12,7 +12,7 @@ env:
   DESTINATION__POSTGRES__CREDENTIALS: postgresql://[email protected]:5432/dlt_data
   DESTINATION__DUCKDB__CREDENTIALS: duckdb:///_storage/test_quack.duckdb
   DESTINATION__REDSHIFT__CREDENTIALS: postgresql://[email protected]:5439/dlt_ci
-  DESTINATION__FILESYSTEM__CREDENTIALS__AWS_ACCESS_KEY_ID: AKIAT4QMVMC4J46G55G4
+  DESTINATION__FILESYSTEM__CREDENTIALS__AWS_ACCESS_KEY_ID: AKIAT4QMVMC4LGORLZOK
   DESTINATION__FILESYSTEM__CREDENTIALS__AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
   DESTINATION__FILESYSTEM__CREDENTIALS__AZURE_STORAGE_ACCOUNT_NAME: dltdata
   DESTINATION__FILESYSTEM__CREDENTIALS__AZURE_STORAGE_ACCOUNT_KEY: ${{ secrets.AZURE_STORAGE_ACCOUNT_KEY }}
@@ -22,7 +22,7 @@ env:
   TESTS__R2_AWS_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }}
   TESTS__R2_ENDPOINT_URL: https://9830548e4e4b582989be0811f2a0a97f.r2.cloudflarestorage.com
 
-  # DESTINATION__ATHENA__CREDENTIALS__AWS_ACCESS_KEY_ID: AKIAT4QMVMC4J46G55G4
+  # DESTINATION__ATHENA__CREDENTIALS__AWS_ACCESS_KEY_ID: AKIAT4QMVMC4LGORLZOK
   # DESTINATION__ATHENA__CREDENTIALS__AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
   # DESTINATION__ATHENA__CREDENTIALS__REGION_NAME: eu-central-1
   # DESTINATION__ATHENA__QUERY_RESULT_BUCKET: s3://dlt-athena-output
@@ -87,7 +87,7 @@ jobs:
 
       - name: Install dependencies
         # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-        run: poetry install --no-interaction -E redshift -E gs -E s3 -E az -E parquet -E duckdb -E cli
+        run: poetry install --no-interaction -E redshift -E gs -E s3 -E az -E parquet -E duckdb -E cli --with sentry-sdk --with pipeline
 
       # - name: Install self
       #   run: poetry install --no-interaction

.github/workflows/test_doc_snippets.yml

@@ -63,7 +63,7 @@ jobs:
 
       - name: Install dependencies
         # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-        run: poetry install --no-interaction -E duckdb -E weaviate -E parquet -E qdrant --with docs --without airflow
+        run: poetry install --no-interaction -E duckdb -E weaviate -E parquet -E qdrant --with docs,sentry-sdk --without airflow
 
       - name: Run linter and tests
         run: make test-and-lint-snippets

.github/workflows/test_local_destinations.yml

@@ -84,7 +84,7 @@ jobs:
           key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-local-destinations
 
       - name: Install dependencies
-        run: poetry install --no-interaction -E postgres -E duckdb -E parquet -E filesystem -E cli -E weaviate
+        run: poetry install --no-interaction -E postgres -E duckdb -E parquet -E filesystem -E cli -E weaviate --with sentry-sdk --with pipeline
 
       - run: poetry run pytest tests/load && poetry run pytest tests/cli
         name: Run tests Linux