From 0c009bfa0d19a2eeb383ddceb72c4d031c8141c5 Mon Sep 17 00:00:00 2001 From: Jasper Ginn Date: Thu, 21 Mar 2024 13:33:50 +0100 Subject: [PATCH] chore: use local redis when developing --- dags/luchtmeetnet_ingestion/.dockerignore | 1 + .../src/luchtmeetnet_ingestion/IO/resources.py | 5 ++++- .../src/luchtmeetnet_ingestion/resource_definitions.py | 8 ++++---- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/dags/luchtmeetnet_ingestion/.dockerignore b/dags/luchtmeetnet_ingestion/.dockerignore index 3bffedf..af4fa55 100644 --- a/dags/luchtmeetnet_ingestion/.dockerignore +++ b/dags/luchtmeetnet_ingestion/.dockerignore @@ -2,3 +2,4 @@ .justfile Dockerfile tests +*.rdb diff --git a/dags/luchtmeetnet_ingestion/src/luchtmeetnet_ingestion/IO/resources.py b/dags/luchtmeetnet_ingestion/src/luchtmeetnet_ingestion/IO/resources.py index 42ff7a3..290d6ea 100644 --- a/dags/luchtmeetnet_ingestion/src/luchtmeetnet_ingestion/IO/resources.py +++ b/dags/luchtmeetnet_ingestion/src/luchtmeetnet_ingestion/IO/resources.py @@ -12,7 +12,7 @@ from luchtmeetnet_ingestion.luchtmeetnet.api import get_results_luchtmeetnet_endpoint from pydantic import PrivateAttr from pyrate_limiter import BucketFullException, Duration, Limiter, Rate, RedisBucket -from redis import Redis +from redis import ConnectionError, Redis class RedisResource(ConfigurableResource): @@ -32,6 +32,9 @@ def connection(self, context: InitResourceContext): username=self.username, ) yield conn + except ConnectionError as e: + context.log.error(f"Failed to connect to Redis backend: {e}") + raise e finally: conn.close() diff --git a/dags/luchtmeetnet_ingestion/src/luchtmeetnet_ingestion/resource_definitions.py b/dags/luchtmeetnet_ingestion/src/luchtmeetnet_ingestion/resource_definitions.py index bb97ff2..6b86c3e 100644 --- a/dags/luchtmeetnet_ingestion/src/luchtmeetnet_ingestion/resource_definitions.py +++ b/dags/luchtmeetnet_ingestion/src/luchtmeetnet_ingestion/resource_definitions.py @@ -22,10 +22,10 @@ rate_minutes=5, bucket_key=f"luchtmeetnet_api_{environment}", redis=RedisResource( - host=EnvVar("DAGSTER_SECRET_REDIS_HOST"), - port=16564, - password=EnvVar("DAGSTER_SECRET_REDIS_PASSWORD"), - username=EnvVar("DAGSTER_SECRET_REDIS_USERNAME"), + host="localhost" if environment == "dev" else EnvVar("DAGSTER_SECRET_REDIS_HOST"), + port=6379 if environment == "dev" else 16564, + password=None if environment == "dev" else EnvVar("DAGSTER_SECRET_REDIS_PASSWORD"), + username=None if environment == "dev" else EnvVar("DAGSTER_SECRET_REDIS_USERNAME"), ), ) ),