chore: use local redis when developing

dags/luchtmeetnet_ingestion/.dockerignore

@@ -2,3 +2,4 @@
 .justfile
 Dockerfile
 tests
+*.rdb

dags/luchtmeetnet_ingestion/src/luchtmeetnet_ingestion/IO/resources.py

@@ -12,7 +12,7 @@
 from luchtmeetnet_ingestion.luchtmeetnet.api import get_results_luchtmeetnet_endpoint
 from pydantic import PrivateAttr
 from pyrate_limiter import BucketFullException, Duration, Limiter, Rate, RedisBucket
-from redis import Redis
+from redis import ConnectionError, Redis
 
 
 class RedisResource(ConfigurableResource):
@@ -32,6 +32,9 @@ def connection(self, context: InitResourceContext):
                 username=self.username,
             )
             yield conn
+        except ConnectionError as e:
+            context.log.error(f"Failed to connect to Redis backend: {e}")
+            raise e
         finally:
             conn.close()
 

dags/luchtmeetnet_ingestion/src/luchtmeetnet_ingestion/resource_definitions.py

@@ -22,10 +22,10 @@
             rate_minutes=5,
             bucket_key=f"luchtmeetnet_api_{environment}",
             redis=RedisResource(
-                host=EnvVar("DAGSTER_SECRET_REDIS_HOST"),
-                port=16564,
-                password=EnvVar("DAGSTER_SECRET_REDIS_PASSWORD"),
-                username=EnvVar("DAGSTER_SECRET_REDIS_USERNAME"),
+                host="localhost" if environment == "dev" else EnvVar("DAGSTER_SECRET_REDIS_HOST"),
+                port=6379 if environment == "dev" else 16564,
+                password=None if environment == "dev" else EnvVar("DAGSTER_SECRET_REDIS_PASSWORD"),
+                username=None if environment == "dev" else EnvVar("DAGSTER_SECRET_REDIS_USERNAME"),
             ),
         )
     ),