chore: format and add pyorihect

.justfile

@@ -19,7 +19,7 @@ fmt:
   pants fmt ::
 
 lint:
-  pants lint check ::
+  pants lint :: # check
 
 test:
   pants test ::

.pre-commit-config.yaml

@@ -27,11 +27,11 @@ repos:
         language: system
         types: [python]
         pass_filenames: false
-  - repo: local
-    hooks:
-      - id: pants-check
-        name: pants-check
-        entry: "pants check --changed-since=HEAD"
-        language: system
-        types: [python]
-        pass_filenames: false
+  # - repo: local
+  #   hooks:
+  #     - id: pants-check
+  #       name: pants-check
+  #       entry: "pants check --changed-since=HEAD"
+  #       language: system
+  #       types: [python]
+  #       pass_filenames: false

dags/luchtmeetnet_ingestion/BUILD

@@ -26,7 +26,10 @@ pex_binary(
     layout="packed",
     include_requirements=False,
     include_tools=True,
-    dependencies=[":pyproject", "dags/luchtmeetnet_ingestion/src/luchtmeetnet_ingestion:luchtmeetnet_ingestion"],
+    dependencies=[
+        ":pyproject",
+        "dags/luchtmeetnet_ingestion/src/luchtmeetnet_ingestion:luchtmeetnet_ingestion",
+    ],
     environment="py39_slim",
 )
 
@@ -70,5 +73,5 @@ docker_image(
         "COPY dags.luchtmeetnet_ingestion/binary-srcs.pex/pyproject.toml /pyproject.toml",
         'ENTRYPOINT ["/bin/app/pex"]',
     ],
-    image_tags=["latest"]
+    image_tags=["latest"],
 )

dags/luchtmeetnet_ingestion/src/luchtmeetnet_ingestion/IO/resources.py

@@ -1,7 +1,6 @@
 from typing import Any, Dict, List, Optional
 
 from dagster import ConfigurableResource
-
 from luchtmeetnet_ingestion.luchtmeetnet.api import get_results_luchtmeetnet_endpoint
 
 

dags/luchtmeetnet_ingestion/src/luchtmeetnet_ingestion/__init__.py

@@ -1,9 +1,7 @@
 from importlib import metadata
 
 from dagster import Definitions, FilesystemIOManager
-
 from luchtmeetnet_ingestion.assets import air_quality_data
-from luchtmeetnet_ingestion.IO.duckdb_io_manager import duckdb_parquet_io_manager
 from luchtmeetnet_ingestion.IO.resources import LuchtMeetNetResource
 from luchtmeetnet_ingestion.jobs import ingestion_job
 
@@ -17,7 +15,9 @@
 
 env_resources = {
     "dev": shared_resources
-    | {"landing_zone": FilesystemIOManager()} #{"landing_zone": duckdb_parquet_io_manager.configured({"path": ".tmp/landing_zone"})}
+    | {
+        "landing_zone": FilesystemIOManager()
+    }  # {"landing_zone": duckdb_parquet_io_manager.configured({"path": ".tmp/landing_zone"})}
 }
 
 definition = Definitions(

dags/luchtmeetnet_ingestion/src/luchtmeetnet_ingestion/assets/__init__.py

@@ -1,6 +1,5 @@
 import pandas as pd
-from dagster import AutoMaterializePolicy, asset
-
+from dagster import asset
 from luchtmeetnet_ingestion.IO.resources import LuchtMeetNetResource
 from luchtmeetnet_ingestion.partitions import daily_partition
 
@@ -10,12 +9,16 @@
     compute_kind="duckdb",
     io_manager_key="landing_zone",
     partitions_def=daily_partition,
-    #auto_materialize_policy=AutoMaterializePolicy.eager()#max_materializations_per_minute=None),
+    # auto_materialize_policy=AutoMaterializePolicy.eager()#max_materializations_per_minute=None),
 )
 def air_quality_data(context, luchtmeetnet_api: LuchtMeetNetResource):
     date = context.partition_key
     context.log.debug(f"Fetching data for {date}")
-    rp = {"start": f"{date}T00:00:00", "end": f"{date}T23:59:59", "station_number": "NL01494"}
+    rp = {
+        "start": f"{date}T00:00:00",
+        "end": f"{date}T23:59:59",
+        "station_number": "NL01494",
+    }
     df = pd.DataFrame(luchtmeetnet_api.request("measurements", request_params=rp))
     return df.to_json()
 

dags/luchtmeetnet_ingestion/src/luchtmeetnet_ingestion/jobs.py

@@ -1,5 +1,4 @@
 from dagster import define_asset_job
-
 from luchtmeetnet_ingestion.assets import air_quality_data
 from luchtmeetnet_ingestion.partitions import daily_partition
 

dags/luchtmeetnet_ingestion/src/luchtmeetnet_ingestion/luchtmeetnet/api.py

@@ -3,7 +3,6 @@
 from typing import Any, Dict, List, Optional
 
 import requests  # type: ignore
-
 from luchtmeetnet_ingestion.luchtmeetnet.const import LUCHTMEETNET_BASE_URL
 
 logger = logging.getLogger("dagster_orchestrators.IO.api")

dags/luchtmeetnet_ingestion/src/luchtmeetnet_ingestion/partitions.py

@@ -1,6 +1,4 @@
-from dagster import (
-    DailyPartitionsDefinition
-)
+from dagster import DailyPartitionsDefinition
 
 daily_partition = DailyPartitionsDefinition(
     start_date="2023-12-20", end_offset=0, timezone="Europe/Amsterdam", fmt="%Y-%m-%d"

pyproject.toml

@@ -0,0 +1,35 @@
+[tool.black]
+line-length = 100
+exclude = '''
+^/(
+  (
+      \.eggs         # exclude a few common directories in the
+    | \.git          # root of the project
+    | \.hg
+    | \.mypy_cache
+    | \.venv
+    | _build
+    | build
+    | dist
+  )
+)
+'''
+
+[tool.isort]
+profile = "black"
+#extend_skip = [".md", ".json"]
+#extend_skip = [".tmp", ".venv"]
+
+[tool.mypy]
+exclude = "^tests/"
+ignore_missing_imports = true
+
+[tool.ruff]
+ignore = ["E501"]
+extend-exclude = [
+  "__pycache__",
+  "docs",
+  "site",
+  "src/dagster_orchestrators/__init__.py",
+]
+line-length = 100