Update dev tools (#309)

Update pre-commit hooks: remove isort, pylint add pyupgrade, ruff Remove all pylint disable statements. Run pre-commit for all files until, updating the code base accordingly. Configure ruff similarly as it is configured in OTEAPI Core. Update tests to help them succeed. Comment out use of `EMMO.isDescriptionFor`, which is an incoming concept in EMMO. Co-authored-by: Francesca L. Bleken <[email protected]>
EMMC-ASBL · Oct 21, 2024 · 32697fe · 32697fe
1 parent dbc694d
commit 32697fe
Show file tree

Hide file tree

Showing 38 changed files with 277 additions and 222 deletions.
diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml
@@ -22,10 +22,7 @@ jobs:
       python_version_pylint_safety: "3.9"
 
       # pylint-specific settings
-      run_pylint: true
-      pylint_runs: |
-        --rcfile=pyproject.toml oteapi_dlite
-        --rcfile=pyproject.toml --disable=import-outside-toplevel,redefined-outer-name tests
+      run_pylint: false
 
       # safety-specific settings
       run_safety: true

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -8,56 +8,90 @@ ci:
   autoupdate_branch: 'master'
   autoupdate_commit_msg: '[pre-commit.ci] pre-commit autoupdate'
   autoupdate_schedule: 'weekly'
-  skip: [pylint, pylint-tests]
+  skip: []
   submodules: false
 
 # hooks
 repos:
+  # pre-commit-hooks supplies a multitude of small hooks
+  # To get an overview of them all as well as the ones used here, please see
+  # https://github.com/pre-commit/pre-commit-hooks#hooks-available
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v5.0.0
     hooks:
     - id: end-of-file-fixer
+      exclude: ^CHANGELOG.md$
     - id: debug-statements
     - id: check-yaml
       name: Check YAML
     - id: check-toml
       name: Check TOML
-    - id: requirements-txt-fixer
-      name: Fix requirements*.txt
-      files: ^requirements.*\.txt$
     - id: trailing-whitespace
       args: [--markdown-linebreak-ext=md]
+      exclude: ^CHANGELOG.md$
 
-  - repo: https://github.com/PyCQA/isort
-    rev: 5.13.2
+  # pyupgrade is a tool for automatically upgrading Python syntax for newer versions of
+  # the language
+  # It works on files in-place
+  - repo: https://github.com/asottile/pyupgrade
+    rev: v3.18.0
     hooks:
-    - id: isort
-      args: ["--profile", "black", "--line-length", "80", "--filter-files", "--skip-gitignore"]
+    - id: pyupgrade
+      args: [--py39-plus, --keep-runtime-typing]
 
+  # Black is a code style and formatter
+  # It works on files in-place
   - repo: https://github.com/psf/black
     rev: 24.10.0
     hooks:
     - id: black
-      args: ["--line-length", "80"]
 
+  # ruff is a Python linter, incl. import sorter and formatter
+  # It works partly on files in-place
+  # More information can be found in its documentation:
+  # https://docs.astral.sh/ruff/
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.7.0
+    hooks:
+    - id: ruff
+      # Fix what can be fixed in-place and exit with non-zero status if files were
+      # changed and/or there are rules violations.
+      args:
+      - "--fix"
+      - "--exit-non-zero-on-fix"
+      - "--show-fixes"
+      - "--no-unsafe-fixes"
+
+  # Bandit is a security linter
+  # More information can be found in its documentation:
+  # https://bandit.readthedocs.io/en/latest/
   - repo: https://github.com/PyCQA/bandit
     rev: 1.7.10
     hooks:
     - id: bandit
       args: ["-r"]
       files: ^oteapi_dlite/.*$
 
+  # mypy is a static typing linter
+  # The main code repository can be found at:
+  # https://github.com/python/mypy
+  # The project's documentation can be found at:
+  # https://mypy.readthedocs.io/en/stable/index.html
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.11.2
+    rev: v1.12.0
     hooks:
     - id: mypy
       exclude: ^tests/.*$
       additional_dependencies:
       - "types-requests"
       - "pydantic>=2,<3"
 
+  # ci-cd is a collection of hooks to prepare for CI/CD workflows
+  # It's mainly related to documentation
+  # More information can be found in its documentation:
+  # https://SINTEF.github.io/ci-cd/
   - repo: https://github.com/SINTEF/ci-cd
-    rev: v2.8.2
+    rev: v2.8.3
     hooks:
     - id: docs-api-reference
       args:
@@ -68,28 +102,3 @@ repos:
       args:
       - --replacement=(LICENSE),(LICENSE.md)
       - --replacement=https://EMMC-ASBL.github.io/oteapi-dlite/latest/,
-
-  - repo: local
-    hooks:
-    - id: pylint
-      name: pylint
-      entry: pylint
-      args:
-      - "--rcfile=pyproject.toml"
-      - "--extension-pkg-whitelist='pydantic'"
-      language: python
-      types: [python]
-      require_serial: true
-      files: ^.*$
-      exclude: ^tests/.*$
-    - id: pylint-tests
-      name: pylint - tests
-      entry: pylint
-      args:
-      - "--rcfile=pyproject.toml"
-      - "--extension-pkg-whitelist='pydantic'"
-      - "--disable=import-outside-toplevel,redefined-outer-name,import-error"
-      language: python
-      types: [python]
-      require_serial: true
-      files: ^tests/.*$
diff --git a/oteapi_dlite/__init__.py b/oteapi_dlite/__init__.py
@@ -6,6 +6,8 @@
 Created from cookiecutter-oteapi-plugin, SINTEF, 2022
 """
 
+from __future__ import annotations
+
 __version__ = "0.2.2"
 __author__ = "[email protected]"
 __author_email__ = "[email protected]"
diff --git a/oteapi_dlite/models/__init__.py b/oteapi_dlite/models/__init__.py
@@ -1,5 +1,7 @@
 """DLite-specific data models."""
 
+from __future__ import annotations
+
 from .session import DLiteSessionUpdate
 
 __all__ = ("DLiteSessionUpdate",)
diff --git a/oteapi_dlite/models/session.py b/oteapi_dlite/models/session.py
@@ -1,5 +1,7 @@
 """Pydantic data models for DLite."""
 
+from __future__ import annotations
+
 from typing import Annotated, Optional
 
 from oteapi.models import SessionUpdate

diff --git a/oteapi_dlite/strategies/convert.py b/oteapi_dlite/strategies/convert.py
@@ -3,7 +3,8 @@
 
 """
 
-# pylint: disable=unused-argument
+from __future__ import annotations
+
 import importlib
 from collections.abc import Sequence
 from typing import TYPE_CHECKING, Annotated, Optional
@@ -121,7 +122,7 @@ class DLiteConvertStrategyConfig(AttrDict):
             description="Additional keyword arguments passed "
             "to the convert function.",
         ),
-    ] = {}
+    ] = {}  # noqa: RUF012
 
 
 class DLiteConvertConfig(FunctionConfig):
@@ -148,13 +149,13 @@ class DLiteConvertStrategy:
 
     def initialize(
         self,
-        session: Optional[dict[str, "Any"]] = None,
+        session: Optional[dict[str, Any]] = None,
     ) -> DLiteSessionUpdate:
         """Initialize."""
         return DLiteSessionUpdate(collection_id=get_collection(session).uuid)
 
     def get(
-        self, session: Optional[dict[str, "Any"]] = None
+        self, session: Optional[dict[str, Any]] = None
     ) -> DLiteSessionUpdate:
         """Execute the strategy.
 

diff --git a/oteapi_dlite/strategies/filter.py b/oteapi_dlite/strategies/filter.py
@@ -1,6 +1,7 @@
 """Filter that removes all but specified instances in the collection."""
 
-# pylint: disable=unused-argument
+from __future__ import annotations
+
 import re
 from typing import TYPE_CHECKING, Annotated, Optional
 
@@ -104,16 +105,15 @@ class DLiteFilterStrategy:
 
     def initialize(
         self,
-        session: "Optional[dict[str, Any]]" = None,
+        session: Optional[dict[str, Any]] = None,
     ) -> DLiteSessionUpdate:
         """Initialize."""
         return DLiteSessionUpdate(collection_id=get_collection(session).uuid)
 
     def get(
-        self, session: "Optional[dict[str, Any]]" = None
+        self, session: Optional[dict[str, Any]] = None
     ) -> DLiteSessionUpdate:
         """Execute the strategy."""
-        # pylint: disable=too-many-branches
         config = self.filter_config.configuration
 
         # Alias for query configuration

diff --git a/oteapi_dlite/strategies/generate.py b/oteapi_dlite/strategies/generate.py
@@ -1,8 +1,10 @@
 """Generic generate strategy using DLite storage plugin."""
 
-# pylint: disable=unused-argument,invalid-name,too-many-branches,too-many-locals
+from __future__ import annotations
+
 import os
 import tempfile
+from pathlib import Path
 from typing import TYPE_CHECKING, Annotated, Optional
 
 from oteapi.datacache import DataCache
@@ -263,13 +265,13 @@ class DLiteGenerateStrategy:
 
     def initialize(
         self,
-        session: Optional[dict[str, "Any"]] = None,
+        session: Optional[dict[str, Any]] = None,
     ) -> DLiteSessionUpdate:
         """Initialize."""
         return DLiteSessionUpdate(collection_id=get_collection(session).uuid)
 
     def get(
-        self, session: Optional[dict[str, "Any"]] = None
+        self, session: Optional[dict[str, Any]] = None
     ) -> DLiteSessionUpdate:
         """Execute the strategy.
 
@@ -282,7 +284,6 @@ def get(
         Returns:
             SessionUpdate instance.
         """
-        # pylint: disable=too-many-statements
         config = self.generate_config.configuration
         cacheconfig = config.datacache_config
 
@@ -326,15 +327,14 @@ def get(
                 key = "generate_data"
             cache = DataCache()
             with tempfile.TemporaryDirectory() as tmpdir:
-                inst.save(driver, "{tmpdir}/data", config.options)
-                with open(f"{tmpdir}/data", "rb") as f:
+                inst.save(driver, f"{tmpdir}/data", config.options)
+                with Path(f"{tmpdir}/data").open("rb") as f:
                     cache.add(f.read(), key=key)
 
         # Store documentation of this instance in the knowledge base
         if config.kb_document_class:
 
             # Import here to avoid hard dependencies on tripper.
-            # pylint: disable=import-outside-toplevel
             from tripper import RDF
             from tripper.convert import save_container
 
@@ -484,4 +484,4 @@ def individual_iri(class_iri, base_iri=":", randbytes=6):
         .rsplit("#", 1)[-1]
         .lower()
     )
-    return f"{base_iri}{basename}-{os.urandom(6).hex()}"
+    return f"{base_iri}{basename}-{os.urandom(randbytes).hex()}"
diff --git a/oteapi_dlite/strategies/mapping.py b/oteapi_dlite/strategies/mapping.py
@@ -1,6 +1,7 @@
 """Mapping filter strategy."""
 
-# pylint: disable=unused-argument,invalid-name
+from __future__ import annotations
+
 from typing import TYPE_CHECKING, Annotated, Optional
 
 from oteapi.models import AttrDict, MappingConfig
@@ -53,7 +54,7 @@ class DLiteMappingStrategy:
     mapping_config: DLiteMappingConfig
 
     def initialize(
-        self, session: Optional[dict[str, "Any"]] = None
+        self, session: Optional[dict[str, Any]] = None
     ) -> DLiteSessionUpdate:
         """Initialize strategy."""
         if session is None:
@@ -72,7 +73,7 @@ def initialize(
                         ts.expand_iri(t) if isinstance(t, str) else t
                         for t in triple
                     ]
-                    for triple in self.mapping_config.triples  # pylint: disable=not-an-iterable
+                    for triple in self.mapping_config.triples
                 ]
             )
 
@@ -81,7 +82,7 @@ def initialize(
         return DLiteSessionUpdate(collection_id=coll.uuid)
 
     def get(
-        self, session: Optional[dict[str, "Any"]] = None
+        self, session: Optional[dict[str, Any]] = None
     ) -> DLiteSessionUpdate:
         """Execute strategy and return a dictionary."""
         return DLiteSessionUpdate(collection_id=get_collection(session).uuid)
diff --git a/oteapi_dlite/strategies/parse.py b/oteapi_dlite/strategies/parse.py
@@ -1,6 +1,7 @@
 """Generic parse strategy using DLite storage plugin."""
 
-# pylint: disable=unused-argument
+from __future__ import annotations
+
 from pathlib import Path
 from typing import TYPE_CHECKING, Annotated, Optional
 
@@ -101,13 +102,13 @@ class DLiteParseStrategy:
 
     def initialize(
         self,
-        session: Optional[dict[str, "Any"]] = None,
+        session: Optional[dict[str, Any]] = None,
     ) -> DLiteSessionUpdate:
         """Initialize."""
         return DLiteSessionUpdate(collection_id=get_collection(session).uuid)
 
     def get(
-        self, session: Optional[dict[str, "Any"]] = None
+        self, session: Optional[dict[str, Any]] = None
     ) -> DLiteSessionUpdate:
         """Execute the strategy.