✨(backends) import backends dynamically

We want to automatically discover backends in the data/backends
sub-directories for CLI and LRS usage.
We also now handle import failures gracefully, thus backends with
unmet dependencies are excluded.

CHANGELOG.md

@@ -14,6 +14,10 @@ and this project adheres to
 - Implement xAPI LMS Profile statements validation
 - `EdX` to `xAPI` converters for enrollment events
 - Helm: Add variable ``ingress.hosts``
+- Backends: Add `Writable` and `Listable` interfaces to distinguish supported
+  functionalities among `data` backends
+- Backends: Add `get_backends` function to automatically discover backends
+  for CLI and LRS usage
 
 ### Changed
 

src/ralph/api/routers/health.py

@@ -6,19 +6,18 @@
 from fastapi import APIRouter, status
 from fastapi.responses import JSONResponse
 
-from ralph.backends.conf import backends_settings
+from ralph.backends.loader import get_lrs_backends
 from ralph.backends.lrs.base import BaseAsyncLRSBackend, BaseLRSBackend
 from ralph.conf import settings
-from ralph.utils import await_if_coroutine, get_backend_instance
+from ralph.utils import await_if_coroutine, get_backend_class
 
 logger = logging.getLogger(__name__)
 
 router = APIRouter()
 
-BACKEND_CLIENT: Union[BaseLRSBackend, BaseAsyncLRSBackend] = get_backend_instance(
-    backend_type=backends_settings.BACKENDS.LRS,
-    backend_name=settings.RUNSERVER_BACKEND,
-)
+BACKEND_CLIENT: Union[BaseLRSBackend, BaseAsyncLRSBackend] = get_backend_class(
+    backends=get_lrs_backends(), name=settings.RUNSERVER_BACKEND
+)()
 
 
 @router.get("/__lbheartbeat__")

src/ralph/api/routers/statements.py

@@ -27,9 +27,10 @@
 from ralph.api.auth.user import AuthenticatedUser
 from ralph.api.forwarding import forward_xapi_statements, get_active_xapi_forwardings
 from ralph.api.models import ErrorDetail, LaxStatement
-from ralph.backends.conf import backends_settings
+from ralph.backends.loader import get_lrs_backends
 from ralph.backends.lrs.base import (
     AgentParameters,
+    BaseAsyncLRSBackend,
     BaseLRSBackend,
     RalphStatementsQuery,
 )
@@ -45,7 +46,7 @@
 from ralph.models.xapi.base.common import IRI
 from ralph.utils import (
     await_if_coroutine,
-    get_backend_instance,
+    get_backend_class,
     now,
     statements_are_equivalent,
 )
@@ -58,10 +59,9 @@
 )
 
 
-BACKEND_CLIENT: BaseLRSBackend = get_backend_instance(
-    backend_type=backends_settings.BACKENDS.LRS,
-    backend_name=settings.RUNSERVER_BACKEND,
-)
+BACKEND_CLIENT: Union[BaseLRSBackend, BaseAsyncLRSBackend] = get_backend_class(
+    backends=get_lrs_backends(), name=settings.RUNSERVER_BACKEND
+)()
 
 POST_PUT_RESPONSES = {
     400: {

src/ralph/backends/data/async_mongo.py

@@ -21,6 +21,8 @@
 from ralph.utils import parse_bytes_to_dict
 
 from ..data.base import (
+    AsyncListable,
+    AsyncWritable,
     BaseAsyncDataBackend,
     DataBackendStatus,
     async_enforce_query_checks,
@@ -29,7 +31,7 @@
 logger = logging.getLogger(__name__)
 
 
-class AsyncMongoDataBackend(BaseAsyncDataBackend):
+class AsyncMongoDataBackend(BaseAsyncDataBackend, AsyncWritable, AsyncListable):
     """Async MongoDB data backend."""
 
     name = "async_mongo"

src/ralph/backends/data/base.py

@@ -147,7 +147,6 @@ def list(
 class BaseDataBackend(ABC):
     """Base data backend interface."""
 
-    type = "data"
     name = "base"
     query_model = BaseQuery
     settings_class = BaseDataBackendSettings
@@ -329,7 +328,6 @@ async def list(
 class BaseAsyncDataBackend(ABC):
     """Base async data backend interface."""
 
-    type = "data"
     name = "base"
     query_model = BaseQuery
     settings_class = BaseDataBackendSettings

src/ralph/backends/http/async_lrs.py

@@ -3,28 +3,23 @@
 import asyncio
 import json
 import logging
-from datetime import datetime
 from itertools import chain
-from typing import Iterable, Iterator, List, Literal, Optional, Union
+from typing import Iterable, Iterator, List, Optional, Union
 from urllib.parse import ParseResult, parse_qs, urljoin, urlparse
-from uuid import UUID
 
 from httpx import AsyncClient, HTTPError, HTTPStatusError, RequestError
 from more_itertools import chunked
-from pydantic import AnyHttpUrl, BaseModel, Field, NonNegativeInt, parse_obj_as
+from pydantic import AnyHttpUrl, BaseModel, Field, parse_obj_as
 from pydantic.types import PositiveInt
 
+from ralph.backends.lrs.base import LRSStatementsQuery
 from ralph.conf import BaseSettingsConfig, HeadersParameters
 from ralph.exceptions import BackendException, BackendParameterException
-from ralph.models.xapi.base.agents import BaseXapiAgent
-from ralph.models.xapi.base.common import IRI
-from ralph.models.xapi.base.groups import BaseXapiGroup
 from ralph.utils import gather_with_limited_concurrency
 
 from .base import (
     BaseHTTPBackend,
     BaseHTTPBackendSettings,
-    BaseQuery,
     HTTPBackendStatus,
     OperationType,
     enforce_query_checks,
@@ -72,31 +67,6 @@ class StatementResponse(BaseModel):
     more: Optional[str]
 
 
-class LRSStatementsQuery(BaseQuery):
-    """Pydantic model for LRS query on Statements resource query parameters.
-
-    LRS Specification:
-    https://github.com/adlnet/xAPI-Spec/blob/1.0.3/xAPI-Communication.md#213-get-statements
-    """
-
-    # pylint: disable=too-many-instance-attributes
-
-    statement_id: Optional[str] = Field(None, alias="statementId")
-    voided_statement_id: Optional[str] = Field(None, alias="voidedStatementId")
-    agent: Optional[Union[BaseXapiAgent, BaseXapiGroup]]
-    verb: Optional[IRI]
-    activity: Optional[IRI]
-    registration: Optional[UUID]
-    related_activities: Optional[bool] = False
-    related_agents: Optional[bool] = False
-    since: Optional[datetime]
-    until: Optional[datetime]
-    limit: Optional[NonNegativeInt] = 0
-    format: Optional[Literal["ids", "exact", "canonical"]] = "exact"
-    attachments: Optional[bool] = False
-    ascending: Optional[bool] = False
-
-
 class AsyncLRSHTTPBackend(BaseHTTPBackend):
     """Asynchronous LRS HTTP backend."""
 

src/ralph/backends/http/base.py

@@ -81,7 +81,6 @@ class Config:
 class BaseHTTPBackend(ABC):
     """Base HTTP backend interface."""
 
-    type = "http"
     name = "base"
     query = BaseQuery
 

src/ralph/backends/loader.py

@@ -0,0 +1,124 @@
+"""Ralph backend loader."""
+
+import logging
+import pkgutil
+from functools import lru_cache
+from importlib import import_module
+from importlib.util import find_spec
+from inspect import getmembers, isabstract, isclass
+from typing import Dict, Tuple, Type
+
+from ralph.backends.data.base import (
+    AsyncListable,
+    AsyncWritable,
+    BaseAsyncDataBackend,
+    BaseDataBackend,
+    Listable,
+    Writable,
+)
+from ralph.backends.http.base import BaseHTTPBackend
+from ralph.backends.lrs.base import BaseAsyncLRSBackend, BaseLRSBackend
+from ralph.backends.stream.base import BaseStreamBackend
+
+logger = logging.getLogger(__name__)
+
+
+@lru_cache()
+def get_backends(packages: Tuple[str], base_backends: Tuple[Type]) -> Dict[str, Type]:
+    """Return sub-classes of `base_backends` found in sub-modules of `packages`.
+
+    Args:
+        packages (tuple of str): A tuple of dotted package names.
+            Ex.: ("ralph.backends.data", "ralph.backends.lrs").
+        base_backends (tuple of type): A tuple of base backend classes to search for.
+            Ex.: ("BaseDataBackend",)
+
+    Return:
+        dict: A dictionary of found non-abstract backend classes by their name property.
+            Ex.: {"fs": FSDataBackend}
+    """
+    module_specs = []
+    for package in packages:
+        try:
+            module_spec = find_spec(package)
+        except ModuleNotFoundError:
+            module_spec = None
+
+        if not module_spec:
+            logger.info("Could not find '%s' package; skipping it", package)
+            continue
+
+        module_specs.append(module_spec)
+
+    modules = []
+    for module_spec in module_specs:
+        paths = module_spec.submodule_search_locations
+        for module_info in pkgutil.iter_modules(paths, prefix=f"{module_spec.name}."):
+            modules.append(module_info.name)
+
+    backend_classes = set()
+    for module in modules:
+        try:
+            backend_module = import_module(module)
+        except Exception as error:  # pylint:disable=broad-except
+            logger.info("Failed to import %s module: %s", module, error)
+            continue
+        for _, class_ in getmembers(backend_module, isclass):
+            if issubclass(class_, base_backends) and not isabstract(class_):
+                backend_classes.add(class_)
+
+    return {
+        backend.name: backend
+        for backend in sorted(backend_classes, key=lambda x: x.name, reverse=True)
+    }
+
+
+@lru_cache(maxsize=1)
+def get_cli_backends() -> Dict[str, Type]:
+    """Return Ralph's backend classes for cli usage."""
+    dotted_paths = (
+        "ralph.backends.data",
+        "ralph.backends.http",
+        "ralph.backends.stream",
+    )
+    base_backends = (
+        BaseAsyncDataBackend,
+        BaseDataBackend,
+        BaseHTTPBackend,
+        BaseStreamBackend,
+    )
+    return get_backends(dotted_paths, base_backends)
+
+
+@lru_cache(maxsize=1)
+def get_cli_write_backends() -> Dict[str, Type]:
+    """Return Ralph's backends classes for cli write usage."""
+    backends = get_cli_backends()
+    return {
+        name: backend
+        for name, backend in backends.items()
+        if issubclass(backend, (Writable, AsyncWritable, BaseHTTPBackend))
+    }
+
+
+@lru_cache(maxsize=1)
+def get_cli_list_backends() -> Dict[str, Type]:
+    """Return Ralph's backends classes for cli list usage."""
+    backends = get_cli_backends()
+    return {
+        name: backend
+        for name, backend in backends.items()
+        if issubclass(backend, (Listable, AsyncListable))
+    }
+
+
+@lru_cache(maxsize=1)
+def get_lrs_backends() -> Dict[str, Type]:
+    """Return Ralph's backend classes for LRS usage."""
+    return get_backends(
+        ("ralph.backends.lrs",),
+        (
+            BaseAsyncLRSBackend,
+            BaseLRSBackend,
+        ),
+    )