diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c4d4410a..cdf3a68e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,10 @@ and this project adheres to - Implement xAPI LMS Profile statements validation - `EdX` to `xAPI` converters for enrollment events - Helm: Add variable ``ingress.hosts`` +- Backends: Add `Writable` and `Listable` interfaces to distinguish supported + functionalities among `data` backends +- Backends: Add `get_backends` function to automatically discover backends + for CLI and LRS usage ### Changed diff --git a/src/ralph/api/routers/health.py b/src/ralph/api/routers/health.py index c8ca015f1..de05b9180 100644 --- a/src/ralph/api/routers/health.py +++ b/src/ralph/api/routers/health.py @@ -6,19 +6,18 @@ from fastapi import APIRouter, status from fastapi.responses import JSONResponse -from ralph.backends.conf import backends_settings +from ralph.backends.loader import get_lrs_backends from ralph.backends.lrs.base import BaseAsyncLRSBackend, BaseLRSBackend from ralph.conf import settings -from ralph.utils import await_if_coroutine, get_backend_instance +from ralph.utils import await_if_coroutine, get_backend_class logger = logging.getLogger(__name__) router = APIRouter() -BACKEND_CLIENT: Union[BaseLRSBackend, BaseAsyncLRSBackend] = get_backend_instance( - backend_type=backends_settings.BACKENDS.LRS, - backend_name=settings.RUNSERVER_BACKEND, -) +BACKEND_CLIENT: Union[BaseLRSBackend, BaseAsyncLRSBackend] = get_backend_class( + backends=get_lrs_backends(), name=settings.RUNSERVER_BACKEND +)() @router.get("/__lbheartbeat__") diff --git a/src/ralph/api/routers/statements.py b/src/ralph/api/routers/statements.py index 436f82c52..e65c4b3c4 100644 --- a/src/ralph/api/routers/statements.py +++ b/src/ralph/api/routers/statements.py @@ -27,9 +27,10 @@ from ralph.api.auth.user import AuthenticatedUser from ralph.api.forwarding import forward_xapi_statements, get_active_xapi_forwardings from ralph.api.models import ErrorDetail, LaxStatement -from ralph.backends.conf import backends_settings +from ralph.backends.loader import get_lrs_backends from ralph.backends.lrs.base import ( AgentParameters, + BaseAsyncLRSBackend, BaseLRSBackend, RalphStatementsQuery, ) @@ -45,7 +46,7 @@ from ralph.models.xapi.base.common import IRI from ralph.utils import ( await_if_coroutine, - get_backend_instance, + get_backend_class, now, statements_are_equivalent, ) @@ -58,10 +59,9 @@ ) -BACKEND_CLIENT: BaseLRSBackend = get_backend_instance( - backend_type=backends_settings.BACKENDS.LRS, - backend_name=settings.RUNSERVER_BACKEND, -) +BACKEND_CLIENT: Union[BaseLRSBackend, BaseAsyncLRSBackend] = get_backend_class( + backends=get_lrs_backends(), name=settings.RUNSERVER_BACKEND +)() POST_PUT_RESPONSES = { 400: { diff --git a/src/ralph/backends/conf.py b/src/ralph/backends/conf.py deleted file mode 100644 index acdb3de87..000000000 --- a/src/ralph/backends/conf.py +++ /dev/null @@ -1,91 +0,0 @@ -"""Configurations for Ralph backends.""" - -from pydantic import BaseModel, BaseSettings - -from ralph.backends.data.clickhouse import ClickHouseDataBackendSettings -from ralph.backends.data.es import ESDataBackendSettings -from ralph.backends.data.fs import FSDataBackendSettings -from ralph.backends.data.ldp import LDPDataBackendSettings -from ralph.backends.data.mongo import MongoDataBackendSettings -from ralph.backends.data.s3 import S3DataBackendSettings -from ralph.backends.data.swift import SwiftDataBackendSettings -from ralph.backends.http.async_lrs import LRSHTTPBackendSettings -from ralph.backends.lrs.clickhouse import ClickHouseLRSBackendSettings -from ralph.backends.lrs.fs import FSLRSBackendSettings -from ralph.backends.stream.ws import WSStreamBackendSettings -from ralph.conf import BaseSettingsConfig, core_settings - -# Active Data backend Settings. - - -class DataBackendSettings(BaseModel): - """Pydantic model for data backend configuration settings.""" - - ASYNC_ES: ESDataBackendSettings = ESDataBackendSettings() - ASYNC_MONGO: MongoDataBackendSettings = MongoDataBackendSettings() - CLICKHOUSE: ClickHouseDataBackendSettings = ClickHouseDataBackendSettings() - ES: ESDataBackendSettings = ESDataBackendSettings() - FS: FSDataBackendSettings = FSDataBackendSettings() - LDP: LDPDataBackendSettings = LDPDataBackendSettings() - MONGO: MongoDataBackendSettings = MongoDataBackendSettings() - SWIFT: SwiftDataBackendSettings = SwiftDataBackendSettings() - S3: S3DataBackendSettings = S3DataBackendSettings() - - -# Active HTTP backend Settings. - - -class HTTPBackendSettings(BaseModel): - """Pydantic model for HTTP backend configuration settings.""" - - LRS: LRSHTTPBackendSettings = LRSHTTPBackendSettings() - - -# Active LRS backend Settings. - - -class LRSBackendSettings(BaseModel): - """Pydantic model for LRS compatible backend configuration settings.""" - - ASYNC_ES: ESDataBackendSettings = ESDataBackendSettings() - ASYNC_MONGO: MongoDataBackendSettings = MongoDataBackendSettings() - CLICKHOUSE: ClickHouseLRSBackendSettings = ClickHouseLRSBackendSettings() - ES: ESDataBackendSettings = ESDataBackendSettings() - FS: FSLRSBackendSettings = FSLRSBackendSettings() - MONGO: MongoDataBackendSettings = MongoDataBackendSettings() - - -# Active Stream backend Settings. - - -class StreamBackendSettings(BaseModel): - """Pydantic model for stream backend configuration settings.""" - - WS: WSStreamBackendSettings = WSStreamBackendSettings() - - -# Active backend Settings. - - -class Backends(BaseModel): - """Pydantic model for backends configuration settings.""" - - DATA: DataBackendSettings = DataBackendSettings() - HTTP: HTTPBackendSettings = HTTPBackendSettings() - LRS: LRSBackendSettings = LRSBackendSettings() - STREAM: StreamBackendSettings = StreamBackendSettings() - - -class BackendSettings(BaseSettings): - """Pydantic model for Ralph's backends environment & configuration settings.""" - - class Config(BaseSettingsConfig): - """Pydantic Configuration.""" - - env_file = ".env" - env_file_encoding = core_settings.LOCALE_ENCODING - - BACKENDS: Backends = Backends() - - -backends_settings = BackendSettings() diff --git a/src/ralph/backends/data/async_mongo.py b/src/ralph/backends/data/async_mongo.py index f0a9485b1..da2197301 100644 --- a/src/ralph/backends/data/async_mongo.py +++ b/src/ralph/backends/data/async_mongo.py @@ -21,6 +21,8 @@ from ralph.utils import parse_bytes_to_dict from ..data.base import ( + AsyncListable, + AsyncWritable, BaseAsyncDataBackend, DataBackendStatus, async_enforce_query_checks, @@ -29,7 +31,7 @@ logger = logging.getLogger(__name__) -class AsyncMongoDataBackend(BaseAsyncDataBackend): +class AsyncMongoDataBackend(BaseAsyncDataBackend, AsyncWritable, AsyncListable): """Async MongoDB data backend.""" name = "async_mongo" diff --git a/src/ralph/backends/data/base.py b/src/ralph/backends/data/base.py index e859334c6..f07660e6a 100644 --- a/src/ralph/backends/data/base.py +++ b/src/ralph/backends/data/base.py @@ -147,7 +147,6 @@ def list( class BaseDataBackend(ABC): """Base data backend interface.""" - type = "data" name = "base" query_model = BaseQuery settings_class = BaseDataBackendSettings @@ -329,7 +328,6 @@ async def list( class BaseAsyncDataBackend(ABC): """Base async data backend interface.""" - type = "data" name = "base" query_model = BaseQuery settings_class = BaseDataBackendSettings diff --git a/src/ralph/backends/http/async_lrs.py b/src/ralph/backends/http/async_lrs.py index 8309397a0..d37e6e0aa 100644 --- a/src/ralph/backends/http/async_lrs.py +++ b/src/ralph/backends/http/async_lrs.py @@ -3,28 +3,23 @@ import asyncio import json import logging -from datetime import datetime from itertools import chain -from typing import Iterable, Iterator, List, Literal, Optional, Union +from typing import Iterable, Iterator, List, Optional, Union from urllib.parse import ParseResult, parse_qs, urljoin, urlparse -from uuid import UUID from httpx import AsyncClient, HTTPError, HTTPStatusError, RequestError from more_itertools import chunked -from pydantic import AnyHttpUrl, BaseModel, Field, NonNegativeInt, parse_obj_as +from pydantic import AnyHttpUrl, BaseModel, Field, parse_obj_as from pydantic.types import PositiveInt +from ralph.backends.lrs.base import LRSStatementsQuery from ralph.conf import BaseSettingsConfig, HeadersParameters from ralph.exceptions import BackendException, BackendParameterException -from ralph.models.xapi.base.agents import BaseXapiAgent -from ralph.models.xapi.base.common import IRI -from ralph.models.xapi.base.groups import BaseXapiGroup from ralph.utils import gather_with_limited_concurrency from .base import ( BaseHTTPBackend, BaseHTTPBackendSettings, - BaseQuery, HTTPBackendStatus, OperationType, enforce_query_checks, @@ -72,31 +67,6 @@ class StatementResponse(BaseModel): more: Optional[str] -class LRSStatementsQuery(BaseQuery): - """Pydantic model for LRS query on Statements resource query parameters. - - LRS Specification: - https://github.com/adlnet/xAPI-Spec/blob/1.0.3/xAPI-Communication.md#213-get-statements - """ - - # pylint: disable=too-many-instance-attributes - - statement_id: Optional[str] = Field(None, alias="statementId") - voided_statement_id: Optional[str] = Field(None, alias="voidedStatementId") - agent: Optional[Union[BaseXapiAgent, BaseXapiGroup]] - verb: Optional[IRI] - activity: Optional[IRI] - registration: Optional[UUID] - related_activities: Optional[bool] = False - related_agents: Optional[bool] = False - since: Optional[datetime] - until: Optional[datetime] - limit: Optional[NonNegativeInt] = 0 - format: Optional[Literal["ids", "exact", "canonical"]] = "exact" - attachments: Optional[bool] = False - ascending: Optional[bool] = False - - class AsyncLRSHTTPBackend(BaseHTTPBackend): """Asynchronous LRS HTTP backend.""" diff --git a/src/ralph/backends/http/base.py b/src/ralph/backends/http/base.py index ae5003b35..7d696d855 100644 --- a/src/ralph/backends/http/base.py +++ b/src/ralph/backends/http/base.py @@ -81,7 +81,6 @@ class Config: class BaseHTTPBackend(ABC): """Base HTTP backend interface.""" - type = "http" name = "base" query = BaseQuery diff --git a/src/ralph/backends/loader.py b/src/ralph/backends/loader.py new file mode 100644 index 000000000..252583880 --- /dev/null +++ b/src/ralph/backends/loader.py @@ -0,0 +1,124 @@ +"""Ralph backend loader.""" + +import logging +import pkgutil +from functools import lru_cache +from importlib import import_module +from importlib.util import find_spec +from inspect import getmembers, isabstract, isclass +from typing import Dict, Tuple, Type + +from ralph.backends.data.base import ( + AsyncListable, + AsyncWritable, + BaseAsyncDataBackend, + BaseDataBackend, + Listable, + Writable, +) +from ralph.backends.http.base import BaseHTTPBackend +from ralph.backends.lrs.base import BaseAsyncLRSBackend, BaseLRSBackend +from ralph.backends.stream.base import BaseStreamBackend + +logger = logging.getLogger(__name__) + + +@lru_cache() +def get_backends(packages: Tuple[str], base_backends: Tuple[Type]) -> Dict[str, Type]: + """Return sub-classes of `base_backends` found in sub-modules of `packages`. + + Args: + packages (tuple of str): A tuple of dotted package names. + Ex.: ("ralph.backends.data", "ralph.backends.lrs"). + base_backends (tuple of type): A tuple of base backend classes to search for. + Ex.: ("BaseDataBackend",) + + Return: + dict: A dictionary of found non-abstract backend classes by their name property. + Ex.: {"fs": FSDataBackend} + """ + module_specs = [] + for package in packages: + try: + module_spec = find_spec(package) + except ModuleNotFoundError: + module_spec = None + + if not module_spec: + logger.info("Could not find '%s' package; skipping it", package) + continue + + module_specs.append(module_spec) + + modules = [] + for module_spec in module_specs: + paths = module_spec.submodule_search_locations + for module_info in pkgutil.iter_modules(paths, prefix=f"{module_spec.name}."): + modules.append(module_info.name) + + backend_classes = set() + for module in modules: + try: + backend_module = import_module(module) + except Exception as error: # pylint:disable=broad-except + logger.info("Failed to import %s module: %s", module, error) + continue + for _, class_ in getmembers(backend_module, isclass): + if issubclass(class_, base_backends) and not isabstract(class_): + backend_classes.add(class_) + + return { + backend.name: backend + for backend in sorted(backend_classes, key=lambda x: x.name, reverse=True) + } + + +@lru_cache(maxsize=1) +def get_cli_backends() -> Dict[str, Type]: + """Return Ralph's backend classes for cli usage.""" + dotted_paths = ( + "ralph.backends.data", + "ralph.backends.http", + "ralph.backends.stream", + ) + base_backends = ( + BaseAsyncDataBackend, + BaseDataBackend, + BaseHTTPBackend, + BaseStreamBackend, + ) + return get_backends(dotted_paths, base_backends) + + +@lru_cache(maxsize=1) +def get_cli_write_backends() -> Dict[str, Type]: + """Return Ralph's backends classes for cli write usage.""" + backends = get_cli_backends() + return { + name: backend + for name, backend in backends.items() + if issubclass(backend, (Writable, AsyncWritable, BaseHTTPBackend)) + } + + +@lru_cache(maxsize=1) +def get_cli_list_backends() -> Dict[str, Type]: + """Return Ralph's backends classes for cli list usage.""" + backends = get_cli_backends() + return { + name: backend + for name, backend in backends.items() + if issubclass(backend, (Listable, AsyncListable)) + } + + +@lru_cache(maxsize=1) +def get_lrs_backends() -> Dict[str, Type]: + """Return Ralph's backend classes for LRS usage.""" + return get_backends( + ("ralph.backends.lrs",), + ( + BaseAsyncLRSBackend, + BaseLRSBackend, + ), + ) diff --git a/src/ralph/backends/lrs/base.py b/src/ralph/backends/lrs/base.py index 0cf552f25..52894a326 100644 --- a/src/ralph/backends/lrs/base.py +++ b/src/ralph/backends/lrs/base.py @@ -2,16 +2,21 @@ from abc import abstractmethod from dataclasses import dataclass -from typing import Iterator, List, Optional +from datetime import datetime +from typing import Iterator, List, Literal, Optional, Union +from uuid import UUID -from pydantic import BaseModel +from pydantic import BaseModel, Field, NonNegativeInt from ralph.backends.data.base import ( BaseAsyncDataBackend, BaseDataBackend, BaseDataBackendSettings, + BaseQuery, ) -from ralph.backends.http.async_lrs import LRSStatementsQuery +from ralph.models.xapi.base.agents import BaseXapiAgent +from ralph.models.xapi.base.common import IRI +from ralph.models.xapi.base.groups import BaseXapiGroup class BaseLRSBackendSettings(BaseDataBackendSettings): @@ -27,6 +32,31 @@ class StatementQueryResult: search_after: Optional[str] +class LRSStatementsQuery(BaseQuery): + """Pydantic model for LRS query on Statements resource query parameters. + + LRS Specification: + https://github.com/adlnet/xAPI-Spec/blob/1.0.3/xAPI-Communication.md#213-get-statements + """ + + # pylint: disable=too-many-instance-attributes + + statement_id: Optional[str] = Field(None, alias="statementId") + voided_statement_id: Optional[str] = Field(None, alias="voidedStatementId") + agent: Optional[Union[BaseXapiAgent, BaseXapiGroup]] + verb: Optional[IRI] + activity: Optional[IRI] + registration: Optional[UUID] + related_activities: Optional[bool] = False + related_agents: Optional[bool] = False + since: Optional[datetime] + until: Optional[datetime] + limit: Optional[NonNegativeInt] = 0 + format: Optional[Literal["ids", "exact", "canonical"]] = "exact" + attachments: Optional[bool] = False + ascending: Optional[bool] = False + + class AgentParameters(BaseModel): """LRS query parameters for query on type Agent. @@ -53,7 +83,6 @@ class RalphStatementsQuery(LRSStatementsQuery): class BaseLRSBackend(BaseDataBackend): """Base LRS backend interface.""" - type = "lrs" settings_class = BaseLRSBackendSettings @abstractmethod @@ -68,7 +97,6 @@ def query_statements_by_ids(self, ids: List[str]) -> Iterator[dict]: class BaseAsyncLRSBackend(BaseAsyncDataBackend): """Base async LRS backend interface.""" - type = "lrs" settings_class = BaseLRSBackendSettings @abstractmethod diff --git a/src/ralph/backends/stream/base.py b/src/ralph/backends/stream/base.py index 008e68d81..8f3c732b6 100644 --- a/src/ralph/backends/stream/base.py +++ b/src/ralph/backends/stream/base.py @@ -22,7 +22,6 @@ class Config(BaseSettingsConfig): class BaseStreamBackend(ABC): """Base stream backend interface.""" - type = "stream" name = "base" settings_class = BaseStreamBackendSettings diff --git a/src/ralph/cli.py b/src/ralph/cli.py index 782944c34..aa0a3fa54 100644 --- a/src/ralph/cli.py +++ b/src/ralph/cli.py @@ -7,7 +7,7 @@ from inspect import isasyncgen, isclass, iscoroutinefunction from pathlib import Path from tempfile import NamedTemporaryFile -from typing import Any, Optional, Sequence +from typing import Any, Dict, Optional, Type import bcrypt @@ -26,11 +26,21 @@ # dependencies are not installed. pass from click_option_group import optgroup -from pydantic import BaseModel from ralph import __version__ as ralph_version -from ralph.backends.conf import backends_settings -from ralph.backends.data.base import BaseOperationType +from ralph.backends.data.base import ( + BaseAsyncDataBackend, + BaseDataBackend, + BaseOperationType, +) +from ralph.backends.http.base import BaseHTTPBackend +from ralph.backends.loader import ( + get_cli_backends, + get_cli_list_backends, + get_cli_write_backends, + get_lrs_backends, +) +from ralph.backends.stream.base import BaseStreamBackend from ralph.conf import ClientOptions, CommaSeparatedTuple, HeadersParameters, settings from ralph.exceptions import UnsupportedBackendException from ralph.logger import configure_logging @@ -39,8 +49,8 @@ from ralph.models.validator import Validator from ralph.utils import ( execute_async, + get_backend_class, get_backend_instance, - get_backend_type, get_root_logger, import_string, iter_over_async, @@ -200,27 +210,22 @@ def cli(verbosity=None): handler.setLevel(level) -def backends_options(name=None, backend_types: Optional[Sequence[BaseModel]] = None): +# Once we have a base backend interface we could use Dict[str, Type[BaseBackend]] +def backends_options(backends: Dict[str, Type], name: Optional[str] = None): """Backend-related options decorator for Ralph commands.""" def wrapper(command): backend_names = [] - for backend_name, backend in sorted( - [ - name_backend - for backend_type in backend_types - for name_backend in backend_type - ], - key=lambda x: x[0], - reverse=True, - ): - backend_name = backend_name.lower() + for backend_name, backend in backends.items(): backend_names.append(backend_name) - for field_name, field in sorted(backend, key=lambda x: x[0], reverse=True): - field_type = backend.__fields__[field_name].type_ + fields = backend.settings_class.__fields__.items() + for field_name, field in sorted(fields, key=lambda x: x[0], reverse=True): + field_type = field.type_ field_name = f"{backend_name}-{field_name.lower()}".replace("_", "-") option = f"--{field_name}" - option_kwargs = {} + option_kwargs = {"default": None} + if field.default: + option_kwargs["type"] = type(field.default) # If the field is a boolean, convert it to a flag option if field_type is bool: option = f"{option}/--no-{field_name}" @@ -236,9 +241,7 @@ def wrapper(command): elif field_type is Path: option_kwargs["type"] = click.Path() - command = optgroup.option( - option.lower(), default=field, **option_kwargs - )(command) + command = optgroup.option(option.lower(), **option_kwargs)(command) command = (optgroup.group(f"{backend_name} backend"))(command) @@ -249,9 +252,7 @@ def wrapper(command): required=True, help="Backend", )(command) - - command = (cli.command(name=name or command.__name__))(command) - return command + return (cli.command(name=name or command.__name__))(command) return wrapper @@ -564,15 +565,8 @@ def convert(from_, to_, ignore_errors, fail_on_unknown, **conversion_set_kwargs) click.echo(event) -read_backend_types = [ - backends_settings.BACKENDS.DATA, - backends_settings.BACKENDS.HTTP, - backends_settings.BACKENDS.STREAM, -] - - @click.argument("archive", required=False) -@backends_options(backend_types=read_backend_types) +@backends_options(get_cli_backends()) @click.option( "-c", "--chunk-size", @@ -625,10 +619,10 @@ def read( ) logger.debug("Backend parameters: %s", options) - backend_type = get_backend_type(read_backend_types, backend) - backend = get_backend_instance(backend_type, backend, options) + backend_class = get_backend_class(get_cli_backends(), backend) + backend = get_backend_instance(backend_class, options) - if backend_type == backends_settings.BACKENDS.DATA: + if isinstance(backend, (BaseDataBackend, BaseAsyncDataBackend)): statements = backend.read( query=query, target=target, @@ -641,9 +635,9 @@ def read( ) for statement in statements: click.echo(statement) - elif backend_type == backends_settings.BACKENDS.STREAM: + elif isinstance(backend, BaseStreamBackend): backend.stream(sys.stdout.buffer) - elif backend_type == backends_settings.BACKENDS.HTTP: + elif isinstance(backend, BaseHTTPBackend): if query is not None: query = backend.query(query=query) for statement in backend.read( @@ -655,20 +649,13 @@ def read( encoding="utf-8", ) ) - elif backend_type is None: + else: msg = "Cannot find an implemented backend type for backend %s" logger.error(msg, backend) raise UnsupportedBackendException(msg, backend) -write_backend_types = [ - backends_settings.BACKENDS.DATA, - backends_settings.BACKENDS.HTTP, -] - - -# pylint: disable=unnecessary-direct-lambda-call, too-many-arguments -@backends_options(backend_types=write_backend_types) +@backends_options(get_cli_write_backends()) @click.option( "-c", "--chunk-size", @@ -725,6 +712,7 @@ def write( **options, ): """Write an archive to a configured backend.""" + # pylint: disable=too-many-arguments logger.info("Writing to target %s for the configured %s backend", target, backend) logger.debug("Backend parameters: %s", options) @@ -732,10 +720,10 @@ def write( if max_num_simultaneous == 1: max_num_simultaneous = None - backend_type = get_backend_type(write_backend_types, backend) - backend = get_backend_instance(backend_type, backend, options) + backend_class = get_backend_class(get_cli_write_backends(), backend) + backend = get_backend_instance(backend_class, options) - if backend_type == backends_settings.BACKENDS.DATA: + if isinstance(backend, (BaseDataBackend, BaseAsyncDataBackend)): writer = ( execute_async(backend.write) if iscoroutinefunction(backend.write) @@ -750,7 +738,7 @@ def write( if force else BaseOperationType.INDEX, ) - elif backend_type == backends_settings.BACKENDS.HTTP: + elif isinstance(backend, BaseHTTPBackend): backend.write( target=target, data=sys.stdin.buffer, @@ -759,16 +747,13 @@ def write( simultaneous=simultaneous, max_num_simultaneous=max_num_simultaneous, ) - elif backend_type is None: + else: msg = "Cannot find an implemented backend type for backend %s" logger.error(msg, backend) raise UnsupportedBackendException(msg, backend) -list_backend_types = [backends_settings.BACKENDS.DATA] - - -@backends_options(name="list", backend_types=list_backend_types) +@backends_options(get_cli_list_backends(), name="list") @click.option( "-t", "--target", @@ -795,8 +780,8 @@ def list_(target, details, new, backend, **options): logger.debug("Fetch details: %s", str(details)) logger.debug("Backend parameters: %s", options) - backend_type = get_backend_type(list_backend_types, backend) - backend = get_backend_instance(backend_type, backend, options) + backend_class = get_backend_class(get_cli_list_backends(), backend) + backend = get_backend_instance(backend_class, options) documents = backend.list(target=target, details=details, new=new) documents = iter_over_async(documents) if isasyncgen(documents) else documents @@ -809,10 +794,7 @@ def list_(target, details, new, backend, **options): logger.warning("Configured %s backend contains no document", backend.name) -runserver_backend_types = [backends_settings.BACKENDS.LRS] - - -@backends_options(name="runserver", backend_types=runserver_backend_types) +@backends_options(get_lrs_backends(), name="runserver") @click.option( "-h", "--host", @@ -852,7 +834,7 @@ def runserver(backend: str, host: str, port: int, **options): if value is None: continue backend_name, field_name = key.split(sep="_", maxsplit=1) - key = f"RALPH_BACKENDS__LRS__{backend_name}__{field_name}".upper() + key = f"RALPH_BACKENDS__DATA__{backend_name}__{field_name}".upper() if isinstance(value, tuple): value = ",".join(value) if issubclass(type(value), ClientOptions): @@ -877,10 +859,10 @@ def runserver(backend: str, host: str, port: int, **options): log_level="debug", reload=True, ) - except NameError as err: # pylint: disable=redefined-outer-name + except NameError as error: raise ModuleNotFoundError( "You need to install 'lrs' optional dependencies to use the runserver " "command: pip install ralph-malph[lrs]" - ) from err + ) from error logger.info("Shutting down uvicorn server.") diff --git a/src/ralph/conf.py b/src/ralph/conf.py index 5b415fa7f..c30e90843 100644 --- a/src/ralph/conf.py +++ b/src/ralph/conf.py @@ -192,7 +192,7 @@ class AuthBackends(Enum): RUNSERVER_AUTH_OIDC_AUDIENCE: str = None RUNSERVER_AUTH_OIDC_ISSUER_URI: AnyHttpUrl = None RUNSERVER_BACKEND: Literal[ - "async_es", "async_mongo", "clickhouse", "es", "mongo" + "async_es", "async_mongo", "clickhouse", "es", "fs", "mongo" ] = "es" RUNSERVER_HOST: str = "0.0.0.0" # nosec RUNSERVER_MAX_SEARCH_HITS_COUNT: int = 100 diff --git a/src/ralph/utils.py b/src/ralph/utils.py index 1e715f206..89b12f538 100644 --- a/src/ralph/utils.py +++ b/src/ralph/utils.py @@ -9,11 +9,11 @@ from importlib import import_module from inspect import getmembers, isclass, iscoroutine from logging import Logger, getLogger -from typing import Any, Dict, Iterable, Iterator, List, Optional, Sequence, Union +from typing import Any, Dict, Iterable, Iterator, List, Optional, Sequence, Type, Union -from pydantic import BaseModel +from ralph.exceptions import BackendException, UnsupportedBackendException -from ralph.exceptions import BackendException +logger = logging.getLogger(__name__) def import_subclass(dotted_path: str, parent_class: Any) -> Any: @@ -56,62 +56,28 @@ def import_string(dotted_path: str) -> Any: ) from err -def get_backend_type( - backend_types: List[BaseModel], backend_name: str -) -> Union[BaseModel, None]: - """Return the backend type from a backend name.""" - backend_name = backend_name.upper() - for backend_type in backend_types: - if hasattr(backend_type, backend_name): - return backend_type - return None - - -def get_backend_class(backend_type: BaseModel, backend_name: str) -> Any: - """Return the backend class given the backend type and backend name.""" - # Get type name from backend_type class name - backend_type_name = backend_type.__class__.__name__[ - : -len("BackendSettings") - ].lower() - backend_name = backend_name.lower() - - module = import_module(f"ralph.backends.{backend_type_name}.{backend_name}") - for _, class_ in getmembers(module, isclass): - if ( - getattr(class_, "type", None) == backend_type_name - and getattr(class_, "name", None) == backend_name - ): - backend_class = class_ - break - +def get_backend_class(backends: Dict[str, Type], name: str) -> Any: + """Return the backend class from available backends by it's name.""" + backend_class = backends.get(name) if not backend_class: - raise BackendException( - f'No backend named "{backend_name}" ' - f'under the backend type "{backend_type_name}"' - ) - + msg = "'%s' backend not found in available backends: %s" + available_backends = ", ".join(backends.keys()) + logger.error(msg, name, available_backends) + raise UnsupportedBackendException(msg % (name, available_backends)) return backend_class -def get_backend_instance( - backend_type: BaseModel, - backend_name: str, - options: Optional[Dict] = None, -) -> Any: +def get_backend_instance(backend_class: Type, options: Dict) -> Any: """Return the instantiated backend given the backend type, name and options.""" - backend_class = get_backend_class(backend_type, backend_name) - backend_settings = getattr(backend_type, backend_name.upper()) - - if not options: - return backend_class(backend_settings) - - prefix = f"{backend_name}_" + prefix = f"{backend_class.name}_" # Filter backend-related parameters. Parameter name is supposed to start # with the backend name - names = filter(lambda key: key.startswith(prefix), options.keys()) - options = {name.replace(prefix, "").upper(): options[name] for name in names} - - return backend_class(backend_settings.__class__(**options)) + options = { + name.replace(prefix, "").upper(): value + for name, value in options.items() + if name.startswith(prefix) and value is not None + } + return backend_class(backend_class.settings_class(**options)) def get_root_logger() -> Logger: diff --git a/tests/backends/data/test_async_es.py b/tests/backends/data/test_async_es.py index a18745897..704253787 100644 --- a/tests/backends/data/test_async_es.py +++ b/tests/backends/data/test_async_es.py @@ -72,6 +72,14 @@ async def test_backends_data_async_es_data_backend_default_instantiation( assert elasticsearch_node.host == "localhost" assert elasticsearch_node.port == 9200 + # Test overriding default values with environment variables. + monkeypatch.setenv( + "RALPH_BACKENDS__DATA__ES__CLIENT_OPTIONS__verify_certs", + True, + ) + backend = AsyncESDataBackend() + assert backend.settings.CLIENT_OPTIONS == ESClientOptions(verify_certs=True) + @pytest.mark.anyio async def test_backends_data_async_es_data_backend_instantiation_with_settings(): diff --git a/tests/backends/data/test_async_mongo.py b/tests/backends/data/test_async_mongo.py index 3f916b449..842bb44cc 100644 --- a/tests/backends/data/test_async_mongo.py +++ b/tests/backends/data/test_async_mongo.py @@ -55,6 +55,12 @@ async def test_backends_data_async_mongo_data_backend_default_instantiation( assert backend.settings.DEFAULT_CHUNK_SIZE == 500 assert backend.settings.LOCALE_ENCODING == "utf8" + # Test overriding default values with environment variables. + monkeypatch.setenv("RALPH_BACKENDS__DATA__MONGO__CLIENT_OPTIONS__tz_aware", True) + backend = AsyncMongoDataBackend() + assert backend.settings.CLIENT_OPTIONS == MongoClientOptions(tz_aware=True) + await backend.close() + @pytest.mark.anyio async def test_backends_data_async_mongo_data_backend_instantiation_with_settings( diff --git a/tests/backends/data/test_clickhouse.py b/tests/backends/data/test_clickhouse.py index 6d5bc4f40..b051f7544 100644 --- a/tests/backends/data/test_clickhouse.py +++ b/tests/backends/data/test_clickhouse.py @@ -11,6 +11,7 @@ from ralph.backends.data.base import BaseOperationType, DataBackendStatus from ralph.backends.data.clickhouse import ( + ClickHouseClientOptions, ClickHouseDataBackend, ClickHouseDataBackendSettings, ClickHouseQuery, @@ -48,10 +49,20 @@ def test_backends_data_clickhouse_data_backend_default_instantiation(monkeypatch assert ClickHouseDataBackend.default_operation_type == BaseOperationType.CREATE assert ClickHouseDataBackend.settings_class == ClickHouseDataBackendSettings backend = ClickHouseDataBackend() + assert backend.settings.CLIENT_OPTIONS == ClickHouseClientOptions() assert backend.event_table_name == "xapi_events_all" assert backend.default_chunk_size == 500 assert backend.locale_encoding == "utf8" - backend.close() + + # Test overriding default values with environment variables. + monkeypatch.setenv( + "RALPH_BACKENDS__DATA__CLICKHOUSE__CLIENT_OPTIONS__date_time_input_format", + "no_effort", + ) + backend = ClickHouseDataBackend() + assert backend.settings.CLIENT_OPTIONS == ClickHouseClientOptions( + date_time_input_format="no_effort" + ) def test_backends_data_clickhouse_data_backend_instantiation_with_settings(): diff --git a/tests/backends/data/test_es.py b/tests/backends/data/test_es.py index 0e3f06072..3a5c6c56a 100644 --- a/tests/backends/data/test_es.py +++ b/tests/backends/data/test_es.py @@ -71,6 +71,14 @@ def test_backends_data_es_data_backend_default_instantiation(monkeypatch, fs): assert elasticsearch_node.host == "localhost" assert elasticsearch_node.port == 9200 + # Test overriding default values with environment variables. + monkeypatch.setenv( + "RALPH_BACKENDS__DATA__ES__CLIENT_OPTIONS__verify_certs", + True, + ) + backend = ESDataBackend() + assert backend.settings.CLIENT_OPTIONS == ESClientOptions(verify_certs=True) + def test_backends_data_es_data_backend_instantiation_with_settings(): """Test the `ESDataBackend` instantiation with settings.""" diff --git a/tests/backends/data/test_fs.py b/tests/backends/data/test_fs.py index 9d6133e72..0ed237f59 100644 --- a/tests/backends/data/test_fs.py +++ b/tests/backends/data/test_fs.py @@ -37,6 +37,11 @@ def test_backends_data_fs_data_backend_default_instantiation(monkeypatch, fs): assert backend.default_chunk_size == 4096 assert backend.locale_encoding == "utf8" + # Test overriding default values with environment variables. + monkeypatch.setenv("RALPH_BACKENDS__DATA__FS__DEFAULT_CHUNK_SIZE", 1) + backend = FSDataBackend() + assert backend.default_chunk_size == 1 + def test_backends_data_fs_data_backend_instantiation_with_settings(fs): """Test the `FSDataBackend` instantiation with settings.""" diff --git a/tests/backends/data/test_ldp.py b/tests/backends/data/test_ldp.py index 1178ecbfc..c153a5369 100644 --- a/tests/backends/data/test_ldp.py +++ b/tests/backends/data/test_ldp.py @@ -44,6 +44,11 @@ def test_backends_data_ldp_data_backend_default_instantiation(monkeypatch, fs): assert backend.stream_id is None assert backend.timeout is None + # Test overriding default values with environment variables. + monkeypatch.setenv("RALPH_BACKENDS__DATA__LDP__SERVICE_NAME", "foo") + backend = LDPDataBackend() + assert backend.service_name == "foo" + def test_backends_data_ldp_data_backend_instantiation_with_settings(ldp_backend): """Test the `LDPDataBackend` instantiation with settings.""" diff --git a/tests/backends/data/test_mongo.py b/tests/backends/data/test_mongo.py index 2c19b2220..8c4a9a517 100644 --- a/tests/backends/data/test_mongo.py +++ b/tests/backends/data/test_mongo.py @@ -51,6 +51,11 @@ def test_backends_data_mongo_data_backend_default_instantiation(monkeypatch, fs) assert backend.settings.CLIENT_OPTIONS == MongoClientOptions() assert backend.settings.DEFAULT_CHUNK_SIZE == 500 assert backend.settings.LOCALE_ENCODING == "utf8" + + # Test overriding default values with environment variables. + monkeypatch.setenv("RALPH_BACKENDS__DATA__MONGO__CLIENT_OPTIONS__tz_aware", True) + backend = MongoDataBackend() + assert backend.settings.CLIENT_OPTIONS == MongoClientOptions(tz_aware=True) backend.close() diff --git a/tests/backends/data/test_s3.py b/tests/backends/data/test_s3.py index 67ac83953..7230e1bbb 100644 --- a/tests/backends/data/test_s3.py +++ b/tests/backends/data/test_s3.py @@ -41,6 +41,11 @@ def test_backends_data_s3_backend_default_instantiation( assert backend.default_chunk_size == 4096 assert backend.locale_encoding == "utf8" + # Test overriding default values with environment variables. + monkeypatch.setenv("RALPH_BACKENDS__DATA__S3__DEFAULT_CHUNK_SIZE", 1) + backend = S3DataBackend() + assert backend.default_chunk_size == 1 + def test_backends_data_s3_data_backend_instantiation_with_settings(): """Test the `S3DataBackend` instantiation with settings.""" diff --git a/tests/backends/data/test_swift.py b/tests/backends/data/test_swift.py index f0f8fa67b..6ba7be346 100644 --- a/tests/backends/data/test_swift.py +++ b/tests/backends/data/test_swift.py @@ -51,6 +51,11 @@ def test_backends_data_swift_data_backend_default_instantiation(monkeypatch, fs) assert backend.options["user_domain_name"] == "Default" assert backend.default_container is None assert backend.locale_encoding == "utf8" + + # Test overriding default values with environment variables. + monkeypatch.setenv("RALPH_BACKENDS__DATA__SWIFT__DEFAULT_CONTAINER", "foo") + backend = SwiftDataBackend() + assert backend.default_container == "foo" backend.close() diff --git a/tests/backends/http/test_async_lrs.py b/tests/backends/http/test_async_lrs.py index 371f0c48f..e56b4fa9e 100644 --- a/tests/backends/http/test_async_lrs.py +++ b/tests/backends/http/test_async_lrs.py @@ -17,10 +17,10 @@ AsyncLRSHTTPBackend, LRSHeaders, LRSHTTPBackendSettings, - LRSStatementsQuery, OperationType, ) from ralph.backends.http.base import HTTPBackendStatus +from ralph.backends.lrs.base import LRSStatementsQuery from ralph.exceptions import BackendException, BackendParameterException from ...helpers import mock_statement @@ -62,6 +62,11 @@ def test_backend_http_lrs_default_instantiation( assert backend.settings.STATUS_ENDPOINT == "/__heartbeat__" assert backend.settings.STATEMENTS_ENDPOINT == "/xAPI/statements" + # Test overriding default values with environment variables. + monkeypatch.setenv("RALPH_BACKENDS__HTTP__LRS__USERNAME", "foo") + backend = AsyncLRSHTTPBackend() + assert backend.auth == ("foo", "secret") + def test_backends_http_lrs_http_instantiation(): """Test the LRS backend default instantiation.""" diff --git a/tests/backends/http/test_lrs.py b/tests/backends/http/test_lrs.py index b5799d6e4..616f8cbc8 100644 --- a/tests/backends/http/test_lrs.py +++ b/tests/backends/http/test_lrs.py @@ -12,9 +12,9 @@ HTTPBackendStatus, LRSHeaders, LRSHTTPBackendSettings, - LRSStatementsQuery, ) from ralph.backends.http.lrs import LRSHTTPBackend +from ralph.backends.lrs.base import LRSStatementsQuery @pytest.mark.anyio @@ -86,6 +86,11 @@ def test_backend_http_lrs_default_instantiation( assert backend.settings.STATUS_ENDPOINT == "/__heartbeat__" assert backend.settings.STATEMENTS_ENDPOINT == "/xAPI/statements" + # Test overriding default values with environment variables. + monkeypatch.setenv("RALPH_BACKENDS__HTTP__LRS__USERNAME", "foo") + backend = LRSHTTPBackend() + assert backend.auth == ("foo", "secret") + def test_backends_http_lrs_http_instantiation(): """Test the LRS backend default instantiation.""" diff --git a/tests/backends/test_conf.py b/tests/backends/test_conf.py deleted file mode 100644 index aa4a6dd09..000000000 --- a/tests/backends/test_conf.py +++ /dev/null @@ -1,144 +0,0 @@ -"""Tests for Ralph's backends configuration loading.""" - -from pathlib import PosixPath - -import pytest -from pydantic import ValidationError - -from ralph.backends.conf import Backends, BackendSettings, DataBackendSettings -from ralph.backends.data.es import ESDataBackendSettings - - -def test_conf_settings_field_value_priority(fs, monkeypatch): - """Test that the BackendSettings object field values are defined in the following - descending order of priority: - - 1. Arguments passed to the initializer. - 2. Environment variables. - 3. Dotenv variables (.env) - 4. Default values. - """ - # pylint: disable=invalid-name - - # 4. Using default value. - assert str(BackendSettings().BACKENDS.DATA.ES.LOCALE_ENCODING) == "utf8" - - # 3. Using dotenv variables (overrides default value). - fs.create_file(".env", contents="RALPH_BACKENDS__DATA__ES__LOCALE_ENCODING=toto\n") - assert str(BackendSettings().BACKENDS.DATA.ES.LOCALE_ENCODING) == "toto" - - # 2. Using environment variable value (overrides dotenv value). - monkeypatch.setenv("RALPH_BACKENDS__DATA__ES__LOCALE_ENCODING", "foo") - assert str(BackendSettings().BACKENDS.DATA.ES.LOCALE_ENCODING) == "foo" - - # 1. Using argument value (overrides environment value). - assert ( - str( - BackendSettings( - BACKENDS=Backends( - DATA=DataBackendSettings( - ES=ESDataBackendSettings(LOCALE_ENCODING="bar") - ) - ) - ).BACKENDS.DATA.ES.LOCALE_ENCODING - ) - == "bar" - ) - - -@pytest.mark.parametrize( - "ca_certs,verify_certs,expected", - [ - ("/path", "True", {"ca_certs": PosixPath("/path"), "verify_certs": True}), - ("/path2", "f", {"ca_certs": PosixPath("/path2"), "verify_certs": False}), - (None, None, {"ca_certs": None, "verify_certs": None}), - ], -) -def test_conf_es_client_options_with_valid_values( - ca_certs, verify_certs, expected, monkeypatch -): - """Test the ESClientOptions pydantic data type with valid values.""" - # Using None here as in "not set by user" - if ca_certs is not None: - monkeypatch.setenv( - "RALPH_BACKENDS__DATA__ES__CLIENT_OPTIONS__ca_certs", f"{ca_certs}" - ) - # Using None here as in "not set by user" - if verify_certs is not None: - monkeypatch.setenv( - "RALPH_BACKENDS__DATA__ES__CLIENT_OPTIONS__verify_certs", - f"{verify_certs}", - ) - assert BackendSettings().BACKENDS.DATA.ES.CLIENT_OPTIONS.dict() == expected - - -@pytest.mark.parametrize( - "ca_certs,verify_certs", - [ - ("/path", 3), - ("/path", None), - ], -) -def test_conf_es_client_options_with_invalid_values( - ca_certs, verify_certs, monkeypatch -): - """Test the ESClientOptions pydantic data type with invalid values.""" - monkeypatch.setenv( - "RALPH_BACKENDS__DATA__ES__CLIENT_OPTIONS__ca_certs", f"{ca_certs}" - ) - monkeypatch.setenv( - "RALPH_BACKENDS__DATA__ES__CLIENT_OPTIONS__verify_certs", - f"{verify_certs}", - ) - with pytest.raises(ValidationError, match="1 validation error for"): - BackendSettings().BACKENDS.DATA.ES.CLIENT_OPTIONS.dict() - - -@pytest.mark.parametrize( - "document_class,tz_aware,expected", - [ - ("dict", "True", {"document_class": "dict", "tz_aware": True}), - ("str", "f", {"document_class": "str", "tz_aware": False}), - (None, None, {"document_class": None, "tz_aware": None}), - ], -) -def test_conf_mongo_client_options_with_valid_values( - document_class, tz_aware, expected, monkeypatch -): - """Test the MongoClientOptions pydantic data type with valid values.""" - # Using None here as in "not set by user" - if document_class is not None: - monkeypatch.setenv( - "RALPH_BACKENDS__DATA__MONGO__CLIENT_OPTIONS__document_class", - f"{document_class}", - ) - # Using None here as in "not set by user" - if tz_aware is not None: - monkeypatch.setenv( - "RALPH_BACKENDS__DATA__MONGO__CLIENT_OPTIONS__tz_aware", - f"{tz_aware}", - ) - assert BackendSettings().BACKENDS.DATA.MONGO.CLIENT_OPTIONS.dict() == expected - - -@pytest.mark.parametrize( - "document_class,tz_aware", - [ - ("dict", 3), - ("str", None), - ], -) -def test_conf_mongo_client_options_with_invalid_values( - document_class, tz_aware, monkeypatch -): - """Test the MongoClientOptions pydantic data type with invalid values.""" - monkeypatch.setenv( - "RALPH_BACKENDS__DATA__MONGO__CLIENT_OPTIONS__document_class", - f"{document_class}", - ) - monkeypatch.setenv( - "RALPH_BACKENDS__DATA__MONGO__CLIENT_OPTIONS__tz_aware", - f"{tz_aware}", - ) - with pytest.raises(ValidationError, match="1 validation error for"): - BackendSettings().BACKENDS.DATA.MONGO.CLIENT_OPTIONS.dict() diff --git a/tests/backends/test_loader.py b/tests/backends/test_loader.py new file mode 100644 index 000000000..96eacf58b --- /dev/null +++ b/tests/backends/test_loader.py @@ -0,0 +1,118 @@ +"""Tests for Ralph's backend utilities.""" + +import logging + +from ralph.backends.data.async_es import AsyncESDataBackend +from ralph.backends.data.async_mongo import AsyncMongoDataBackend +from ralph.backends.data.clickhouse import ClickHouseDataBackend +from ralph.backends.data.es import ESDataBackend +from ralph.backends.data.fs import FSDataBackend +from ralph.backends.data.ldp import LDPDataBackend +from ralph.backends.data.mongo import MongoDataBackend +from ralph.backends.data.s3 import S3DataBackend +from ralph.backends.data.swift import SwiftDataBackend +from ralph.backends.http.async_lrs import AsyncLRSHTTPBackend +from ralph.backends.http.lrs import LRSHTTPBackend +from ralph.backends.loader import ( + get_backends, + get_cli_backends, + get_cli_list_backends, + get_cli_write_backends, + get_lrs_backends, +) +from ralph.backends.lrs.async_es import AsyncESLRSBackend +from ralph.backends.lrs.async_mongo import AsyncMongoLRSBackend +from ralph.backends.lrs.clickhouse import ClickHouseLRSBackend +from ralph.backends.lrs.es import ESLRSBackend +from ralph.backends.lrs.fs import FSLRSBackend +from ralph.backends.lrs.mongo import MongoLRSBackend +from ralph.backends.stream.base import BaseStreamBackend +from ralph.backends.stream.ws import WSStreamBackend + + +def test_backends_loader_get_backends(caplog): + """Test the `get_backends` function.""" + + # Given a non existing module name, the `get_backends` function should skip it. + with caplog.at_level(logging.INFO): + assert not get_backends(("non_existent_package.foo",), (BaseStreamBackend,)) + + assert ( + "ralph.backends.loader", + logging.INFO, + "Could not find 'non_existent_package.foo' package; skipping it", + ) in caplog.record_tuples + + # Given a module with a sub-module raising an exception during the import, + # the `get_backends` function should skip it. + paths = ("tests.backends.test_utils_backends",) + with caplog.at_level(logging.INFO): + assert get_backends(paths, (BaseStreamBackend,)) == {"ws": WSStreamBackend} + + assert ( + "ralph.backends.loader", + logging.INFO, + "Failed to import tests.backends.test_utils_backends.invalid_backends module: " + "No module named 'invalid_backends'", + ) in caplog.record_tuples + + +def test_backends_loader_get_cli_backends(): + """Test the `get_cli_backends` function.""" + assert get_cli_backends() == { + "async_es": AsyncESDataBackend, + "async_lrs": AsyncLRSHTTPBackend, + "async_mongo": AsyncMongoDataBackend, + "clickhouse": ClickHouseDataBackend, + "es": ESDataBackend, + "fs": FSDataBackend, + "ldp": LDPDataBackend, + "lrs": LRSHTTPBackend, + "mongo": MongoDataBackend, + "s3": S3DataBackend, + "swift": SwiftDataBackend, + "ws": WSStreamBackend, + } + + +def test_backends_loader_get_cli_write_backends(): + """Test the `get_cli_write_backends` function.""" + assert get_cli_write_backends() == { + "async_es": AsyncESDataBackend, + "async_lrs": AsyncLRSHTTPBackend, + "async_mongo": AsyncMongoDataBackend, + "clickhouse": ClickHouseDataBackend, + "es": ESDataBackend, + "fs": FSDataBackend, + "lrs": LRSHTTPBackend, + "mongo": MongoDataBackend, + "s3": S3DataBackend, + "swift": SwiftDataBackend, + } + + +def test_backends_loader_get_cli_list_backends(): + """Test the `get_cli_list_backends` function.""" + assert get_cli_list_backends() == { + "async_es": AsyncESDataBackend, + "async_mongo": AsyncMongoDataBackend, + "clickhouse": ClickHouseDataBackend, + "es": ESDataBackend, + "fs": FSDataBackend, + "ldp": LDPDataBackend, + "mongo": MongoDataBackend, + "s3": S3DataBackend, + "swift": SwiftDataBackend, + } + + +def test_backends_loader_get_lrs_backends(): + """Test the `get_lrs_backends` function.""" + assert get_lrs_backends() == { + "async_es": AsyncESLRSBackend, + "async_mongo": AsyncMongoLRSBackend, + "clickhouse": ClickHouseLRSBackend, + "es": ESLRSBackend, + "fs": FSLRSBackend, + "mongo": MongoLRSBackend, + } diff --git a/tests/backends/test_utils_backends/__init__.py b/tests/backends/test_utils_backends/__init__.py new file mode 100644 index 000000000..ce857d9a4 --- /dev/null +++ b/tests/backends/test_utils_backends/__init__.py @@ -0,0 +1 @@ +"""Backend module fixture for test_utils.py.""" diff --git a/tests/backends/test_utils_backends/invalid_backends.py b/tests/backends/test_utils_backends/invalid_backends.py new file mode 100644 index 000000000..025a0fb98 --- /dev/null +++ b/tests/backends/test_utils_backends/invalid_backends.py @@ -0,0 +1,3 @@ +"""A module containing invalid Ralph Backends that raises an exception during import.""" + +raise ModuleNotFoundError("No module named 'invalid_backends'") diff --git a/tests/backends/test_utils_backends/valid_backends.py b/tests/backends/test_utils_backends/valid_backends.py new file mode 100644 index 000000000..3fa143d6e --- /dev/null +++ b/tests/backends/test_utils_backends/valid_backends.py @@ -0,0 +1,6 @@ +"""A module containing valid Ralph Backends.""" + +# pylint: disable=unused-import + +from ralph.backends.data.fs import FSDataBackend # noqa: F401 +from ralph.backends.stream.ws import WSStreamBackend # noqa: F401 diff --git a/tests/test_cli.py b/tests/test_cli.py index 827543378..6ecb1c8c9 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -11,7 +11,6 @@ from hypothesis import settings as hypothesis_settings from pydantic import ValidationError -from ralph.backends.conf import backends_settings from ralph.backends.data.fs import FSDataBackend from ralph.backends.data.ldp import LDPDataBackend from ralph.cli import ( @@ -921,50 +920,13 @@ def test_cli_runserver_command_environment_file_generation(monkeypatch): def mock_uvicorn_run(_, env_file=None, **kwargs): """Mock uvicorn.run asserting environment file content.""" + expected_env_lines = [ + f"RALPH_RUNSERVER_BACKEND={settings.RUNSERVER_BACKEND}\n", + "RALPH_BACKENDS__DATA__ES__DEFAULT_INDEX=foo\n", + "RALPH_BACKENDS__DATA__ES__CLIENT_OPTIONS__verify_certs=True\n", + ] with open(env_file, mode="r", encoding=settings.LOCALE_ENCODING) as file: - env_lines = [ - f"RALPH_RUNSERVER_BACKEND={settings.RUNSERVER_BACKEND}\n", - "RALPH_BACKENDS__LRS__ES__DEFAULT_INDEX=foo\n", - "RALPH_BACKENDS__LRS__ES__CLIENT_OPTIONS__verify_certs=True\n", - "RALPH_BACKENDS__LRS__MONGO__DEFAULT_CHUNK_SIZE=" - f"{backends_settings.BACKENDS.LRS.MONGO.DEFAULT_CHUNK_SIZE}\n", - "RALPH_BACKENDS__LRS__MONGO__DEFAULT_COLLECTION=" - f"{backends_settings.BACKENDS.LRS.MONGO.DEFAULT_COLLECTION}\n", - "RALPH_BACKENDS__LRS__MONGO__DEFAULT_DATABASE=" - f"{backends_settings.BACKENDS.LRS.MONGO.DEFAULT_DATABASE}\n", - "RALPH_BACKENDS__LRS__MONGO__CONNECTION_URI=" - f"{backends_settings.BACKENDS.LRS.MONGO.CONNECTION_URI}\n", - "RALPH_BACKENDS__LRS__FS__DEFAULT_LRS_FILE=" - f"{backends_settings.BACKENDS.LRS.FS.DEFAULT_LRS_FILE}\n", - "RALPH_BACKENDS__LRS__FS__DEFAULT_QUERY_STRING=" - f"{backends_settings.BACKENDS.LRS.FS.DEFAULT_QUERY_STRING}\n", - "RALPH_BACKENDS__LRS__FS__DEFAULT_DIRECTORY_PATH=" - f"{backends_settings.BACKENDS.LRS.FS.DEFAULT_DIRECTORY_PATH}\n", - "RALPH_BACKENDS__LRS__FS__DEFAULT_CHUNK_SIZE=" - f"{backends_settings.BACKENDS.LRS.FS.DEFAULT_CHUNK_SIZE}\n", - "RALPH_BACKENDS__LRS__ES__POINT_IN_TIME_KEEP_ALIVE=" - f"{backends_settings.BACKENDS.LRS.ES.POINT_IN_TIME_KEEP_ALIVE}\n", - "RALPH_BACKENDS__LRS__ES__HOSTS=" - f"{','.join(backends_settings.BACKENDS.LRS.ES.HOSTS)}\n", - "RALPH_BACKENDS__LRS__ES__DEFAULT_CHUNK_SIZE=" - f"{backends_settings.BACKENDS.LRS.ES.DEFAULT_CHUNK_SIZE}\n", - "RALPH_BACKENDS__LRS__ES__ALLOW_YELLOW_STATUS=" - f"{backends_settings.BACKENDS.LRS.ES.ALLOW_YELLOW_STATUS}\n", - "RALPH_BACKENDS__LRS__CLICKHOUSE__IDS_CHUNK_SIZE=" - f"{backends_settings.BACKENDS.LRS.CLICKHOUSE.IDS_CHUNK_SIZE}\n", - "RALPH_BACKENDS__LRS__CLICKHOUSE__DEFAULT_CHUNK_SIZE=" - f"{backends_settings.BACKENDS.LRS.CLICKHOUSE.DEFAULT_CHUNK_SIZE}\n", - "RALPH_BACKENDS__LRS__CLICKHOUSE__EVENT_TABLE_NAME=" - f"{backends_settings.BACKENDS.LRS.CLICKHOUSE.EVENT_TABLE_NAME}\n", - "RALPH_BACKENDS__LRS__CLICKHOUSE__DATABASE=" - f"{backends_settings.BACKENDS.LRS.CLICKHOUSE.DATABASE}\n", - "RALPH_BACKENDS__LRS__CLICKHOUSE__PORT=" - f"{backends_settings.BACKENDS.LRS.CLICKHOUSE.PORT}\n", - "RALPH_BACKENDS__LRS__CLICKHOUSE__HOST=" - f"{backends_settings.BACKENDS.LRS.CLICKHOUSE.HOST}\n", - ] - env_lines_created = file.readlines() - assert all(line in env_lines_created for line in env_lines) + assert file.readlines() == expected_env_lines monkeypatch.setattr("ralph.cli.uvicorn.run", mock_uvicorn_run) runner = CliRunner() diff --git a/tests/test_cli_usage.py b/tests/test_cli_usage.py index 859eb0141..1261173b4 100644 --- a/tests/test_cli_usage.py +++ b/tests/test_cli_usage.py @@ -111,8 +111,8 @@ def test_cli_read_command_usage(): "Usage: ralph read [OPTIONS] [ARCHIVE]\n\n" " Read an archive or records from a configured backend.\n\n" "Options:\n" - " -b, --backend [async_es|async_mongo|clickhouse|es|fs|ldp|lrs|mongo|s3|swift|" - "ws]\n" + " -b, --backend " + "[async_es|async_lrs|async_mongo|clickhouse|es|fs|ldp|lrs|mongo|s3|swift|ws]\n" " Backend [required]\n" " async_es backend: \n" " --async-es-allow-yellow-status / --no-async-es-allow-yellow-status\n" @@ -123,9 +123,16 @@ def test_cli_read_command_usage(): " --async-es-locale-encoding TEXT\n" " --async-es-point-in-time-keep-alive TEXT\n" " --async-es-refresh-after-write TEXT\n" + " async_lrs backend: \n" + " --async-lrs-base-url TEXT\n" + " --async-lrs-headers KEY=VALUE,KEY=VALUE\n" + " --async-lrs-password TEXT\n" + " --async-lrs-statements-endpoint TEXT\n" + " --async-lrs-status-endpoint TEXT\n" + " --async-lrs-username TEXT\n" " async_mongo backend: \n" " --async-mongo-client-options KEY=VALUE,KEY=VALUE\n" - " --async-mongo-connection-uri TEXT\n" + " --async-mongo-connection-uri MONGODSN\n" " --async-mongo-default-chunk-size INTEGER\n" " --async-mongo-default-collection TEXT\n" " --async-mongo-default-database TEXT\n" @@ -171,7 +178,7 @@ def test_cli_read_command_usage(): " --lrs-username TEXT\n" " mongo backend: \n" " --mongo-client-options KEY=VALUE,KEY=VALUE\n" - " --mongo-connection-uri TEXT\n" + " --mongo-connection-uri MONGODSN\n" " --mongo-default-chunk-size INTEGER\n" " --mongo-default-collection TEXT\n" " --mongo-default-database TEXT\n" @@ -217,8 +224,8 @@ def test_cli_read_command_usage(): assert result.exit_code > 0 assert ( "Error: Missing option '-b' / '--backend'. " - "Choose from:\n\tasync_es,\n\tasync_mongo,\n\tclickhouse,\n\tes,\n\tfs,\n\tldp," - "\n\tlrs,\n\tmongo,\n\ts3,\n\tswift,\n\tws\n" + "Choose from:\n\tasync_es,\n\tasync_lrs,\n\tasync_mongo,\n\tclickhouse,\n\tes," + "\n\tfs,\n\tldp,\n\tlrs,\n\tmongo,\n\ts3,\n\tswift,\n\tws\n" ) in result.output @@ -245,7 +252,7 @@ def test_cli_list_command_usage(): " --async-es-refresh-after-write TEXT\n" " async_mongo backend: \n" " --async-mongo-client-options KEY=VALUE,KEY=VALUE\n" - " --async-mongo-connection-uri TEXT\n" + " --async-mongo-connection-uri MONGODSN\n" " --async-mongo-default-chunk-size INTEGER\n" " --async-mongo-default-collection TEXT\n" " --async-mongo-default-database TEXT\n" @@ -284,7 +291,7 @@ def test_cli_list_command_usage(): " --ldp-service-name TEXT\n" " mongo backend: \n" " --mongo-client-options KEY=VALUE,KEY=VALUE\n" - " --mongo-connection-uri TEXT\n" + " --mongo-connection-uri MONGODSN\n" " --mongo-default-chunk-size INTEGER\n" " --mongo-default-collection TEXT\n" " --mongo-default-database TEXT\n" @@ -337,7 +344,8 @@ def test_cli_write_command_usage(): "Usage: ralph write [OPTIONS]\n\n" " Write an archive to a configured backend.\n\n" "Options:\n" - " -b, --backend [async_es|async_mongo|clickhouse|es|fs|ldp|lrs|mongo|s3|swift]" + " -b, --backend " + "[async_es|async_lrs|async_mongo|clickhouse|es|fs|lrs|mongo|s3|swift]" "\n" " Backend [required]\n" " async_es backend: \n" @@ -349,9 +357,16 @@ def test_cli_write_command_usage(): " --async-es-locale-encoding TEXT\n" " --async-es-point-in-time-keep-alive TEXT\n" " --async-es-refresh-after-write TEXT\n" + " async_lrs backend: \n" + " --async-lrs-base-url TEXT\n" + " --async-lrs-headers KEY=VALUE,KEY=VALUE\n" + " --async-lrs-password TEXT\n" + " --async-lrs-statements-endpoint TEXT\n" + " --async-lrs-status-endpoint TEXT\n" + " --async-lrs-username TEXT\n" " async_mongo backend: \n" " --async-mongo-client-options KEY=VALUE,KEY=VALUE\n" - " --async-mongo-connection-uri TEXT\n" + " --async-mongo-connection-uri MONGODSN\n" " --async-mongo-default-chunk-size INTEGER\n" " --async-mongo-default-collection TEXT\n" " --async-mongo-default-database TEXT\n" @@ -380,14 +395,6 @@ def test_cli_write_command_usage(): " --fs-default-directory-path PATH\n" " --fs-default-query-string TEXT\n" " --fs-locale-encoding TEXT\n" - " ldp backend: \n" - " --ldp-application-key TEXT\n" - " --ldp-application-secret TEXT\n" - " --ldp-consumer-key TEXT\n" - " --ldp-default-stream-id TEXT\n" - " --ldp-endpoint TEXT\n" - " --ldp-request-timeout TEXT\n" - " --ldp-service-name TEXT\n" " lrs backend: \n" " --lrs-base-url TEXT\n" " --lrs-headers KEY=VALUE,KEY=VALUE\n" @@ -397,7 +404,7 @@ def test_cli_write_command_usage(): " --lrs-username TEXT\n" " mongo backend: \n" " --mongo-client-options KEY=VALUE,KEY=VALUE\n" - " --mongo-connection-uri TEXT\n" + " --mongo-connection-uri MONGODSN\n" " --mongo-default-chunk-size INTEGER\n" " --mongo-default-collection TEXT\n" " --mongo-default-database TEXT\n" @@ -444,8 +451,8 @@ def test_cli_write_command_usage(): result = runner.invoke(cli, ["write"]) assert result.exit_code > 0 assert ( - "Missing option '-b' / '--backend'. Choose from:\n\tasync_es,\n\tasync_mongo,\n" - "\tclickhouse,\n\tes,\n\tfs,\n\tldp,\n\tlrs,\n\tmongo,\n\ts3,\n\tswift\n" + "Missing option '-b' / '--backend'. Choose from:\n\tasync_es,\n\tasync_lrs,\n\t" + "async_mongo,\n\tclickhouse,\n\tes,\n\tfs,\n\tlrs,\n\tmongo,\n\ts3,\n\tswift\n" ) in result.output @@ -472,7 +479,7 @@ def test_cli_runserver_command_usage(): " --async-es-refresh-after-write TEXT\n" " async_mongo backend: \n" " --async-mongo-client-options KEY=VALUE,KEY=VALUE\n" - " --async-mongo-connection-uri TEXT\n" + " --async-mongo-connection-uri MONGODSN\n" " --async-mongo-default-chunk-size INTEGER\n" " --async-mongo-default-collection TEXT\n" " --async-mongo-default-database TEXT\n" @@ -505,7 +512,7 @@ def test_cli_runserver_command_usage(): " --fs-locale-encoding TEXT\n" " mongo backend: \n" " --mongo-client-options KEY=VALUE,KEY=VALUE\n" - " --mongo-connection-uri TEXT\n" + " --mongo-connection-uri MONGODSN\n" " --mongo-default-chunk-size INTEGER\n" " --mongo-default-collection TEXT\n" " --mongo-default-database TEXT\n" diff --git a/tests/test_conf.py b/tests/test_conf.py index e9c681d79..ed0d184c0 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -5,7 +5,7 @@ import pytest from ralph import conf -from ralph.backends.conf import BackendSettings +from ralph.backends.data.es import ESDataBackend from ralph.conf import CommaSeparatedTuple, Settings, settings from ralph.exceptions import ConfigurationException @@ -49,7 +49,7 @@ def test_conf_comma_separated_list_with_valid_values(value, expected, monkeypatc """Test the CommaSeparatedTuple pydantic data type with valid values.""" assert next(CommaSeparatedTuple.__get_validators__())(value) == expected monkeypatch.setenv("RALPH_BACKENDS__DATA__ES__HOSTS", "".join(value)) - assert BackendSettings().BACKENDS.DATA.ES.HOSTS == expected + assert ESDataBackend().settings.HOSTS == expected @pytest.mark.parametrize("value", [{}, None]) diff --git a/tests/test_utils.py b/tests/test_utils.py index 39e9b1ba6..45e38414c 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,14 +1,12 @@ """Tests for Ralph utils.""" -from abc import ABC -from types import ModuleType +import logging import pytest from pydantic import BaseModel from ralph import utils as ralph_utils -from ralph.backends.conf import backends_settings -from ralph.conf import InstantiableSettingsItem +from ralph.exceptions import UnsupportedBackendException def test_utils_import_string(): @@ -25,79 +23,50 @@ def test_utils_import_string(): assert http_status.OK == 200 -def test_utils_get_backend_type(): - """Test get_backend_type utility.""" - backend_types = [backend_type[1] for backend_type in backends_settings.BACKENDS] - assert ( - ralph_utils.get_backend_type(backend_types, "es") - == backends_settings.BACKENDS.DATA - ) - assert ( - ralph_utils.get_backend_type(backend_types, "lrs") - == backends_settings.BACKENDS.HTTP - ) - assert ( - ralph_utils.get_backend_type(backend_types, "ws") - == backends_settings.BACKENDS.STREAM - ) - assert ralph_utils.get_backend_type(backend_types, "foo") is None +def test_utils_get_backend_class(caplog): + """Test the `get_backend_class` utility should return the expected result.""" + backends = {"es": type("es_backend", (), {}), "fs": type("fs_backend", (), {})} + assert ralph_utils.get_backend_class(backends, "es") == backends["es"] + assert ralph_utils.get_backend_class(backends, "fs") == backends["fs"] + msg = "'foo' backend not found in available backends: es, fs" + with caplog.at_level(logging.ERROR): + with pytest.raises(UnsupportedBackendException, match=msg): + ralph_utils.get_backend_class(backends, "foo") + + assert ("ralph.utils", logging.ERROR, msg) in caplog.record_tuples @pytest.mark.parametrize( "options,expected", [ # Empty options should produce default result. - ({}, {}), + ({}, {"FOO": "FOO"}), # Options not matching the backend name are ignored. - ({"foo": "bar", "not_dummy_foo": "baz"}, {}), + ({"foo": "bar", "not_dummy_foo": "baz"}, {"FOO": "FOO"}), + # Options matching the backend name update the defaults. + ({"dummy_foo": "bar"}, {"FOO": "bar"}), ], ) -def test_utils_get_backend_instance(monkeypatch, options, expected): - """Test get_backend_instance utility should return the expected result.""" +def test_utils_get_backend_instance(options, expected): + """Test the `get_backend_instance` utility should return the expected result.""" - class DummyTestBackendSettings(InstantiableSettingsItem): + class DummyTestBackendSettings(BaseModel): """Represent a dummy backend setting.""" - FOO: str = "FOO" # pylint: disable=disallowed-name - - def get_instance(self, **init_parameters): # pylint: disable=no-self-use - """Return the init_parameters.""" - return init_parameters + FOO: str = "FOO" - class DummyTestBackend(ABC): + class DummyTestBackend: """Represent a dummy backend instance.""" - type = "test" name = "dummy" + settings_class = DummyTestBackendSettings - def __init__(self, *args, **kargs): # pylint: disable=unused-argument - return - - def __call__(self, *args, **kwargs): # pylint: disable=unused-argument - return {} - - def mock_import_module(*args, **kwargs): # pylint: disable=unused-argument - """Mock import_module.""" - test_module = ModuleType(name="ralph.backends.test.dummy") - - test_module.DummyTestBackendSettings = DummyTestBackendSettings - test_module.DummyTestBackend = DummyTestBackend - - return test_module - - class TestBackendSettings(BaseModel): # DATA-backend-type - """A backend type including the DummyTestBackendSettings.""" - - DUMMY: DummyTestBackendSettings = ( - DummyTestBackendSettings() - ) # Es-Backend-settings + def __init__(self, settings): + self.settings = settings - monkeypatch.setattr(ralph_utils, "import_module", mock_import_module) - backend_instance = ralph_utils.get_backend_instance( - TestBackendSettings(), "dummy", options - ) - assert isinstance(backend_instance, DummyTestBackend) - assert backend_instance() == expected + backend = ralph_utils.get_backend_instance(DummyTestBackend, options) + assert isinstance(backend, DummyTestBackend) + assert backend.settings.dict() == expected @pytest.mark.parametrize("path,value", [(["foo", "bar"], "bar_value")])