From 5b0cfaea77706e37fbeea7416ba57c2bce7a0875 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Tue, 23 Jan 2024 11:42:47 +0100 Subject: [PATCH 1/6] Separate out entity models in SOFT and DLite Create a "minimum set" of fields for the versioned SOFT models and then create the DLite implementations of these separately. --- dlite_entities_service/main.py | 4 +- dlite_entities_service/models/__init__.py | 18 ++- dlite_entities_service/models/dlite.py | 111 ++++++++++++++ dlite_entities_service/models/soft.py | 148 ++++++++++++++++++ dlite_entities_service/models/soft5.py | 173 ++------------------- dlite_entities_service/models/soft7.py | 177 +--------------------- tests/cli/test_upload.py | 8 +- tests/models/test_init.py | 24 ++- tests/test_route.py | 13 +- 9 files changed, 324 insertions(+), 352 deletions(-) create mode 100644 dlite_entities_service/models/dlite.py create mode 100644 dlite_entities_service/models/soft.py diff --git a/dlite_entities_service/main.py b/dlite_entities_service/main.py index dfcea251..a0107ade 100644 --- a/dlite_entities_service/main.py +++ b/dlite_entities_service/main.py @@ -9,7 +9,7 @@ from fastapi import FastAPI, HTTPException, Path, status from dlite_entities_service import __version__ -from dlite_entities_service.models import VersionedSOFTEntity +from dlite_entities_service.models import Entity from dlite_entities_service.service.backend import ENTITIES_COLLECTION from dlite_entities_service.service.config import CONFIG from dlite_entities_service.service.logger import setup_logger @@ -61,7 +61,7 @@ async def lifespan(_: FastAPI): @APP.get( "/{version}/{name}", - response_model=VersionedSOFTEntity, + response_model=Entity, response_model_by_alias=True, response_model_exclude_unset=True, ) diff --git a/dlite_entities_service/models/__init__.py b/dlite_entities_service/models/__init__.py index 269fb96c..c44431f3 100644 --- a/dlite_entities_service/models/__init__.py +++ b/dlite_entities_service/models/__init__.py @@ -5,18 +5,20 @@ from pydantic import ValidationError -from .soft5 import URI_REGEX, SOFT5Entity +from .dlite import DLiteEntity +from .soft import URI_REGEX +from .soft5 import SOFT5Entity from .soft7 import SOFT7Entity -VersionedSOFTEntity = SOFT7Entity | SOFT5Entity +Entity = SOFT7Entity | SOFT5Entity | DLiteEntity def soft_entity( *, return_errors: bool = False, **fields -) -> VersionedSOFTEntity | list[ValidationError]: - """Return the correct version of the SOFT Entity.""" +) -> Entity | list[ValidationError]: + """Return the correct version of the Entity.""" errors = [] - for versioned_entity_cls in get_args(VersionedSOFTEntity): + for versioned_entity_cls in get_args(Entity): try: new_object = versioned_entity_cls(**fields) break @@ -34,7 +36,7 @@ def soft_entity( return new_object # type: ignore[return-value] -def get_uri(entity: VersionedSOFTEntity) -> str: +def get_uri(entity: Entity) -> str: """Return the URI of the entity.""" if entity.uri is not None: return str(entity.uri) @@ -42,7 +44,7 @@ def get_uri(entity: VersionedSOFTEntity) -> str: return f"{entity.namespace}/{entity.version}/{entity.name}" -def get_version(entity: VersionedSOFTEntity) -> str: +def get_version(entity: Entity) -> str: """Return the version of the entity.""" if entity.version is not None: return entity.version @@ -53,7 +55,7 @@ def get_version(entity: VersionedSOFTEntity) -> str: raise ValueError("Cannot parse URI to get version.") -def get_updated_version(entity: VersionedSOFTEntity) -> str: +def get_updated_version(entity: Entity) -> str: """Return the updated version of the entity.""" current_version = get_version(entity) diff --git a/dlite_entities_service/models/dlite.py b/dlite_entities_service/models/dlite.py new file mode 100644 index 00000000..f8e11065 --- /dev/null +++ b/dlite_entities_service/models/dlite.py @@ -0,0 +1,111 @@ +"""DLite model.""" +from __future__ import annotations + +from typing import Annotated + +from pydantic import AliasChoices, Field, field_validator +from pydantic.networks import AnyHttpUrl + +from dlite_entities_service.models.soft5 import SOFT5Entity, SOFT5Property +from dlite_entities_service.models.soft7 import SOFT7Entity, SOFT7Property + + +class DLiteSOFT5Property(SOFT5Property): + """The defining metadata for a (SOFT5-based) DLite Entity's property.""" + + ref: Annotated[ + AnyHttpUrl | None, + Field( + validation_alias=AliasChoices("$ref", "ref"), + serialization_alias="$ref", + description=( + "Formally a part of type. `$ref` is used together with the `ref` type, " + "which is a special datatype for referring to other instances." + ), + ), + ] = None + + +class DLiteSOFT5Entity(SOFT5Entity): + """A (SOFT5-based) DLite Entity.""" + + meta: Annotated[ + AnyHttpUrl, + Field( + description=( + "URI for the metadata entity. For all entities at onto-ns.com, the " + "EntitySchema v0.3 is used." + ), + ), + ] = AnyHttpUrl("http://onto-ns.com/meta/0.3/EntitySchema") + + properties: Annotated[ # type: ignore[assignment] + list[DLiteSOFT5Property], Field(description="A list of properties.") + ] + + @field_validator("meta", mode="after") + @classmethod + def _only_support_onto_ns(cls, value: AnyHttpUrl) -> AnyHttpUrl: + """Validate `meta` only refers to onto-ns.com EntitySchema v0.3.""" + if str(value).rstrip("/") != "http://onto-ns.com/meta/0.3/EntitySchema": + error_message = ( + "This service only works with DLite entities using EntitySchema " + "v0.3 at onto-ns.com as the metadata entity.\n" + ) + raise ValueError(error_message) + return value + + +class DLiteSOFT7Property(SOFT7Property): + """The defining metadata for a (SOFT7-based) DLite Entity's property.""" + + ref: Annotated[ + AnyHttpUrl | None, + Field( + validation_alias=AliasChoices("$ref", "ref"), + serialization_alias="$ref", + description=( + "Formally a part of type. `$ref` is used together with the `ref` type, " + "which is a special datatype for referring to other instances." + ), + ), + ] = None + + +class DLiteSOFT7Entity(SOFT7Entity): + """A (SOFT7-based) DLite Entity.""" + + meta: Annotated[ + AnyHttpUrl, + Field( + description=( + "URI for the metadata entity. For all entities at onto-ns.com, the " + "EntitySchema v0.3 is used." + ), + ), + ] = AnyHttpUrl("http://onto-ns.com/meta/0.3/EntitySchema") + + properties: Annotated[ # type: ignore[assignment] + dict[str, DLiteSOFT7Property], + Field( + description=( + "A dictionary of properties, mapping the property name to a dictionary " + "of metadata defining the property." + ), + ), + ] + + @field_validator("meta", mode="after") + @classmethod + def _only_support_onto_ns(cls, value: AnyHttpUrl) -> AnyHttpUrl: + """Validate `meta` only refers to onto-ns.com EntitySchema v0.3.""" + if str(value).rstrip("/") != "http://onto-ns.com/meta/0.3/EntitySchema": + error_message = ( + "This service only works with DLite entities using EntitySchema " + "v0.3 at onto-ns.com as the metadata entity.\n" + ) + raise ValueError(error_message) + return value + + +DLiteEntity = DLiteSOFT7Entity | DLiteSOFT5Entity diff --git a/dlite_entities_service/models/soft.py b/dlite_entities_service/models/soft.py new file mode 100644 index 00000000..33041dd5 --- /dev/null +++ b/dlite_entities_service/models/soft.py @@ -0,0 +1,148 @@ +"""Base (minimum set) models for SOFT entities.""" +from __future__ import annotations + +import difflib +import re +from typing import Annotated, Any + +from pydantic import ( + AliasChoices, + BaseModel, + ConfigDict, + Field, + field_validator, + model_validator, +) +from pydantic.networks import AnyHttpUrl + +from dlite_entities_service.service.config import CONFIG + +URI_REGEX = re.compile( + r"^(?Phttps?://.+)/(?P\d(?:\.\d+){0,2})/(?P[^/#?]+)$" +) +"""Regular expression to parse a SOFT entity URI.""" + + +class SOFTProperty(BaseModel): + """The minimum set of defining metadata for a SOFT Entity's property.""" + + model_config = ConfigDict(extra="forbid") + + type_: Annotated[ + str, + Field( + alias="type", + description="The type of the described property, e.g., an integer.", + ), + ] + shape: Annotated[ + list[str] | None, + Field( + description=( + "The dimension of multi-dimensional properties. This is a list of " + "dimension expressions referring to the dimensions defined above. For " + "instance, if an entity have dimensions with names `H`, `K`, and `L` " + "and a property with shape `['K', 'H+1']`, the property of an instance " + "of this entity with dimension values `H=2`, `K=2`, `L=6` will have " + "shape `[2, 3]`." + ), + validation_alias=AliasChoices("dims", "shape"), + ), + ] = None + unit: Annotated[str | None, Field(description="The unit of the property.")] = None + description: Annotated[ + str, Field(description="A human-readable description of the property.") + ] + + +class SOFTEntity(BaseModel): + """A minimum Field set SOFT Entity to be used in a versioned SOFT Entity model.""" + + model_config = ConfigDict(extra="forbid") + + name: Annotated[str | None, Field(description="The name of the entity.")] = None + version: Annotated[ + str | None, Field(description="The version of the entity.") + ] = None + namespace: Annotated[ + AnyHttpUrl | None, Field(description="The namespace of the entity.") + ] = None + uri: Annotated[ + AnyHttpUrl | None, + Field( + description=( + "The universal identifier for the entity. This MUST start with the base" + " URL." + ), + ), + ] = None + description: Annotated[str, Field(description="Description of the entity.")] = "" + + @field_validator("uri", "namespace", mode="after") + @classmethod + def _validate_base_url(cls, value: AnyHttpUrl) -> AnyHttpUrl: + """Validate `uri` and `namespace` starts with the current base URL for the + service.""" + if not str(value).startswith(str(CONFIG.base_url)): + error_message = ( + "This service only works with SOFT entities at " f"{CONFIG.base_url}.\n" + ) + raise ValueError(error_message) + return value + + @field_validator("uri", mode="after") + @classmethod + def _validate_uri(cls, value: AnyHttpUrl) -> AnyHttpUrl: + """Validate `uri` is consistent with `name`, `version`, and `namespace`.""" + if URI_REGEX.match(str(value)) is None: + error_message = ( + "The 'uri' is not a valid SOFT7 entity URI. It must be of the form " + f"{str(CONFIG.base_url).rstrip('/')}/{{version}}/{{name}}.\n" + ) + raise ValueError(error_message) + return value + + @model_validator(mode="before") + @classmethod + def _check_cross_dependent_fields(cls, data: Any) -> Any: + """Check that `name`, `version`, and `namespace` are all set or all unset.""" + if ( + isinstance(data, dict) + and any(data.get(_) is None for _ in ("name", "version", "namespace")) + and not all(data.get(_) is None for _ in ("name", "version", "namespace")) + ): + error_message = ( + "Either all of `name`, `version`, and `namespace` must be set " + "or all must be unset.\n" + ) + raise ValueError(error_message) + + if ( + isinstance(data, dict) + and any(data.get(_) is None for _ in ("name", "version", "namespace")) + and data.get("uri") is None + ): + error_message = ( + "Either `name`, `version`, and `namespace` or `uri` must be set.\n" + ) + raise ValueError(error_message) + + if ( + isinstance(data, dict) + and all(data.get(_) is not None for _ in ("name", "version", "namespace")) + and data.get("uri") is not None + and data["uri"] != f"{data['namespace']}/{data['version']}/{data['name']}" + ): + # Ensure that `uri` is consistent with `name`, `version`, and `namespace`. + diff = "\n ".join( + difflib.ndiff( + [data["uri"]], + [f"{data['namespace']}/{data['version']}/{data['name']}"], + ) + ) + error_message = ( + "The `uri` is not consistent with `name`, `version`, and " + f"`namespace`:\n\n {diff}\n\n" + ) + raise ValueError(error_message) + return data diff --git a/dlite_entities_service/models/soft5.py b/dlite_entities_service/models/soft5.py index e16407c7..0d3e5865 100644 --- a/dlite_entities_service/models/soft5.py +++ b/dlite_entities_service/models/soft5.py @@ -1,19 +1,11 @@ -"""SOFT5 models.""" +"""SOFT5 model.""" from __future__ import annotations -import difflib -import re -from typing import Annotated, Any +from typing import Annotated -from pydantic import AliasChoices, BaseModel, Field, field_validator, model_validator -from pydantic.networks import AnyHttpUrl +from pydantic import BaseModel, Field -from dlite_entities_service.service.config import CONFIG - -URI_REGEX = re.compile( - r"^(?Phttps?://.+)/(?P\d(?:\.\d+){0,2})/(?P[^/#?]+)$" -) -"""Regular expression to parse a SOFT entity URI.""" +from dlite_entities_service.models.soft import SOFTEntity, SOFTProperty class SOFT5Dimension(BaseModel): @@ -25,81 +17,15 @@ class SOFT5Dimension(BaseModel): ] -class SOFT5Property(BaseModel): +class SOFT5Property(SOFTProperty): """The defining metadata for a SOFT5 Entity's property.""" - name: Annotated[ - str | None, - Field( - description=("The name of the property."), - ), - ] = None - type_: Annotated[ - str, - Field( - alias="type", - description="The type of the described property, e.g., an integer.", - ), - ] - ref: Annotated[ - AnyHttpUrl | None, - Field( - validation_alias=AliasChoices("$ref", "ref"), - serialization_alias="$ref", - description=( - "Formally a part of type. `$ref` is used together with the `ref` type, " - "which is a special datatype for referring to other instances." - ), - ), - ] = None - dims: Annotated[ - list[str] | None, - Field( - description=( - "The dimension of multi-dimensional properties. This is a list of " - "dimension expressions referring to the dimensions defined above. For " - "instance, if an entity have dimensions with names `H`, `K`, and `L` " - "and a property with shape `['K', 'H+1']`, the property of an instance " - "of this entity with dimension values `H=2`, `K=2`, `L=6` will have " - "shape `[2, 3]`." - ), - ), - ] = None - unit: Annotated[str | None, Field(description="The unit of the property.")] = None - description: Annotated[ - str, Field(description="A human-readable description of the property.") - ] + name: Annotated[str, Field(description=("The name of the property."))] -class SOFT5Entity(BaseModel): - """A SOFT5 Entity returned from this service.""" +class SOFT5Entity(SOFTEntity): + """A SOFT5 Entity.""" - name: Annotated[str | None, Field(description="The name of the entity.")] = None - version: Annotated[ - str | None, Field(description="The version of the entity.") - ] = None - namespace: Annotated[ - AnyHttpUrl | None, Field(description="The namespace of the entity.") - ] = None - uri: Annotated[ - AnyHttpUrl | None, - Field( - description=( - "The universal identifier for the entity. This MUST start with the base" - " URL." - ), - ), - ] = None - meta: Annotated[ - AnyHttpUrl, - Field( - description=( - "URI for the metadata entity. For all entities at onto-ns.com, the " - "EntitySchema v0.3 is used." - ), - ), - ] = AnyHttpUrl("http://onto-ns.com/meta/0.3/EntitySchema") - description: Annotated[str, Field(description="Description of the entity.")] = "" dimensions: Annotated[ list[SOFT5Dimension], Field( @@ -107,88 +33,7 @@ class SOFT5Entity(BaseModel): "A list of dimensions with name and an accompanying description." ), ), - ] = [] + ] = [] # noqa: RUF012 properties: Annotated[ list[SOFT5Property], Field(description="A list of properties.") ] - - @field_validator("uri", "namespace") - @classmethod - def _validate_base_url(cls, value: AnyHttpUrl) -> AnyHttpUrl: - """Validate `uri` starts with the current base URL for the service.""" - if not str(value).startswith(str(CONFIG.base_url)): - error_message = ( - "This service only works with DLite/SOFT entities at " - f"{CONFIG.base_url}.\n" - ) - raise ValueError(error_message) - return value - - @field_validator("uri", mode="after") - @classmethod - def _validate_uri(cls, value: AnyHttpUrl) -> AnyHttpUrl: - """Validate `uri` is consistent with `name`, `version`, and `namespace`.""" - if URI_REGEX.match(str(value)) is None: - error_message = ( - "The 'uri' is not a valid SOFT7 entity URI. It must be of the form " - f"{str(CONFIG.base_url).rstrip('/')}/{{version}}/{{name}}.\n" - ) - raise ValueError(error_message) - return value - - @field_validator("meta") - @classmethod - def _only_support_onto_ns(cls, value: AnyHttpUrl) -> AnyHttpUrl: - """Validate `meta` only refers to onto-ns.com EntitySchema v0.3.""" - if str(value) != "http://onto-ns.com/meta/0.3/EntitySchema": - error_message = ( - "This service only works with DLite/SOFT entities using EntitySchema " - "v0.3 at onto-ns.com as the metadata entity.\n" - ) - raise ValueError(error_message) - return value - - @model_validator(mode="before") - @classmethod - def _check_cross_dependent_fields(cls, data: Any) -> Any: - """Check that `name`, `version`, and `namespace` are all set or all unset.""" - if ( - isinstance(data, dict) - and any(data.get(_) is None for _ in ("name", "version", "namespace")) - and not all(data.get(_) is None for _ in ("name", "version", "namespace")) - ): - error_message = ( - "Either all of `name`, `version`, and `namespace` must be set " - "or all must be unset.\n" - ) - raise ValueError(error_message) - - if ( - isinstance(data, dict) - and any(data.get(_) is None for _ in ("name", "version", "namespace")) - and data.get("uri") is None - ): - error_message = ( - "Either `name`, `version`, and `namespace` or `uri` must be set.\n" - ) - raise ValueError(error_message) - - if ( - isinstance(data, dict) - and all(data.get(_) is not None for _ in ("name", "version", "namespace")) - and data.get("uri") is not None - and data["uri"] != f"{data['namespace']}/{data['version']}/{data['name']}" - ): - # Ensure that `uri` is consistent with `name`, `version`, and `namespace`. - diff = "\n ".join( - difflib.ndiff( - [data["uri"]], - [f"{data['namespace']}/{data['version']}/{data['name']}"], - ) - ) - error_message = ( - "The `uri` is not consistent with `name`, `version`, and " - f"`namespace`:\n\n {diff}\n\n" - ) - raise ValueError(error_message) - return data diff --git a/dlite_entities_service/models/soft7.py b/dlite_entities_service/models/soft7.py index 9be221b8..73c39810 100644 --- a/dlite_entities_service/models/soft7.py +++ b/dlite_entities_service/models/soft7.py @@ -1,103 +1,24 @@ -"""SOFT7 models.""" +"""SOFT7 model.""" from __future__ import annotations -import difflib -import re -from typing import Annotated, Any +from typing import Annotated -from pydantic import AliasChoices, BaseModel, Field, field_validator, model_validator -from pydantic.networks import AnyHttpUrl +from pydantic import Field -from dlite_entities_service.service.config import CONFIG +from dlite_entities_service.models.soft import SOFTEntity, SOFTProperty -URI_REGEX = re.compile( - r"^(?Phttps?://.+)/(?P\d(?:\.\d+){0,2})/(?P[^/#?]+)$" -) -"""Regular expression to parse a SOFT entity URI.""" - -class SOFT7Property(BaseModel): +class SOFT7Property(SOFTProperty): """The defining metadata for a SOFT7 Entity's property.""" - name: Annotated[ - str | None, - Field( - description=( - "The name of the property. This is not necessary if the SOFT7 approach " - "to entities are taken." - ), - ), - ] = None - type_: Annotated[ - str, - Field( - alias="type", - description="The type of the described property, e.g., an integer.", - ), - ] - ref: Annotated[ - AnyHttpUrl | None, - Field( - validation_alias=AliasChoices("$ref", "ref"), - serialization_alias="$ref", - description=( - "Formally a part of type. `$ref` is used together with the `ref` type, " - "which is a special datatype for referring to other instances." - ), - ), - ] = None - shape: Annotated[ - list[str] | None, - Field( - description=( - "The dimension of multi-dimensional properties. This is a list of " - "dimension expressions referring to the dimensions defined above. For " - "instance, if an entity have dimensions with names `H`, `K`, and `L` " - "and a property with shape `['K', 'H+1']`, the property of an instance " - "of this entity with dimension values `H=2`, `K=2`, `L=6` will have " - "shape `[2, 3]`. Note, this was called `dims` in SOFT5." - ), - ), - ] = None - unit: Annotated[str | None, Field(description="The unit of the property.")] = None - description: Annotated[ - str, Field(description="A human-readable description of the property.") - ] - -class SOFT7Entity(BaseModel): - """A SOFT7 Entity returned from this service.""" +class SOFT7Entity(SOFTEntity): + """A SOFT7 Entity.""" - name: Annotated[str | None, Field(description="The name of the entity.")] = None - version: Annotated[ - str | None, Field(description="The version of the entity.") - ] = None - namespace: Annotated[ - AnyHttpUrl | None, Field(description="The namespace of the entity.") - ] = None - uri: Annotated[ - AnyHttpUrl | None, - Field( - description=( - "The universal identifier for the entity. This MUST start with the " - "base URL." - ), - ), - ] = None - meta: Annotated[ - AnyHttpUrl, - Field( - description=( - "URI for the metadata entity. For all entities at onto-ns.com, the " - "EntitySchema v0.3 is used." - ), - ), - ] = AnyHttpUrl("http://onto-ns.com/meta/0.3/EntitySchema") - description: Annotated[str, Field(description="Description of the entity.")] = "" dimensions: Annotated[ dict[str, str], Field(description="A dict of dimensions with an accompanying description."), - ] = {} + ] = {} # noqa: RUF012 properties: Annotated[ dict[str, SOFT7Property], Field( @@ -107,85 +28,3 @@ class SOFT7Entity(BaseModel): ), ), ] - - @field_validator("uri", "namespace", mode="after") - @classmethod - def _validate_base_url(cls, value: AnyHttpUrl) -> AnyHttpUrl: - """Validate `uri` and `namespace` starts with the current base URL for the - service.""" - if not str(value).startswith(str(CONFIG.base_url)): - error_message = ( - "This service only works with DLite/SOFT entities at " - f"{CONFIG.base_url}.\n" - ) - raise ValueError(error_message) - return value - - @field_validator("uri", mode="after") - @classmethod - def _validate_uri(cls, value: AnyHttpUrl) -> AnyHttpUrl: - """Validate `uri` is consistent with `name`, `version`, and `namespace`.""" - if URI_REGEX.match(str(value)) is None: - error_message = ( - "The 'uri' is not a valid SOFT7 entity URI. It must be of the form " - f"{str(CONFIG.base_url).rstrip('/')}/{{version}}/{{name}}.\n" - ) - raise ValueError(error_message) - return value - - @field_validator("meta", mode="after") - @classmethod - def _only_support_onto_ns(cls, value: AnyHttpUrl) -> AnyHttpUrl: - """Validate `meta` only refers to onto-ns.com EntitySchema v0.3.""" - if str(value) != "http://onto-ns.com/meta/0.3/EntitySchema": - error_message = ( - "This service only works with DLite/SOFT entities using EntitySchema " - "v0.3 at onto-ns.com as the metadata entity.\n" - ) - raise ValueError(error_message) - return value - - @model_validator(mode="before") - @classmethod - def _check_cross_dependent_fields(cls, data: Any) -> Any: - """Check that `name`, `version`, and `namespace` are all set or all unset.""" - if ( - isinstance(data, dict) - and any(data.get(_) is None for _ in ("name", "version", "namespace")) - and not all(data.get(_) is None for _ in ("name", "version", "namespace")) - ): - error_message = ( - "Either all of `name`, `version`, and `namespace` must be set " - "or all must be unset.\n" - ) - raise ValueError(error_message) - - if ( - isinstance(data, dict) - and any(data.get(_) is None for _ in ("name", "version", "namespace")) - and data.get("uri") is None - ): - error_message = ( - "Either `name`, `version`, and `namespace` or `uri` must be set.\n" - ) - raise ValueError(error_message) - - if ( - isinstance(data, dict) - and all(data.get(_) is not None for _ in ("name", "version", "namespace")) - and data.get("uri") is not None - and data["uri"] != f"{data['namespace']}/{data['version']}/{data['name']}" - ): - # Ensure that `uri` is consistent with `name`, `version`, and `namespace`. - diff = "\n ".join( - difflib.ndiff( - [data["uri"]], - [f"{data['namespace']}/{data['version']}/{data['name']}"], - ) - ) - error_message = ( - "The `uri` is not consistent with `name`, `version`, and " - f"`namespace`:\n\n {diff}\n\n" - ) - raise ValueError(error_message) - return data diff --git a/tests/cli/test_upload.py b/tests/cli/test_upload.py index d3907d9f..905014ac 100644 --- a/tests/cli/test_upload.py +++ b/tests/cli/test_upload.py @@ -61,8 +61,8 @@ def test_upload_filepath_invalid( ) assert result.exit_code == 1, result.stdout assert "Person.json is not a valid SOFT entity:" in result.stderr.replace("\n", "") - assert "validation error for SOFT7Entity" in result.stderr.replace("\n", "") - assert "validation errors for SOFT5Entity" in result.stderr.replace("\n", "") + assert "validation error for DLiteSOFT7Entity" in result.stderr.replace("\n", "") + assert "validation errors for DLiteSOFT5Entity" in result.stderr.replace("\n", "") assert not result.stdout if fail_fast: assert ( @@ -215,13 +215,13 @@ def test_upload_directory_invalid_entities( assert result.exit_code == 1, result.stderr assert ( re.search( - r"validation errors? for SOFT7Entity", result.stderr.replace("\n", "") + r"validation errors? for DLiteSOFT7Entity", result.stderr.replace("\n", "") ) is not None ) assert ( re.search( - r"validation errors? for SOFT5Entity", result.stderr.replace("\n", "") + r"validation errors? for DLiteSOFT5Entity", result.stderr.replace("\n", "") ) is not None ) diff --git a/tests/models/test_init.py b/tests/models/test_init.py index cff52837..2f4a4426 100644 --- a/tests/models/test_init.py +++ b/tests/models/test_init.py @@ -15,8 +15,7 @@ def test_soft_entity(static_dir: Path) -> None: import json from dlite_entities_service.models import soft_entity - from dlite_entities_service.models.soft5 import SOFT5Entity - from dlite_entities_service.models.soft7 import SOFT7Entity + from dlite_entities_service.models.dlite import DLiteSOFT5Entity, DLiteSOFT7Entity # Test that the function returns the correct version of the entity soft5_model_file = static_dir / "valid_entities" / "Cat.json" @@ -25,8 +24,8 @@ def test_soft_entity(static_dir: Path) -> None: soft5_model = json.loads(soft5_model_file.read_text()) soft7_model = json.loads(soft7_model_file.read_text()) - assert soft_entity(**soft5_model) == SOFT5Entity(**soft5_model) - assert soft_entity(**soft7_model) == SOFT7Entity(**soft7_model) + assert soft_entity(**soft5_model) == DLiteSOFT5Entity(**soft5_model) + assert soft_entity(**soft7_model) == DLiteSOFT7Entity(**soft7_model) def test_soft_entity_error(static_dir: Path) -> None: @@ -36,6 +35,7 @@ def test_soft_entity_error(static_dir: Path) -> None: from pydantic import ValidationError from dlite_entities_service.models import soft_entity + from dlite_entities_service.models.dlite import DLiteSOFT5Entity, DLiteSOFT7Entity from dlite_entities_service.models.soft5 import SOFT5Entity from dlite_entities_service.models.soft7 import SOFT7Entity @@ -49,6 +49,12 @@ def test_soft_entity_error(static_dir: Path) -> None: errors = soft_entity(return_errors=True, **invalid_model) expected_errors = [] + # The order here is important, as it represents the order in which the models + # are tried in the soft_entity function. + # The order is defined by the Union arguments in the Entity type: + # Entity = SOFT7Entity | SOFT5Entity | DLiteEntity + # And again the order of the Union arguments in the DLiteEntity type: + # DLiteEntity = DLiteSOFT7Entity | DLiteSOFT5Entity try: SOFT7Entity(**invalid_model) except ValidationError as exc: @@ -59,6 +65,16 @@ def test_soft_entity_error(static_dir: Path) -> None: except ValidationError as exc: expected_errors.append(exc) + try: + DLiteSOFT7Entity(**invalid_model) + except ValidationError as exc: + expected_errors.append(exc) + + try: + DLiteSOFT5Entity(**invalid_model) + except ValidationError as exc: + expected_errors.append(exc) + assert [str(_) for _ in errors] == [str(_) for _ in expected_errors] diff --git a/tests/test_route.py b/tests/test_route.py index d84e9cf7..d24fa267 100644 --- a/tests/test_route.py +++ b/tests/test_route.py @@ -25,7 +25,7 @@ def test_get_entity( name: str, client: TestClient, ) -> None: - """Test the route to retrieve a DLite/SOFT entity.""" + """Test the route to retrieve an entity.""" from fastapi import status with client as client: @@ -35,6 +35,12 @@ def test_get_entity( response.is_success ), f"Response: {response.json()}. Request: {response.request}" assert response.status_code == status.HTTP_200_OK, response.json() + + # Convert SOFT5 properties' 'dims' to 'shape' + for entity_property in entity["properties"]: + if "dims" in entity_property: + entity_property["shape"] = entity_property.pop("dims") + assert (resolved_entity := response.json()) == entity, resolved_entity @@ -59,6 +65,11 @@ def test_get_entity_instance( with client as client: response = client.get(f"/{version}/{name}", timeout=5) + # Convert SOFT5 properties' 'dims' to 'shape' + for entity_property in entity["properties"]: + if "dims" in entity_property: + entity_property["shape"] = entity_property.pop("dims") + assert (resolve_entity := response.json()) == entity, resolve_entity Instance.from_dict(resolve_entity) From 2c887a1cbc8d9b7085ea1805f575d414ea320dbf Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Mon, 29 Jan 2024 14:38:13 +0100 Subject: [PATCH 2/6] Split DLite models to base and multi-inheritance --- entities_service/models/__init__.py | 5 +- entities_service/models/dlite.py | 90 +++----------------------- entities_service/models/dlite_soft5.py | 21 ++++++ entities_service/models/dlite_soft7.py | 27 ++++++++ tests/models/test_init.py | 10 +-- 5 files changed, 64 insertions(+), 89 deletions(-) create mode 100644 entities_service/models/dlite_soft5.py create mode 100644 entities_service/models/dlite_soft7.py diff --git a/entities_service/models/__init__.py b/entities_service/models/__init__.py index c44431f3..f1d7eaa9 100644 --- a/entities_service/models/__init__.py +++ b/entities_service/models/__init__.py @@ -5,12 +5,13 @@ from pydantic import ValidationError -from .dlite import DLiteEntity +from .dlite_soft5 import DLiteSOFT5Entity +from .dlite_soft7 import DLiteSOFT7Entity from .soft import URI_REGEX from .soft5 import SOFT5Entity from .soft7 import SOFT7Entity -Entity = SOFT7Entity | SOFT5Entity | DLiteEntity +Entity = SOFT7Entity | SOFT5Entity | DLiteSOFT7Entity | DLiteSOFT5Entity def soft_entity( diff --git a/entities_service/models/dlite.py b/entities_service/models/dlite.py index 8f2f251c..461682e2 100644 --- a/entities_service/models/dlite.py +++ b/entities_service/models/dlite.py @@ -1,17 +1,14 @@ """DLite model.""" from __future__ import annotations -from typing import Annotated +from typing import Annotated, Literal -from pydantic import AliasChoices, Field, field_validator +from pydantic import AliasChoices, BaseModel, Field from pydantic.networks import AnyHttpUrl -from entities_service.models.soft5 import SOFT5Entity, SOFT5Property -from entities_service.models.soft7 import SOFT7Entity, SOFT7Property - -class DLiteSOFT5Property(SOFT5Property): - """The defining metadata for a (SOFT5-based) DLite Entity's property.""" +class DLiteProperty(BaseModel): + """The defining metadata for a DLite Entity's property.""" ref: Annotated[ AnyHttpUrl | None, @@ -26,86 +23,15 @@ class DLiteSOFT5Property(SOFT5Property): ] = None -class DLiteSOFT5Entity(SOFT5Entity): - """A (SOFT5-based) DLite Entity.""" +class DLiteEntity(BaseModel): + """A DLite Entity.""" meta: Annotated[ - AnyHttpUrl, + Literal["http://onto-ns.com/meta/0.3/EntitySchema"], Field( description=( "URI for the metadata entity. For all entities at onto-ns.com, the " "EntitySchema v0.3 is used." ), ), - ] = AnyHttpUrl("http://onto-ns.com/meta/0.3/EntitySchema") - - properties: Annotated[ # type: ignore[assignment] - list[DLiteSOFT5Property], Field(description="A list of properties.") - ] - - @field_validator("meta", mode="after") - @classmethod - def _only_support_onto_ns(cls, value: AnyHttpUrl) -> AnyHttpUrl: - """Validate `meta` only refers to onto-ns.com EntitySchema v0.3.""" - if str(value).rstrip("/") != "http://onto-ns.com/meta/0.3/EntitySchema": - error_message = ( - "This service only works with DLite entities using EntitySchema " - "v0.3 at onto-ns.com as the metadata entity.\n" - ) - raise ValueError(error_message) - return value - - -class DLiteSOFT7Property(SOFT7Property): - """The defining metadata for a (SOFT7-based) DLite Entity's property.""" - - ref: Annotated[ - AnyHttpUrl | None, - Field( - validation_alias=AliasChoices("$ref", "ref"), - serialization_alias="$ref", - description=( - "Formally a part of type. `$ref` is used together with the `ref` type, " - "which is a special datatype for referring to other instances." - ), - ), - ] = None - - -class DLiteSOFT7Entity(SOFT7Entity): - """A (SOFT7-based) DLite Entity.""" - - meta: Annotated[ - AnyHttpUrl, - Field( - description=( - "URI for the metadata entity. For all entities at onto-ns.com, the " - "EntitySchema v0.3 is used." - ), - ), - ] = AnyHttpUrl("http://onto-ns.com/meta/0.3/EntitySchema") - - properties: Annotated[ # type: ignore[assignment] - dict[str, DLiteSOFT7Property], - Field( - description=( - "A dictionary of properties, mapping the property name to a dictionary " - "of metadata defining the property." - ), - ), - ] - - @field_validator("meta", mode="after") - @classmethod - def _only_support_onto_ns(cls, value: AnyHttpUrl) -> AnyHttpUrl: - """Validate `meta` only refers to onto-ns.com EntitySchema v0.3.""" - if str(value).rstrip("/") != "http://onto-ns.com/meta/0.3/EntitySchema": - error_message = ( - "This service only works with DLite entities using EntitySchema " - "v0.3 at onto-ns.com as the metadata entity.\n" - ) - raise ValueError(error_message) - return value - - -DLiteEntity = DLiteSOFT7Entity | DLiteSOFT5Entity + ] = "http://onto-ns.com/meta/0.3/EntitySchema" diff --git a/entities_service/models/dlite_soft5.py b/entities_service/models/dlite_soft5.py new file mode 100644 index 00000000..2170e3c1 --- /dev/null +++ b/entities_service/models/dlite_soft5.py @@ -0,0 +1,21 @@ +"""DLite Entity model based on the SOFT5 Entity model.""" +from __future__ import annotations + +from typing import Annotated + +from pydantic import Field + +from entities_service.models.dlite import DLiteEntity, DLiteProperty +from entities_service.models.soft5 import SOFT5Entity, SOFT5Property + + +class DLiteSOFT5Property(SOFT5Property, DLiteProperty): + """The defining metadata for a (SOFT5-based) DLite Entity's property.""" + + +class DLiteSOFT5Entity(SOFT5Entity, DLiteEntity): + """A (SOFT5-based) DLite Entity.""" + + properties: Annotated[ # type: ignore[assignment] + list[DLiteSOFT5Property], Field(description="A list of properties.") + ] diff --git a/entities_service/models/dlite_soft7.py b/entities_service/models/dlite_soft7.py new file mode 100644 index 00000000..c52d7d3b --- /dev/null +++ b/entities_service/models/dlite_soft7.py @@ -0,0 +1,27 @@ +"""DLite Entity model based on the SOFT7 Entity model.""" +from __future__ import annotations + +from typing import Annotated + +from pydantic import Field + +from entities_service.models.dlite import DLiteEntity, DLiteProperty +from entities_service.models.soft7 import SOFT7Entity, SOFT7Property + + +class DLiteSOFT7Property(SOFT7Property, DLiteProperty): + """The defining metadata for a (SOFT7-based) DLite Entity's property.""" + + +class DLiteSOFT7Entity(SOFT7Entity, DLiteEntity): + """A (SOFT7-based) DLite Entity.""" + + properties: Annotated[ # type: ignore[assignment] + dict[str, DLiteSOFT7Property], + Field( + description=( + "A dictionary of properties, mapping the property name to a dictionary " + "of metadata defining the property." + ), + ), + ] diff --git a/tests/models/test_init.py b/tests/models/test_init.py index dc235ec7..f3d2927e 100644 --- a/tests/models/test_init.py +++ b/tests/models/test_init.py @@ -15,7 +15,8 @@ def test_soft_entity(static_dir: Path) -> None: import json from entities_service.models import soft_entity - from entities_service.models.dlite import DLiteSOFT5Entity, DLiteSOFT7Entity + from entities_service.models.dlite_soft5 import DLiteSOFT5Entity + from entities_service.models.dlite_soft7 import DLiteSOFT7Entity # Test that the function returns the correct version of the entity soft5_model_file = static_dir / "valid_entities" / "Cat.json" @@ -35,7 +36,8 @@ def test_soft_entity_error(static_dir: Path) -> None: from pydantic import ValidationError from entities_service.models import soft_entity - from entities_service.models.dlite import DLiteSOFT5Entity, DLiteSOFT7Entity + from entities_service.models.dlite_soft5 import DLiteSOFT5Entity + from entities_service.models.dlite_soft7 import DLiteSOFT7Entity from entities_service.models.soft5 import SOFT5Entity from entities_service.models.soft7 import SOFT7Entity @@ -52,9 +54,7 @@ def test_soft_entity_error(static_dir: Path) -> None: # The order here is important, as it represents the order in which the models # are tried in the soft_entity function. # The order is defined by the Union arguments in the Entity type: - # Entity = SOFT7Entity | SOFT5Entity | DLiteEntity - # And again the order of the Union arguments in the DLiteEntity type: - # DLiteEntity = DLiteSOFT7Entity | DLiteSOFT5Entity + # Entity = SOFT7Entity | SOFT5Entity | DLiteSOFT7Entity | DLiteSOFT5Entity try: SOFT7Entity(**invalid_model) except ValidationError as exc: From 92138bbaedd2d5c797784adf45124f9a1d6630f5 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Mon, 29 Jan 2024 14:39:57 +0100 Subject: [PATCH 3/6] Rename base model module for DLite models --- entities_service/models/{dlite.py => dlite_soft.py} | 0 entities_service/models/dlite_soft5.py | 2 +- entities_service/models/dlite_soft7.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename entities_service/models/{dlite.py => dlite_soft.py} (100%) diff --git a/entities_service/models/dlite.py b/entities_service/models/dlite_soft.py similarity index 100% rename from entities_service/models/dlite.py rename to entities_service/models/dlite_soft.py diff --git a/entities_service/models/dlite_soft5.py b/entities_service/models/dlite_soft5.py index 2170e3c1..8f04559e 100644 --- a/entities_service/models/dlite_soft5.py +++ b/entities_service/models/dlite_soft5.py @@ -5,7 +5,7 @@ from pydantic import Field -from entities_service.models.dlite import DLiteEntity, DLiteProperty +from entities_service.models.dlite_soft import DLiteEntity, DLiteProperty from entities_service.models.soft5 import SOFT5Entity, SOFT5Property diff --git a/entities_service/models/dlite_soft7.py b/entities_service/models/dlite_soft7.py index c52d7d3b..6b4d679f 100644 --- a/entities_service/models/dlite_soft7.py +++ b/entities_service/models/dlite_soft7.py @@ -5,7 +5,7 @@ from pydantic import Field -from entities_service.models.dlite import DLiteEntity, DLiteProperty +from entities_service.models.dlite_soft import DLiteEntity, DLiteProperty from entities_service.models.soft7 import SOFT7Entity, SOFT7Property From 56f6cff6ac2bfe5f740df6050fce6845f61635a3 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Wed, 31 Jan 2024 14:15:01 +0100 Subject: [PATCH 4/6] Fix test for specific Entity model parsing --- tests/models/test_init.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/tests/models/test_init.py b/tests/models/test_init.py index d46c7a97..4cf5f4e0 100644 --- a/tests/models/test_init.py +++ b/tests/models/test_init.py @@ -17,16 +17,26 @@ def test_soft_entity(static_dir: Path) -> None: from entities_service.models import soft_entity from entities_service.models.dlite_soft5 import DLiteSOFT5Entity from entities_service.models.dlite_soft7 import DLiteSOFT7Entity + from entities_service.models.soft5 import SOFT5Entity + from entities_service.models.soft7 import SOFT7Entity # Test that the function returns the correct version of the entity - soft5_model_file = static_dir / "valid_entities" / "Cat.json" + dlite_soft5_model_file = static_dir / "valid_entities" / "Cat.json" + dlite_soft7_model_file = static_dir / "valid_entities" / "Person.json" soft7_model_file = static_dir / "valid_entities" / "Dog.json" - soft5_model = json.loads(soft5_model_file.read_text()) - soft7_model = json.loads(soft7_model_file.read_text()) + dlite_soft5_model: dict[str, Any] = json.loads(dlite_soft5_model_file.read_text()) + dlite_soft7_model: dict[str, Any] = json.loads(dlite_soft7_model_file.read_text()) + soft7_model: dict[str, Any] = json.loads(soft7_model_file.read_text()) + + assert soft_entity(**dlite_soft5_model) == DLiteSOFT5Entity(**dlite_soft5_model) + assert soft_entity(**dlite_soft7_model) == DLiteSOFT7Entity(**dlite_soft7_model) + assert soft_entity(**soft7_model) == SOFT7Entity(**soft7_model) - assert soft_entity(**soft5_model) == DLiteSOFT5Entity(**soft5_model) - assert soft_entity(**soft7_model) == DLiteSOFT7Entity(**soft7_model) + # Force dlite_soft5_model to parse as a SOFT5Entity + # by removing the `meta` field. + dlite_soft5_model.pop("meta") + assert soft_entity(**dlite_soft5_model) == SOFT5Entity(**dlite_soft5_model) def test_soft_entity_error(static_dir: Path) -> None: From 0cd60090741f84d4392df8a4728e83760f3a420b Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Wed, 31 Jan 2024 14:17:37 +0100 Subject: [PATCH 5/6] Define EntityType smarter --- entities_service/models/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/entities_service/models/__init__.py b/entities_service/models/__init__.py index 30ebe4c5..36b56f7b 100644 --- a/entities_service/models/__init__.py +++ b/entities_service/models/__init__.py @@ -15,7 +15,7 @@ from typing import Literal Entity = SOFT7Entity | SOFT5Entity | DLiteSOFT7Entity | DLiteSOFT5Entity -EntityType = (SOFT7Entity, SOFT5Entity, DLiteSOFT7Entity, DLiteSOFT5Entity) +EntityType = get_args(Entity) @overload From c431f73e7be7d68880eb209ac7f939ed24a50ceb Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Fri, 2 Feb 2024 09:19:47 +0100 Subject: [PATCH 6/6] Run newest version of black --- entities_service/models/dlite_soft.py | 1 + entities_service/models/dlite_soft5.py | 1 + entities_service/models/dlite_soft7.py | 1 + entities_service/models/soft.py | 7 ++++--- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/entities_service/models/dlite_soft.py b/entities_service/models/dlite_soft.py index 461682e2..63f3165b 100644 --- a/entities_service/models/dlite_soft.py +++ b/entities_service/models/dlite_soft.py @@ -1,4 +1,5 @@ """DLite model.""" + from __future__ import annotations from typing import Annotated, Literal diff --git a/entities_service/models/dlite_soft5.py b/entities_service/models/dlite_soft5.py index 8f04559e..efb1db69 100644 --- a/entities_service/models/dlite_soft5.py +++ b/entities_service/models/dlite_soft5.py @@ -1,4 +1,5 @@ """DLite Entity model based on the SOFT5 Entity model.""" + from __future__ import annotations from typing import Annotated diff --git a/entities_service/models/dlite_soft7.py b/entities_service/models/dlite_soft7.py index 6b4d679f..c8a9e1f6 100644 --- a/entities_service/models/dlite_soft7.py +++ b/entities_service/models/dlite_soft7.py @@ -1,4 +1,5 @@ """DLite Entity model based on the SOFT7 Entity model.""" + from __future__ import annotations from typing import Annotated diff --git a/entities_service/models/soft.py b/entities_service/models/soft.py index 6cdf6dd2..8cc47c87 100644 --- a/entities_service/models/soft.py +++ b/entities_service/models/soft.py @@ -1,4 +1,5 @@ """Base (minimum set) models for SOFT entities.""" + from __future__ import annotations import difflib @@ -61,9 +62,9 @@ class SOFTEntity(BaseModel): model_config = ConfigDict(extra="forbid") name: Annotated[str | None, Field(description="The name of the entity.")] = None - version: Annotated[ - str | None, Field(description="The version of the entity.") - ] = None + version: Annotated[str | None, Field(description="The version of the entity.")] = ( + None + ) namespace: Annotated[ AnyHttpUrl | None, Field(description="The namespace of the entity.") ] = None