From ce33e7ee5c874e9832e595b49d9c296d04fc0553 Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Fri, 27 Oct 2023 12:40:09 +0200 Subject: [PATCH 1/8] make catalog and /collections customizable --- CHANGES.md | 34 +++++++++ tipg/collections.py | 12 +++- tipg/dependencies.py | 165 +++++++++++++++++++++++++++++++++++-------- tipg/factory.py | 139 ++++++------------------------------ tipg/middleware.py | 9 ++- 5 files changed, 212 insertions(+), 147 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 6c2cf124..0e6bcf20 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,9 +8,43 @@ Note: Minor version `0.X.0` update might break the API, It's recommended to pin ## [unreleased] +### fixed + - hide map element in HTML pages when collections/items do not have spatial component + +### changed + - split endpoints registration for more customization +- make `Catalog` a Pydantic model and add `matched`, `next` and `prev` attributes + + ```python + # Before + class Catalog(TypedDict): + """Collection Catalog.""" + + collections: Dict[str, Collection] + last_updated: datetime.datetime + + # Now + class Catalog(BaseModel): + """Collection Catalog.""" + + collections: Dict[str, Collection] + last_updated: datetime.datetime + matched: Optional[int] = None + next: Optional[int] = None + prev: Optional[int] = None + ``` + +- move `/collections` QueryParameters in the `CatalogParams` dependency + +- the `CatalogParams` now returns a `Catalog` object + +- move `s_intersects` and `t_intersects` functions from `tipg.factory` to `tipg.dependencies` + +- move the `catalog_dependency` attribute from the `EndpointsFactory` to `OGCFeaturesFactory` class + ## [0.4.4] - 2023-10-03 ### fixed diff --git a/tipg/collections.py b/tipg/collections.py index da88cce9..05577ee5 100644 --- a/tipg/collections.py +++ b/tipg/collections.py @@ -876,11 +876,21 @@ def queryables(self) -> Dict: return {**geoms, **props} -class Catalog(TypedDict): +class Catalog(BaseModel): """Collection Catalog.""" collections: Dict[str, Collection] last_updated: datetime.datetime + matched: Optional[int] = None + next: Optional[int] = None + prev: Optional[int] = None + + @model_validator(mode="after") + def compute_matched(self): + """Compute matched if it does not exist.""" + if self.matched is None: + self.matched = len(self.collections) + return self async def get_collection_index( # noqa: C901 diff --git a/tipg/dependencies.py b/tipg/dependencies.py index a1f87c1d..9c75807f 100644 --- a/tipg/dependencies.py +++ b/tipg/dependencies.py @@ -3,6 +3,7 @@ import re from typing import Dict, List, Literal, Optional, Tuple, get_args +from ciso8601 import parse_rfc3339 from morecantile import Tile from morecantile import tms as default_tms from pygeofilter.ast import AstType @@ -15,7 +16,7 @@ from tipg.resources.enums import MediaType from tipg.settings import TMSSettings -from fastapi import HTTPException, Path, Query +from fastapi import Depends, HTTPException, Path, Query from starlette.requests import Request @@ -30,41 +31,43 @@ FilterLang = Literal["cql2-text", "cql2-json"] -def CollectionParams( - request: Request, - collectionId: Annotated[str, Path(description="Collection identifier")], -) -> Collection: - """Return Layer Object.""" - collection_pattern = re.match( # type: ignore - r"^(?P.+)\.(?P.+)$", collectionId +def s_intersects(bbox: List[float], spatial_extent: List[float]) -> bool: + """Check if bbox intersects with spatial extent.""" + return ( + (bbox[0] < spatial_extent[2]) + and (bbox[2] > spatial_extent[0]) + and (bbox[3] > spatial_extent[1]) + and (bbox[1] < spatial_extent[3]) ) - if not collection_pattern: - raise HTTPException( - status_code=422, detail=f"Invalid Collection format '{collectionId}'." - ) - assert collection_pattern.groupdict()["schema"] - assert collection_pattern.groupdict()["collection"] - collection_catalog: Catalog = getattr(request.app.state, "collection_catalog", None) - if not collection_catalog: - raise MissingCollectionCatalog("Could not find collections catalog.") +def t_intersects(interval: List[str], temporal_extent: List[str]) -> bool: + """Check if dates intersect with temporal extent.""" + if len(interval) == 1: + start = end = parse_rfc3339(interval[0]) - if collectionId in collection_catalog["collections"]: - return collection_catalog["collections"][collectionId] + else: + start = parse_rfc3339(interval[0]) if interval[0] not in ["..", ""] else None + end = parse_rfc3339(interval[1]) if interval[1] not in ["..", ""] else None - raise HTTPException( - status_code=404, detail=f"Table/Function '{collectionId}' not found." - ) + mint, maxt = temporal_extent + min_ext = parse_rfc3339(mint) if mint is not None else None + max_ext = parse_rfc3339(maxt) if maxt is not None else None + if len(interval) == 1: + if start == min_ext or start == max_ext: + return True -def CatalogParams(request: Request) -> Catalog: - """Return Collections Catalog.""" - collection_catalog: Catalog = getattr(request.app.state, "collection_catalog", None) - if not collection_catalog: - raise MissingCollectionCatalog("Could not find collections catalog.") + if not start: + return max_ext <= end or min_ext <= end + + elif not end: + return min_ext >= start or max_ext >= start - return collection_catalog + else: + return min_ext >= start and max_ext <= end + + return False def accept_media_type(accept: str, mediatypes: List[MediaType]) -> Optional[MediaType]: @@ -397,3 +400,109 @@ def function_parameters_query( # noqa: C901 ) return function_parameters + + +def CollectionParams( + request: Request, + collectionId: Annotated[str, Path(description="Collection identifier")], +) -> Collection: + """Return Layer Object.""" + collection_pattern = re.match( # type: ignore + r"^(?P.+)\.(?P.+)$", collectionId + ) + if not collection_pattern: + raise HTTPException( + status_code=422, detail=f"Invalid Collection format '{collectionId}'." + ) + + assert collection_pattern.groupdict()["schema"] + assert collection_pattern.groupdict()["collection"] + + catalog: Catalog = getattr(request.app.state, "collection_catalog", None) + if not catalog: + raise MissingCollectionCatalog("Could not find collections catalog.") + + if collectionId in catalog.collections: + return catalog.collections[collectionId] + + raise HTTPException( + status_code=404, detail=f"Table/Function '{collectionId}' not found." + ) + + +def CatalogParams( + request: Request, + bbox_filter: Annotated[Optional[List[float]], Depends(bbox_query)], + datetime_filter: Annotated[Optional[List[str]], Depends(datetime_query)], + type_filter: Annotated[ + Optional[Literal["Function", "Table"]], + Query(alias="type", description="Filter based on Collection type."), + ] = None, + limit: Annotated[ + Optional[int], + Query( + ge=0, + le=1000, + description="Limits the number of collection in the response.", + ), + ] = None, + offset: Annotated[ + Optional[int], + Query( + ge=0, + description="Starts the response at an offset.", + ), + ] = None, +) -> Catalog: + """Return Collections Catalog.""" + limit = limit or 0 + offset = offset or 0 + + catalog: Catalog = getattr(request.app.state, "collection_catalog", None) + if not catalog: + raise MissingCollectionCatalog("Could not find collections catalog.") + + collections_list = list(catalog.collections.values()) + + # type filter + if type_filter is not None: + collections_list = [ + collection + for collection in collections_list + if collection.type == type_filter + ] + + # bbox filter + if bbox_filter is not None: + collections_list = [ + collection + for collection in collections_list + if collection.bounds is not None + and s_intersects(bbox_filter, collection.bounds) + ] + + # datetime filter + if datetime_filter is not None: + collections_list = [ + collection + for collection in collections_list + if collection.dt_bounds is not None + and t_intersects(datetime_filter, collection.dt_bounds) + ] + + matched = len(collections_list) + + if limit: + collections_list = collections_list[offset : offset + limit] + else: + collections_list = collections_list[offset:] + + returned = len(collections_list) + + return Catalog( + collections={col.id: col for col in collections_list}, + last_updated=catalog.last_updated, + matched=matched, + next=offset + returned if matched - returned > offset else None, + prev=max(offset - returned, 0) if offset else None, + ) diff --git a/tipg/factory.py b/tipg/factory.py index 021cf7b6..2a9afe16 100644 --- a/tipg/factory.py +++ b/tipg/factory.py @@ -9,7 +9,6 @@ import jinja2 import orjson -from ciso8601 import parse_rfc3339 from morecantile import Tile, TileMatrixSet from morecantile import tms as default_tms from morecantile.defaults import TileMatrixSets @@ -110,45 +109,6 @@ def write(self, line: str): yield writer.writerow(row) -def s_intersects(bbox: List[float], spatial_extent: List[float]) -> bool: - """Check if bbox intersects with spatial extent.""" - return ( - (bbox[0] < spatial_extent[2]) - and (bbox[2] > spatial_extent[0]) - and (bbox[3] > spatial_extent[1]) - and (bbox[1] < spatial_extent[3]) - ) - - -def t_intersects(interval: List[str], temporal_extent: List[str]) -> bool: - """Check if dates intersect with temporal extent.""" - if len(interval) == 1: - start = end = parse_rfc3339(interval[0]) - - else: - start = parse_rfc3339(interval[0]) if interval[0] not in ["..", ""] else None - end = parse_rfc3339(interval[1]) if interval[1] not in ["..", ""] else None - - mint, maxt = temporal_extent - min_ext = parse_rfc3339(mint) if mint is not None else None - max_ext = parse_rfc3339(maxt) if maxt is not None else None - - if len(interval) == 1: - if start == min_ext or start == max_ext: - return True - - if not start: - return max_ext <= end or min_ext <= end - - elif not end: - return min_ext >= start or max_ext >= start - - else: - return min_ext >= start and max_ext <= end - - return False - - def create_html_response( request: Request, data: str, @@ -201,7 +161,6 @@ class EndpointsFactory(metaclass=abc.ABCMeta): router: APIRouter = field(default_factory=APIRouter) # collection dependency - catalog_dependency: Callable[..., Catalog] = CatalogParams collection_dependency: Callable[..., Collection] = CollectionParams # Router Prefix is needed to find the path for routes when prefixed @@ -372,6 +331,9 @@ def landing( class OGCFeaturesFactory(EndpointsFactory): """OGC Features Endpoints Factory.""" + # catalog dependency + catalog_dependency: Callable[..., Catalog] = CatalogParams + @property def conforms_to(self) -> List[str]: """Factory conformances.""" @@ -448,73 +410,18 @@ def _collections_route(self): # noqa: C901 }, }, ) - def collections( # noqa: C901 + def collections( request: Request, - bbox_filter: Annotated[Optional[List[float]], Depends(bbox_query)], - datetime_filter: Annotated[Optional[List[str]], Depends(datetime_query)], - type_filter: Annotated[ - Optional[Literal["Function", "Table"]], - Query(alias="type", description="Filter based on Collection type."), - ] = None, - limit: Annotated[ - Optional[int], - Query( - ge=0, - le=1000, - description="Limits the number of collection in the response.", - ), - ] = None, - offset: Annotated[ - Optional[int], - Query( - ge=0, - description="Starts the response at an offset.", - ), + catalog: Annotated[ + Catalog, + Depends(self.catalog_dependency), + ], + output_type: Annotated[ + Optional[MediaType], + Depends(OutputType), ] = None, - output_type: Annotated[Optional[MediaType], Depends(OutputType)] = None, - collection_catalog=Depends(self.catalog_dependency), ): """List of collections.""" - collections_list = list(collection_catalog["collections"].values()) - - limit = limit or 0 - offset = offset or 0 - - # type filter - if type_filter is not None: - collections_list = [ - collection - for collection in collections_list - if collection.type == type_filter - ] - - # bbox filter - if bbox_filter is not None: - collections_list = [ - collection - for collection in collections_list - if collection.bounds is not None - and s_intersects(bbox_filter, collection.bounds) - ] - - # datetime filter - if datetime_filter is not None: - collections_list = [ - collection - for collection in collections_list - if collection.dt_bounds is not None - and t_intersects(datetime_filter, collection.dt_bounds) - ] - - matched_items = len(collections_list) - - if limit: - collections_list = collections_list[offset : offset + limit] - else: - collections_list = collections_list[offset:] - - items_returned = len(collections_list) - links: list = [ model.Link( href=self.url_for(request, "collections"), @@ -523,10 +430,9 @@ def collections( # noqa: C901 ), ] - if (matched_items - items_returned) > offset: - next_offset = offset + items_returned + if catalog.next: query_params = QueryParams( - {**request.query_params, "offset": next_offset} + {**request.query_params, "offset": catalog.next} ) url = self.url_for(request, "collections") + f"?{query_params}" links.append( @@ -538,12 +444,11 @@ def collections( # noqa: C901 ), ) - if offset: + if catalog.prev is not None: qp = dict(request.query_params) - qp.pop("offset") - prev_offset = max(offset - items_returned, 0) - if prev_offset: - query_params = QueryParams({**qp, "offset": prev_offset}) + qp.pop("offset", None) + if catalog.prev: + query_params = QueryParams({**qp, "offset": catalog.prev}) else: query_params = QueryParams({**qp}) @@ -562,8 +467,8 @@ def collections( # noqa: C901 data = model.Collections( links=links, - numberMatched=matched_items, - numberReturned=items_returned, + numberMatched=catalog.matched, + numberReturned=len(catalog.collections), collections=[ model.Collection( id=collection.id, @@ -600,7 +505,7 @@ def collections( # noqa: C901 ), ], ) - for collection in collections_list + for collection in catalog.collections.values() ], ) @@ -1947,6 +1852,9 @@ def viewer_endpoint( class Endpoints(EndpointsFactory): """OGC Features and Tiles Endpoints Factory.""" + # OGC Features dependency + catalog_dependency: Callable[..., Catalog] = CatalogParams + # OGC Tiles dependency supported_tms: TileMatrixSets = default_tms with_tiles_viewer: bool = True @@ -1982,7 +1890,6 @@ def register_routes(self): self.router.include_router(self.ogc_features.router, tags=["OGC Features API"]) self.ogc_tiles = OGCTilesFactory( - catalog_dependency=self.catalog_dependency, collection_dependency=self.collection_dependency, router_prefix=self.router_prefix, templates=self.templates, diff --git a/tipg/middleware.py b/tipg/middleware.py index 92023635..566909e6 100644 --- a/tipg/middleware.py +++ b/tipg/middleware.py @@ -4,6 +4,8 @@ from datetime import datetime, timedelta from typing import Any, Optional, Protocol, Set +from tipg.collections import Catalog +from tipg.errors import MissingCollectionCatalog from tipg.logger import logger from starlette.background import BackgroundTask @@ -97,8 +99,11 @@ async def __call__(self, scope: Scope, receive: Receive, send: Send): request = Request(scope) background: Optional[BackgroundTask] = None - collection_catalog = getattr(request.app.state, "collection_catalog", {}) - last_updated = collection_catalog.get("last_updated") + catalog: Catalog = getattr(request.app.state, "collection_catalog", None) + if not catalog: + raise MissingCollectionCatalog("Could not find collections catalog.") + + last_updated = catalog.last_updated if not last_updated or datetime.now() > ( last_updated + timedelta(seconds=self.ttl) ): From 043d846fef2514aa8c82668d7d642aef02a05900 Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Fri, 27 Oct 2023 12:42:40 +0200 Subject: [PATCH 2/8] update docs --- docs/src/user_guide/factories.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/src/user_guide/factories.md b/docs/src/user_guide/factories.md index 82ea4ee0..7c49a2a5 100644 --- a/docs/src/user_guide/factories.md +++ b/docs/src/user_guide/factories.md @@ -61,6 +61,8 @@ app.include_router(endpoints.router, tags=["OGC Features API"]) #### Creation Options +- **catalog_dependency** (Callable[..., tipg.collections.Catalog]): Callable which return a Catalog instance + - **collection_dependency** (Callable[..., tipg.collections.Collection]): Callable which return a Collection instance - **with_common** (bool, optional): Create Full OGC Features API set of endpoints with OGC Common endpoints (landing `/` and conformance `/conformance`). Defaults to `True` @@ -141,6 +143,8 @@ app.include_router(endpoints.router) #### Creation Options +- **catalog_dependency** (Callable[..., tipg.collections.Catalog]): Callable which return a Catalog instance + - **collection_dependency** (Callable[..., tipg.collections.Collection]): Callable which return a Collection instance - **supported_tms** (morecantile.TileMatrixSets): morecantile TileMatrixSets instance (holds a set of TileMatrixSet documents) From f36eb3c6445bdb0004fc339ea1da6deca03fc117 Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Mon, 30 Oct 2023 23:13:02 +0100 Subject: [PATCH 3/8] revert and add Item/CollectionList --- CHANGES.md | 50 +++++++++++------- tipg/collections.py | 48 ++++++++++-------- tipg/dependencies.py | 15 +++--- tipg/factory.py | 118 +++++++++++++++---------------------------- tipg/middleware.py | 2 +- 5 files changed, 108 insertions(+), 125 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index a50e736a..14529ef8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -12,6 +12,26 @@ Note: Minor version `0.X.0` update might break the API, It's recommended to pin - add `py.typed` file +- add `ItemList` and `CollectionList` *TypedDict* + + ```python + class ItemList(TypedDict): + """Items.""" + + items: List[Feature] + matched: Optional[int] + next: Optional[int] + prev: Optional[int] + + class CollectionList(TypedDict): + """Collections.""" + + collections: List[Collection] + matched: Optional[int] + next: Optional[int] + prev: Optional[int] + ``` + ### fixed - hide map element in HTML pages when collections/items do not have spatial component (https://github.com/developmentseed/tipg/issues/132) @@ -47,30 +67,24 @@ Note: Minor version `0.X.0` update might break the API, It's recommended to pin ... ``` -- make `Catalog` a Pydantic model and add `matched`, `next` and `prev` attributes +- `Collection.features()` method now returns an `ItemList` dict ```python - # Before - class Catalog(TypedDict): - """Collection Catalog.""" - - collections: Dict[str, Collection] - last_updated: datetime.datetime - - # Now - class Catalog(BaseModel): - """Collection Catalog.""" - - collections: Dict[str, Collection] - last_updated: datetime.datetime - matched: Optional[int] = None - next: Optional[int] = None - prev: Optional[int] = None + #before + collection = Collection() + features_collection, matched = collection.features(...) + + #now + collection = Collection() + items_list = collection.features(...) + print(items_list["matched"]) # Number of matched items for the query + print(items_list["next"]) # Next Offset + print(items_list["prev"]) # Previous Offset ``` - move `/collections` QueryParameters in the `CatalogParams` dependency -- the `CatalogParams` now returns a `Catalog` object +- the `CatalogParams` now returns a `CollectionList` object - move `s_intersects` and `t_intersects` functions from `tipg.factory` to `tipg.dependencies` diff --git a/tipg/collections.py b/tipg/collections.py index 05577ee5..e96cf1ed 100644 --- a/tipg/collections.py +++ b/tipg/collections.py @@ -76,12 +76,13 @@ class Feature(TypedDict, total=False): bbox: Optional[List[float]] -class FeatureCollection(TypedDict, total=False): - """Simple FeatureCollection model.""" +class ItemList(TypedDict): + """Items.""" - type: str - features: List[Feature] - bbox: Optional[List[float]] + items: List[Feature] + matched: Optional[int] + next: Optional[int] + prev: Optional[int] class Column(BaseModel): @@ -739,8 +740,9 @@ async def features( simplify: Optional[float] = None, geom_as_wkt: bool = False, function_parameters: Optional[Dict[str, str]] = None, - ) -> Tuple[FeatureCollection, int]: + ) -> ItemList: """Build and run Pg query.""" + offset = offset or 0 function_parameters = function_parameters or {} if geom and geom.lower() != "none" and not self.get_geometry_column(geom): @@ -751,7 +753,7 @@ async def features( f"Limit can not be set higher than the `tipg_max_features_per_query` setting of {features_settings.max_features_per_query}" ) - count = await self._features_count_query( + matched = await self._features_count_query( pool=pool, ids_filter=ids_filter, datetime_filter=datetime_filter, @@ -784,10 +786,13 @@ async def features( function_parameters=function_parameters, ) ] + returned = len(features) - return ( - FeatureCollection(type="FeatureCollection", features=features), - count, + return ItemList( + items=features, + matched=matched, + next=offset + returned if matched - returned > offset else None, + prev=max(offset - returned, 0) if offset else None, ) async def get_tile( @@ -876,21 +881,20 @@ def queryables(self) -> Dict: return {**geoms, **props} -class Catalog(BaseModel): - """Collection Catalog.""" +class CollectionList(TypedDict): + """Collections.""" + + collections: List[Collection] + matched: Optional[int] + next: Optional[int] + prev: Optional[int] + + +class Catalog(TypedDict): + """Internal Collection Catalog.""" collections: Dict[str, Collection] last_updated: datetime.datetime - matched: Optional[int] = None - next: Optional[int] = None - prev: Optional[int] = None - - @model_validator(mode="after") - def compute_matched(self): - """Compute matched if it does not exist.""" - if self.matched is None: - self.matched = len(self.collections) - return self async def get_collection_index( # noqa: C901 diff --git a/tipg/dependencies.py b/tipg/dependencies.py index 9c75807f..cd4bf280 100644 --- a/tipg/dependencies.py +++ b/tipg/dependencies.py @@ -11,7 +11,7 @@ from pygeofilter.parsers.cql2_text import parse as cql2_text_parser from typing_extensions import Annotated -from tipg.collections import Catalog, Collection +from tipg.collections import Catalog, Collection, CollectionList from tipg.errors import InvalidBBox, MissingCollectionCatalog, MissingFunctionParameter from tipg.resources.enums import MediaType from tipg.settings import TMSSettings @@ -422,8 +422,8 @@ def CollectionParams( if not catalog: raise MissingCollectionCatalog("Could not find collections catalog.") - if collectionId in catalog.collections: - return catalog.collections[collectionId] + if collectionId in catalog["collections"]: + return catalog["collections"][collectionId] raise HTTPException( status_code=404, detail=f"Table/Function '{collectionId}' not found." @@ -453,7 +453,7 @@ def CatalogParams( description="Starts the response at an offset.", ), ] = None, -) -> Catalog: +) -> CollectionList: """Return Collections Catalog.""" limit = limit or 0 offset = offset or 0 @@ -462,7 +462,7 @@ def CatalogParams( if not catalog: raise MissingCollectionCatalog("Could not find collections catalog.") - collections_list = list(catalog.collections.values()) + collections_list = list(catalog["collections"].values()) # type filter if type_filter is not None: @@ -499,9 +499,8 @@ def CatalogParams( returned = len(collections_list) - return Catalog( - collections={col.id: col for col in collections_list}, - last_updated=catalog.last_updated, + return CollectionList( + collections=collections_list, matched=matched, next=offset + returned if matched - returned > offset else None, prev=max(offset - returned, 0) if offset else None, diff --git a/tipg/factory.py b/tipg/factory.py index 2a9afe16..04046699 100644 --- a/tipg/factory.py +++ b/tipg/factory.py @@ -16,7 +16,7 @@ from typing_extensions import Annotated from tipg import model -from tipg.collections import Catalog, Collection +from tipg.collections import Collection, CollectionList from tipg.dependencies import ( CatalogParams, CollectionParams, @@ -332,7 +332,7 @@ class OGCFeaturesFactory(EndpointsFactory): """OGC Features Endpoints Factory.""" # catalog dependency - catalog_dependency: Callable[..., Catalog] = CatalogParams + catalog_dependency: Callable[..., CollectionList] = CatalogParams @property def conforms_to(self) -> List[str]: @@ -412,8 +412,8 @@ def _collections_route(self): # noqa: C901 ) def collections( request: Request, - catalog: Annotated[ - Catalog, + collection_list: Annotated[ + CollectionList, Depends(self.catalog_dependency), ], output_type: Annotated[ @@ -430,9 +430,9 @@ def collections( ), ] - if catalog.next: + if next_token := collection_list["next"]: query_params = QueryParams( - {**request.query_params, "offset": catalog.next} + {**request.query_params, "offset": next_token} ) url = self.url_for(request, "collections") + f"?{query_params}" links.append( @@ -444,13 +444,11 @@ def collections( ), ) - if catalog.prev is not None: + if collection_list["prev"] is not None: + prev_token = collection_list["prev"] qp = dict(request.query_params) qp.pop("offset", None) - if catalog.prev: - query_params = QueryParams({**qp, "offset": catalog.prev}) - else: - query_params = QueryParams({**qp}) + query_params = QueryParams({**qp, "offset": prev_token}) url = self.url_for(request, "collections") if qp: @@ -467,8 +465,8 @@ def collections( data = model.Collections( links=links, - numberMatched=catalog.matched, - numberReturned=len(catalog.collections), + numberMatched=collection_list["matched"], + numberReturned=len(collection_list["collections"]), collections=[ model.Collection( id=collection.id, @@ -505,7 +503,7 @@ def collections( ), ], ) - for collection in catalog.collections.values() + for collection in collection_list["collections"] ], ) @@ -730,7 +728,7 @@ async def items( # noqa: C901 MediaType.html, ] - items, matched_items = await collection.features( + item_list = await collection.features( request.app.state.pool, ids_filter=ids_filter, bbox_filter=bbox_filter, @@ -754,29 +752,19 @@ async def items( # noqa: C901 MediaType.json, MediaType.ndjson, ): - if ( - items["features"] - and items["features"][0].get("geometry") is not None - ): - rows = ( - { - "collectionId": collection.id, - "itemId": f.get("id"), - **f.get("properties", {}), - "geometry": f["geometry"], - } - for f in items["features"] - ) - - else: - rows = ( - { + rows = ( + { + k: v + for k, v in { "collectionId": collection.id, "itemId": f.get("id"), **f.get("properties", {}), - } - for f in items["features"] - ) + "geometry": f.get("geometry", None), + }.items() + if v is not None + } + for f in item_list["items"] + ) # CSV Response if output_type == MediaType.csv: @@ -821,12 +809,9 @@ async def items( # noqa: C901 }, ] - items_returned = len(items["features"]) - - if (matched_items - items_returned) > offset: - next_offset = offset + items_returned + if next_token := item_list["next"]: query_params = QueryParams( - {**request.query_params, "offset": next_offset} + {**request.query_params, "offset": next_token} ) url = ( self.url_for(request, "items", collectionId=collection.id) @@ -841,15 +826,11 @@ async def items( # noqa: C901 }, ) - if offset: + if item_list["prev"] is not None: + prev_token = item_list["prev"] qp = dict(request.query_params) qp.pop("offset") - prev_offset = max(offset - items_returned, 0) - if prev_offset: - query_params = QueryParams({**qp, "offset": prev_offset}) - else: - query_params = QueryParams({**qp}) - + query_params = QueryParams({**qp, "offset": prev_token}) url = self.url_for(request, "items", collectionId=collection.id) if qp: url += f"?{query_params}" @@ -870,8 +851,8 @@ async def items( # noqa: C901 "description": collection.description or collection.title or collection.id, - "numberMatched": matched_items, - "numberReturned": items_returned, + "numberMatched": item_list["matched"], + "numberReturned": len(item_list["items"]), "links": links, "features": [ { @@ -900,7 +881,7 @@ async def items( # noqa: C901 }, ], } - for feature in items["features"] + for feature in item_list["items"] ], } @@ -987,7 +968,7 @@ async def item( MediaType.html, ] - items, _ = await collection.features( + item_list = await collection.features( pool=request.app.state.pool, bbox_only=bbox_only, simplify=simplify, @@ -999,41 +980,26 @@ async def item( geom_as_wkt=geom_as_wkt, ) - features = items.get("features", []) - if not features: + if not item_list["items"]: raise NotFound( f"Item {itemId} in Collection {collection.id} does not exist." ) - feature = features[0] + feature = item_list["items"][0] if output_type in ( MediaType.csv, MediaType.json, MediaType.ndjson, ): + row = { + "collectionId": collection.id, + "itemId": feature.get("id"), + **feature.get("properties", {}), + } if feature.get("geometry") is not None: - rows = iter( - [ - { - "collectionId": collection.id, - "itemId": feature.get("id"), - **feature.get("properties", {}), - "geometry": feature["geometry"], - }, - ] - ) - - else: - rows = iter( - [ - { - "collectionId": collection.id, - "itemId": feature.get("id"), - **feature.get("properties", {}), - }, - ] - ) + row["geometry"] = (feature["geometry"],) + rows = iter([row]) # CSV Response if output_type == MediaType.csv: @@ -1853,7 +1819,7 @@ class Endpoints(EndpointsFactory): """OGC Features and Tiles Endpoints Factory.""" # OGC Features dependency - catalog_dependency: Callable[..., Catalog] = CatalogParams + catalog_dependency: Callable[..., CollectionList] = CatalogParams # OGC Tiles dependency supported_tms: TileMatrixSets = default_tms diff --git a/tipg/middleware.py b/tipg/middleware.py index 566909e6..476e4362 100644 --- a/tipg/middleware.py +++ b/tipg/middleware.py @@ -103,7 +103,7 @@ async def __call__(self, scope: Scope, receive: Receive, send: Send): if not catalog: raise MissingCollectionCatalog("Could not find collections catalog.") - last_updated = catalog.last_updated + last_updated = catalog["last_updated"] if not last_updated or datetime.now() > ( last_updated + timedelta(seconds=self.ttl) ): From 766718aba2d455ce5209ee99b012cb2865673a10 Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Tue, 31 Oct 2023 11:44:04 +0100 Subject: [PATCH 4/8] rename --- CHANGES.md | 10 +++++++--- tipg/dependencies.py | 2 +- tipg/factory.py | 12 ++++++------ 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 14529ef8..0eb44508 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -81,14 +81,18 @@ Note: Minor version `0.X.0` update might break the API, It's recommended to pin print(items_list["next"]) # Next Offset print(items_list["prev"]) # Previous Offset ``` +- rename `catalog_dependency` attribute to `collections_dependency` -- move `/collections` QueryParameters in the `CatalogParams` dependency +- move the `collections_dependency` attribute from the `EndpointsFactory` to `OGCFeaturesFactory` class -- the `CatalogParams` now returns a `CollectionList` object +- move `/collections` QueryParameters in the `CollectionsParams` dependency + +- rename `CatalogParams` to `CollectionsParams` + +- the `CollectionsParams` now returns a `CollectionList` object - move `s_intersects` and `t_intersects` functions from `tipg.factory` to `tipg.dependencies` -- move the `catalog_dependency` attribute from the `EndpointsFactory` to `OGCFeaturesFactory` class ## [0.4.4] - 2023-10-03 diff --git a/tipg/dependencies.py b/tipg/dependencies.py index cd4bf280..c8d7b676 100644 --- a/tipg/dependencies.py +++ b/tipg/dependencies.py @@ -430,7 +430,7 @@ def CollectionParams( ) -def CatalogParams( +def CollectionsParams( request: Request, bbox_filter: Annotated[Optional[List[float]], Depends(bbox_query)], datetime_filter: Annotated[Optional[List[str]], Depends(datetime_query)], diff --git a/tipg/factory.py b/tipg/factory.py index 04046699..5023080e 100644 --- a/tipg/factory.py +++ b/tipg/factory.py @@ -18,8 +18,8 @@ from tipg import model from tipg.collections import Collection, CollectionList from tipg.dependencies import ( - CatalogParams, CollectionParams, + CollectionsParams, ItemsOutputType, OutputType, QueryablesOutputType, @@ -331,8 +331,8 @@ def landing( class OGCFeaturesFactory(EndpointsFactory): """OGC Features Endpoints Factory.""" - # catalog dependency - catalog_dependency: Callable[..., CollectionList] = CatalogParams + # collections dependency + collections_dependency: Callable[..., CollectionList] = CollectionsParams @property def conforms_to(self) -> List[str]: @@ -414,7 +414,7 @@ def collections( request: Request, collection_list: Annotated[ CollectionList, - Depends(self.catalog_dependency), + Depends(self.collections_dependency), ], output_type: Annotated[ Optional[MediaType], @@ -1819,7 +1819,7 @@ class Endpoints(EndpointsFactory): """OGC Features and Tiles Endpoints Factory.""" # OGC Features dependency - catalog_dependency: Callable[..., CollectionList] = CatalogParams + collections_dependency: Callable[..., CollectionList] = CollectionsParams # OGC Tiles dependency supported_tms: TileMatrixSets = default_tms @@ -1846,7 +1846,7 @@ def links(self, request: Request) -> List[model.Link]: def register_routes(self): """Register factory Routes.""" self.ogc_features = OGCFeaturesFactory( - catalog_dependency=self.catalog_dependency, + collections_dependency=self.collections_dependency, collection_dependency=self.collection_dependency, router_prefix=self.router_prefix, templates=self.templates, From f4ef32eb3085e080ee362d541becedee7a095f82 Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Tue, 31 Oct 2023 11:49:29 +0100 Subject: [PATCH 5/8] update docs --- docs/src/user_guide/factories.md | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/docs/src/user_guide/factories.md b/docs/src/user_guide/factories.md index 7c49a2a5..14d3c012 100644 --- a/docs/src/user_guide/factories.md +++ b/docs/src/user_guide/factories.md @@ -5,9 +5,11 @@ # pseudo code class Factory: + collections_dependency: Callable collection_dependency: Callable - def __init__(self, collection_dependency: Callable): + def __init__(self, collections_dependency: Callable, collection_dependency: Callable): + self.collections_dependency = collections_dependency self.collection_dependency = collection_dependency self.router = APIRouter() @@ -15,6 +17,13 @@ class Factory: def register_routes(self): + @self.router.get("/collections") + def collections( + request: Request, + collection_list=Depends(self.collections_dependency), + ): + ... + @self.router.get("/collections/{collectionId}") def collection( request: Request, @@ -27,6 +36,7 @@ class Factory: request: Request, collection=Depends(self.collection_dependency), ): + item_list = collection.items(...) ... @self.router.get("/collections/{collectionId}/items/{itemId}") @@ -35,6 +45,7 @@ class Factory: collection=Depends(self.collection_dependency), itemId: str = Path(..., description="Item identifier"), ): + item_list = collection.items(item_id=[itemId]) ... @@ -61,7 +72,7 @@ app.include_router(endpoints.router, tags=["OGC Features API"]) #### Creation Options -- **catalog_dependency** (Callable[..., tipg.collections.Catalog]): Callable which return a Catalog instance +- **collections_dependency** (Callable[..., tipg.collections.CollectionList]): Callable which return a CollectionList dictionary - **collection_dependency** (Callable[..., tipg.collections.Collection]): Callable which return a Collection instance @@ -143,7 +154,7 @@ app.include_router(endpoints.router) #### Creation Options -- **catalog_dependency** (Callable[..., tipg.collections.Catalog]): Callable which return a Catalog instance +- **collections_dependency** (Callable[..., tipg.collections.CollectionList]): Callable which return a CollectionList dictionary - **collection_dependency** (Callable[..., tipg.collections.Collection]): Callable which return a Collection instance From 6387c27afa91f992b0fce97d6cb579320cf4119d Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Tue, 31 Oct 2023 11:52:12 +0100 Subject: [PATCH 6/8] update docs --- docs/src/user_guide/factories.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/user_guide/factories.md b/docs/src/user_guide/factories.md index 14d3c012..95722f43 100644 --- a/docs/src/user_guide/factories.md +++ b/docs/src/user_guide/factories.md @@ -36,7 +36,7 @@ class Factory: request: Request, collection=Depends(self.collection_dependency), ): - item_list = collection.items(...) + item_list = collection.features(...) ... @self.router.get("/collections/{collectionId}/items/{itemId}") @@ -45,7 +45,7 @@ class Factory: collection=Depends(self.collection_dependency), itemId: str = Path(..., description="Item identifier"), ): - item_list = collection.items(item_id=[itemId]) + item_list = collection.features(ids_filter=[itemId]) ... From dfae2753f1a6c701289d67a55c02389d8a9160a4 Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Tue, 31 Oct 2023 11:56:08 +0100 Subject: [PATCH 7/8] remove useless --- tipg/factory.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tipg/factory.py b/tipg/factory.py index 5023080e..ed3e713b 100644 --- a/tipg/factory.py +++ b/tipg/factory.py @@ -719,8 +719,6 @@ async def items( # noqa: C901 Optional[MediaType], Depends(ItemsOutputType) ] = None, ): - offset = offset or 0 - output_type = output_type or MediaType.geojson geom_as_wkt = output_type not in [ MediaType.geojson, From c71507badb970af6288b36a092efbdfc95c76056 Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Tue, 31 Oct 2023 14:53:11 +0100 Subject: [PATCH 8/8] update changelog --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 0eb44508..777d6e11 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -12,7 +12,7 @@ Note: Minor version `0.X.0` update might break the API, It's recommended to pin - add `py.typed` file -- add `ItemList` and `CollectionList` *TypedDict* +- add `tipg.collections.ItemList` and `tipg.collections.CollectionList` *TypedDict* ```python class ItemList(TypedDict):