Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add filter extension #165

Merged
merged 23 commits into from
Aug 18, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
324c115
first pass at adding filter extension
Jun 29, 2021
3b21ee7
Fixing implementation. Added conformance classes to extensions so tha…
Jun 29, 2021
f38360e
implemented queryables endpoints in collections and landing page. Als…
Jun 30, 2021
399fc02
Merge branch 'master' of https://github.com/stac-utils/stac-fastapi i…
Jun 30, 2021
9b8478d
removing unused import and fixing pre-commit errors
Jun 30, 2021
3af8d20
fixing pre-commit errors
Jun 30, 2021
1645307
Update filter.py
Jul 1, 2021
247098f
Merge branch 'master' into add-filter-extension
Jul 2, 2021
212781f
updated base conformance classes
Jul 6, 2021
1f31da2
Merge branch 'add-filter-extension' of https://github.com/rsmith013/s…
Jul 6, 2021
a3324d3
Merge branch 'master' into add-filter-extension
Jul 6, 2021
6f1a995
Merge branch 'master' into add-filter-extension
Jul 12, 2021
3989ce0
merging master
Jul 26, 2021
eec6e90
refactoring structure of filter extension to make use of async framew…
Jul 26, 2021
def82b1
added async BaseFiltersClient
Jul 27, 2021
4f1bdb6
Merge branch 'master' of https://github.com/stac-utils/stac-fastapi i…
Jul 27, 2021
0095fd9
Merge branch 'master' into add-filter-extension
lossyrob Aug 11, 2021
e2d49c5
Merge remote-tracking branch 'origin/master' into add-filter-extension
lossyrob Aug 18, 2021
cbcf04a
Merge branch 'master' into add-filter-extension
lossyrob Aug 18, 2021
6dea64c
Merge remote-tracking branch 'origin/master' into add-filter-extension
lossyrob Aug 18, 2021
8fe1283
Merge remote-tracking branch 'rsmith/add-filter-extension' into add-f…
lossyrob Aug 18, 2021
025b31f
client is union of async and base client; add default value
lossyrob Aug 18, 2021
ddc7cb1
Uncomment async endpoint logic
lossyrob Aug 18, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions stac_fastapi/api/stac_fastapi/api/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ class ApiExtensions(enum.Enum):

context = "context"
fields = "fields"
filter = "filter"
query = "query"
sort = "sort"
transaction = "transaction"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,15 @@

from .context import ContextExtension
from .fields import FieldsExtension
from .filter import FilterExtension
from .query import QueryExtension
from .sort import SortExtension
from .transaction import TransactionExtension

__all__ = (
"ContextExtension",
"FieldsExtension",
"FilterExtension",
"QueryExtension",
"SortExtension",
"TilesExtension",
Expand Down
25 changes: 14 additions & 11 deletions stac_fastapi/extensions/stac_fastapi/extensions/core/fields.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ class FieldsExtension(ApiExtension):

Attributes:
default_includes (set): defines the default set of included fields.
conformance_classes (list): Defines the list of conformance classes for the extension

"""

Expand All @@ -28,17 +29,19 @@ class FieldsExtension(ApiExtension):
)
schema_href: Optional[str] = attr.ib(default=None)
default_includes: Set[str] = attr.ib(
default=attr.Factory(
lambda: {
"id",
"type",
"geometry",
"bbox",
"links",
"assets",
"properties.datetime",
}
)
factory=lambda: {
"id",
"type",
"geometry",
"bbox",
"links",
"assets",
"properties.datetime",
}
)

conformance_classes: List[str] = attr.ib(
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These should be class variables, as they won't change between instances of the class.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

While that is true in most cases, it is not in all. The filter extension is a good example. As the filter extension is complex, there are varying degrees of implementation. There is a base level to be compliant but then there are other conformance classes that are optional and implementation-specific. Having these as instance variables allows the implementor to specify which classes their implementation conforms to.

https://github.com/radiantearth/stac-api-spec/tree/master/fragments/filter#conformance-classes

default=["https://api.stacspec.org/v1.0.0-beta.2/item-search#fields"]
)

def register(self, app: FastAPI) -> None:
Expand Down
85 changes: 85 additions & 0 deletions stac_fastapi/extensions/stac_fastapi/extensions/core/filter.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
# encoding: utf-8
"""Filter Extension."""

from typing import Callable, List, Type, Union

import attr
from fastapi import APIRouter, FastAPI
from starlette.responses import JSONResponse, Response

from stac_fastapi.api.models import APIRequest, CollectionUri, EmptyRequest
from stac_fastapi.api.routes import create_async_endpoint, create_sync_endpoint
from stac_fastapi.types.core import AsyncBaseFiltersClient, BaseFiltersClient
from stac_fastapi.types.extension import ApiExtension


@attr.s
class FilterExtension(ApiExtension):
"""Filter Extension.

The filter extension adds several endpoints which allow the retrieval of queryables and
provides an expressive mechanism for searching based on Item Attributes:
GET /queryables
GET /collections/{collectionId}/queryables

https://github.com/radiantearth/stac-api-spec/blob/master/fragments/filter/README.md

Attributes:
client: Queryables endpoint logic
conformance_classes: Conformance classes provided by the extension

"""

client: Union[AsyncBaseFiltersClient, BaseFiltersClient] = attr.ib(
factory=BaseFiltersClient
)
conformance_classes: List[str] = attr.ib(
default=[
"https://api.stacspec.org/v1.0.0-beta.2/item-search#filter",
"https://api.stacspec.org/v1.0.0-beta.2/item-search#filter:simple-cql",
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

FYI, these are about to change. radiantearth/stac-api-spec#163

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Have changed the conformance classes to:

        default=[
            "https://api.stacspec.org/v1.0.0-beta.2/item-search#filter:basic-cql",
            "https://api.stacspec.org/v1.0.0-beta.2/item-search#filter:item-search-filter",
            "https://api.stacspec.org/v1.0.0-beta.2/item-search#filter:cql-text",
        ]

this was my interpretation of the PR where the basic spatial and temporal operators were recommended. I added cql-text as default, as item-search must implement GET search. It would follow that the base implementation of the filters extension would also be a GET operation using cql-text

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry, I was actually just trying to let you know that they were going to change for beta.3. For beta.2, it should still use simple-cql

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

oh, ok. will revert.

"https://api.stacspec.org/v1.0.0-beta.2/item-search#filter:item-search-filter",
]
)
router: APIRouter = attr.ib(factory=APIRouter)
response_class: Type[Response] = attr.ib(default=JSONResponse)

def _create_endpoint(
self,
func: Callable,
request_type: Union[
Type[APIRequest],
],
) -> Callable:
"""Create a FastAPI endpoint."""
if isinstance(self.client, AsyncBaseFiltersClient):
return create_async_endpoint(
func, request_type, response_class=self.response_class
)
if isinstance(self.client, BaseFiltersClient):
return create_sync_endpoint(
func, request_type, response_class=self.response_class
)
raise NotImplementedError

def register(self, app: FastAPI) -> None:
"""Register the extension with a FastAPI application.

Args:
app: target FastAPI application.

Returns:
None
"""
self.router.add_api_route(
name="Queryables",
path="/queryables",
methods=["GET"],
endpoint=self._create_endpoint(self.client.get_queryables, EmptyRequest),
)
self.router.add_api_route(
name="Collection Queryables",
path="/collections/{collectionId}/queryables",
methods=["GET"],
endpoint=self._create_endpoint(self.client.get_queryables, CollectionUri),
)
app.include_router(self.router, tags=["Filter Extension"])
15 changes: 8 additions & 7 deletions stac_fastapi/sqlalchemy/stac_fastapi/sqlalchemy/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
from stac_pydantic.links import Relations
from stac_pydantic.version import STAC_VERSION

from stac_fastapi.extensions.core import ContextExtension, FieldsExtension
from stac_fastapi.sqlalchemy import serializers
from stac_fastapi.sqlalchemy.models import database
from stac_fastapi.sqlalchemy.session import Session
Expand Down Expand Up @@ -90,7 +89,7 @@ def item_collection(
.order_by(self.item_table.datetime.desc(), self.item_table.id)
)
count = None
if self.extension_is_enabled(ContextExtension):
if self.extension_is_enabled("ContextExtension"):
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What's the reasoning behind this change?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This change links somewhat to the thinking which spawned #178. It would be really good to try and un-bake the extensions from the core client. The reasoning behind this change is to remove the need to import the Extension classes to check if the extension is active. The client instance has the Extension instances passed to it at runtime so I changed the is_extension_enabled method to enable simple name checking https://github.com/rsmith013/stac-fastapi/blob/247098fbb28345096080ea161158ce4461868abb/stac_fastapi/types/stac_fastapi/types/core.py#L127

count_query = collection_children.statement.with_only_columns(
[func.count()]
).order_by(None)
Expand Down Expand Up @@ -136,7 +135,7 @@ def item_collection(
)

context_obj = None
if self.extension_is_enabled(ContextExtension):
if self.extension_is_enabled("ContextExtension"):
context_obj = {
"returned": len(page),
"limit": limit,
Expand Down Expand Up @@ -279,7 +278,7 @@ def post_search(
)
items = query.filter(id_filter).order_by(self.item_table.id)
page = get_page(items, per_page=search_request.limit, page=token)
if self.extension_is_enabled(ContextExtension):
if self.extension_is_enabled("ContextExtension"):
count = len(search_request.ids)
page.next = (
self.insert_token(keyset=page.paging.bookmark_next)
Expand Down Expand Up @@ -336,7 +335,7 @@ def post_search(
for (op, value) in expr.items():
query = query.filter(op.operator(field, value))

if self.extension_is_enabled(ContextExtension):
if self.extension_is_enabled("ContextExtension"):
count_query = query.statement.with_only_columns(
[func.count()]
).order_by(None)
Expand Down Expand Up @@ -379,13 +378,15 @@ def post_search(
)

response_features = []
filter_kwargs = {}

for item in page:
response_features.append(
self.item_serializer.db_to_stac(item, base_url=base_url)
)

# Use pydantic includes/excludes syntax to implement fields extension
if self.extension_is_enabled(FieldsExtension):
if self.extension_is_enabled("FieldsExtension"):
if search_request.query is not None:
query_include: Set[str] = set(
[
Expand All @@ -409,7 +410,7 @@ def post_search(
]

context_obj = None
if self.extension_is_enabled(ContextExtension):
if self.extension_is_enabled("ContextExtension"):
context_obj = {
"returned": len(page),
"limit": search_request.limit,
Expand Down
80 changes: 75 additions & 5 deletions stac_fastapi/types/stac_fastapi/types/core.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
"""Base clients."""
import abc
from datetime import datetime
from typing import Any, Dict, List, Optional, Type, Union
from typing import Any, Dict, List, Optional, Union
from urllib.parse import urljoin

import attr
Expand Down Expand Up @@ -315,9 +315,25 @@ def conformance_classes(self) -> List[str]:

return list(set(base_conformance_classes))

def extension_is_enabled(self, extension: Type[ApiExtension]) -> bool:
def extension_is_enabled(self, extension: str) -> bool:
"""Check if an api extension is enabled."""
return any([isinstance(ext, extension) for ext in self.extensions])
return any([type(ext).__name__ == extension for ext in self.extensions])

def list_conformance_classes(self):
"""Return a list of conformance classes, including implemented extensions."""
base_conformance = [
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I believe this is the set of conformance classes we want to use #159

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes, these are the old conformance classes here. #159 has the most up to date ones.

Copy link
Author

@rsmith013 rsmith013 Jul 6, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

presumably we are on beta2 now.

e.g. https://api.stacspec.org/v1.0.0-beta.2/core

Copy link
Collaborator

@philvarner philvarner Jul 6, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Correct, beta.2

"https://api.stacspec.org/v1.0.0-beta.2/core",
"https://api.stacspec.org/v1.0.0-beta.2/item-search",
"http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/core",
"http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/oas30",
"http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/geojson",
]

for extension in self.extensions:
extension_classes = getattr(extension, "conformance_classes", [])
base_conformance.extend(extension_classes)

return base_conformance

def landing_page(self, **kwargs) -> stac_types.LandingPage:
"""Landing page.
Expand Down Expand Up @@ -501,9 +517,9 @@ def conformance_classes(self) -> List[str]:

return list(set(conformance_classes))

def extension_is_enabled(self, extension: Type[ApiExtension]) -> bool:
def extension_is_enabled(self, extension: str) -> bool:
"""Check if an api extension is enabled."""
return any([isinstance(ext, extension) for ext in self.extensions])
return any([type(ext).__name__ == extension for ext in self.extensions])

async def landing_page(self, **kwargs) -> stac_types.LandingPage:
"""Landing page.
Expand Down Expand Up @@ -656,3 +672,57 @@ async def item_collection(
An ItemCollection.
"""
...


@attr.s
class AsyncBaseFiltersClient(abc.ABC):
"""Defines a pattern for implementing the STAC filter extension."""

async def get_queryables(
self, collection_id: Optional[str] = None, **kwargs
) -> Dict[str, Any]:
"""Get the queryables available for the given collection_id.

If collection_id is None, returns the intersection of all
queryables over all collections.

This base implementation returns a blank queryable schema. This is not allowed
under OGC CQL but it is allowed by the STAC API Filter Extension

https://github.com/radiantearth/stac-api-spec/tree/master/fragments/filter#queryables
"""
return {
"$schema": "https://json-schema.org/draft/2019-09/schema",
"$id": "https://example.org/queryables",
"type": "object",
"title": "Queryables for Example STAC API",
"description": "Queryable names for the example STAC API Item Search filter.",
"properties": {},
}


@attr.s
class BaseFiltersClient(abc.ABC):
"""Defines a pattern for implementing the STAC filter extension."""

def get_queryables(
self, collection_id: Optional[str] = None, **kwargs
) -> Dict[str, Any]:
"""Get the queryables available for the given collection_id.

If collection_id is None, returns the intersection of all
queryables over all collections.

This base implementation returns a blank queryable schema. This is not allowed
under OGC CQL but it is allowed by the STAC API Filter Extension

https://github.com/radiantearth/stac-api-spec/tree/master/fragments/filter#queryables
"""
return {
"$schema": "https://json-schema.org/draft/2019-09/schema",
"$id": "https://example.org/queryables",
"type": "object",
"title": "Queryables for Example STAC API",
"description": "Queryable names for the example STAC API Item Search filter.",
"properties": {},
}