Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

use factory from titiler.xarray #72

Open
wants to merge 8 commits into
base: dev
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 2 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,10 @@ source .venv/bin/activate

python -m pip install -e . uvicorn
export TEST_ENVIRONMENT=true # set this when running locally to mock redis
uvicorn titiler.xarray.main:app --reload
uvicorn titiler.xarray_api.main:app --reload
```

To access the docs, visit http://127.0.0.1:8000/api.html.
To access the docs, visit <http://127.0.0.1:8000/api.html>.
![](https://github.com/developmentseed/titiler-xarray/assets/10407788/4368546b-5b60-4cd5-86be-fdd959374b17)

## Testing
Expand Down Expand Up @@ -46,7 +46,6 @@ The Github Actions workflow defined in [.github/workflows/ci.yml](./.github/work
* The production stack is deployed when the `main` branch is tagged, creating a new release. The production stack will deploy to a stack with an API Gateway associated with the domain prod-titiler-xarray.delta-backend.com/.
* The development stack will be deployed upon pushes to the `dev` and `main` branches. The development stack will deploy to a stack with an API Gateway associated with the domain dev-titiler-xarray.delta-backend.com/.


## New Deployments

The following steps detail how to to setup and deploy the CDK stack from your local machine.
Expand Down Expand Up @@ -94,7 +93,6 @@ The following steps detail how to to setup and deploy the CDK stack from your lo
AWS_DEFAULT_REGION=us-west-2 AWS_REGION=us-west-2 AWS_PROFILE=smce-veda STACK_STAGE=production npm --prefix infrastructure/aws run cdk -- deploy titiler-xarray-production
```


**Important**

In AWS Lambda environment we need to have specific version of botocore, S3FS, FSPEC and other libraries.
Expand Down
26 changes: 16 additions & 10 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[project]
name = "titiler.xarray"
description = "TiTiler extension for xarray."
name = "titiler-xarray-api"
description = "TiTiler application extension for titiler.xarray."
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
description = "TiTiler application extension for titiler.xarray."
description = "TiTiler application using titiler.xarray."

?

readme = "README.md"
requires-python = ">=3.8"
authors = [
Expand All @@ -25,6 +25,8 @@ classifiers = [
]
dynamic = ["version"]
dependencies = [
"titiler.core>=0.19.0,<0.20",
"titiler.xarray>=0.19.0,<0.20",
"cftime",
"h5netcdf",
"numpy<2.0.0",
Expand All @@ -36,13 +38,12 @@ dependencies = [
"s3fs",
"aiohttp",
"requests",
"pydantic==2.0.2",
"titiler.core>=0.14.1,<0.15",
"pydantic-settings~=2.0",
"pydantic>=2.4,<3.0",
"pandas==1.5.3",
"redis",
"fastapi>=0.100.0,<0.107.0",
"starlette<0.28",
"fastapi>=0.108.0,<0.109.0",
"starlette>=0.29.0,<0.33.0",
]

[project.optional-dependencies]
Expand Down Expand Up @@ -111,13 +112,18 @@ namespace_packages = true
explicit_package_bases = true

[build-system]
requires = ["pdm-pep517"]
build-backend = "pdm.pep517.api"
requires = ["pdm-backend"]
build-backend = "pdm.backend"


[tool.pdm.version]
source = "file"
path = "titiler/xarray/__init__.py"
path = "src/titiler/xarray_api/__init__.py"


[tool.pdm]
package-dir = "src/"

[tool.pdm.build]
includes = ["titiler/xarray"]
includes = ["src/titiler/xarray_api/"]
excludes = ["tests/", "**/.mypy_cache", "**/.DS_Store"]
3 changes: 3 additions & 0 deletions src/titiler/xarray_api/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
"""titiler.xarray_api"""

__version__ = "0.3.0"
196 changes: 196 additions & 0 deletions src/titiler/xarray_api/factory.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,196 @@
"""TiTiler.xarray factory."""

from typing import List, Literal, Optional, Type, Union
from urllib.parse import urlencode

import jinja2
import numpy as np
from attrs import define
from fastapi import Depends, Query
from starlette.requests import Request
from starlette.responses import HTMLResponse
from starlette.templating import Jinja2Templates
from typing_extensions import Annotated

from titiler.core.dependencies import ColorFormulaParams, DefaultDependency
from titiler.core.resources.enums import ImageType
from titiler.core.resources.responses import JSONResponse
from titiler.xarray.dependencies import XarrayIOParams, XarrayParams
from titiler.xarray.factory import TilerFactory as BaseTilerFactory
from titiler.xarray_api.reader import XarrayReader


def nodata_dependency(
nodata: Annotated[
Optional[Union[str, int, float]],
Query(
title="Nodata value",
description="Overwrite internal Nodata value",
),
] = None,
) -> Optional[float]:
"""Nodata dependency."""
if nodata is not None:
nodata = np.nan if nodata == "nan" else float(nodata)

return None
hrodmn marked this conversation as resolved.
Show resolved Hide resolved


@define(kw_only=True)
class XarrayTilerFactory(BaseTilerFactory):
"""Xarray Tiler Factory."""

reader: Type[XarrayReader] = XarrayReader
reader_dependency: Type[DefaultDependency] = XarrayParams

def register_routes(self) -> None: # noqa: C901
"""Register Info / Tiles / TileJSON endoints."""
super().register_routes()
self.variables()

def variables(self) -> None:
"""Register /variables endpoint"""

@self.router.get(
"/variables",
response_class=JSONResponse,
responses={200: {"description": "Return dataset's Variables."}},
)
def get_variables(
src_path=Depends(self.path_dependency),
io_params=Depends(XarrayIOParams),
) -> List[str]:
"""return available variables."""
return XarrayReader.list_variables(
hrodmn marked this conversation as resolved.
Show resolved Hide resolved
src_path=src_path,
group=io_params.group,
decode_times=io_params.decode_times,
)

def statistics(self) -> None:
"""Register /statistics and /histogram endpoints"""
super().statistics()

@self.router.get(
"/histogram",
response_class=JSONResponse,
responses={200: {"description": "Return histogram for this data variable"}},
response_model_exclude_none=True,
)
def histogram(
src_path=Depends(self.path_dependency),
reader_params=Depends(self.reader_dependency),
):
with self.reader(
src_path=src_path,
variable=reader_params.variable,
group=reader_params.group,
decode_times=reader_params.decode_times,
datetime=reader_params.datetime,
) as src_dst:
boolean_mask = ~np.isnan(src_dst.input)
data_values = src_dst.input.values[boolean_mask]
counts, values = np.histogram(data_values, bins=10)
counts, values = counts.tolist(), values.tolist()
buckets = list(
zip(values, [values[i + 1] for i in range(len(values) - 1)])
)
hist_dict = []
for idx, bucket in enumerate(buckets):
hist_dict.append({"bucket": bucket, "value": counts[idx]})
return hist_dict

def map_viewer(self) -> None:
"""Register /map endpoints"""

@self.router.get("/{tileMatrixSetId}/map", response_class=HTMLResponse)
def map_viewer(
request: Request,
tileMatrixSetId: Annotated[ # type: ignore
Literal[tuple(self.supported_tms.list())],
"Identifier selecting one of the supported TileMatrixSetIds",
],
url: Annotated[Optional[str], Query(description="Dataset URL")] = None,
variable: Annotated[
Optional[str],
Query(description="Xarray Variable"),
] = None,
group: Annotated[
Optional[int],
Query(
description="Select a specific zarr group from a zarr hierarchy, can be for pyramids or datasets. Can be used to open a dataset in HDF5 files."
),
] = None,
decode_times: Annotated[
bool,
Query(
title="decode_times",
description="Whether to decode times",
),
] = True,
drop_dim: Annotated[
Optional[str],
Query(description="Dimension to drop"),
] = None,
datetime: Annotated[
Optional[str], Query(description="Slice of time to read (if available)")
] = None,
Copy link
Member

@vincentsarago vincentsarago Nov 26, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes but I need the url parameter to be Optional for this endpoint. is there a way to override the type of a particular parameter in a dependency?

when no url is provided the user gets the interactive form instead of the actual map

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is there a way to override the type of a particular parameter in a dependency?

Yes but not if it's only for a specific endpoint, you'll need to hardcode things directly at the endpoint level

I wasn't talking about the url parameter but maybe a better way of doing this would be to have separate endpoints for both webpage!

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah it might make more sense to have two endpoints like /map and /map-builder instead of this conditional logic.

tile_format: Annotated[
Optional[ImageType],
Query(
description="Default will be automatically defined if the output image needs a mask (png) or not (jpeg).",
),
] = None,
tile_scale: Annotated[
int,
Query(
gt=0, lt=4, description="Tile size scale. 1=256x256, 2=512x512..."
),
] = 1,
minzoom: Annotated[
Optional[int],
Query(description="Overwrite default minzoom."),
] = None,
maxzoom: Annotated[
Optional[int],
Query(description="Overwrite default maxzoom."),
] = None,
post_process=Depends(self.process_dependency),
rescale=Depends(self.rescale_dependency),
color_formula=Depends(ColorFormulaParams),
colormap=Depends(self.colormap_dependency),
render_params=Depends(self.render_dependency),
nodata=Depends(nodata_dependency),
hrodmn marked this conversation as resolved.
Show resolved Hide resolved
):
"""Return map Viewer."""
jinja2_env = jinja2.Environment(
loader=jinja2.ChoiceLoader([jinja2.PackageLoader(__package__, ".")])
)
templates = Jinja2Templates(env=jinja2_env)

if url:
tilejson_url = self.url_for(
request, "tilejson", tileMatrixSetId=tileMatrixSetId
)
if request.query_params._list:
tilejson_url += f"?{urlencode(request.query_params._list)}"

tms = self.supported_tms.get(tileMatrixSetId)
return templates.TemplateResponse(
name="map.html",
context={
"request": request,
"tilejson_endpoint": tilejson_url,
"tms": tms,
"resolutions": [matrix.cellSize for matrix in tms],
},
media_type="text/html",
)
else:
return templates.TemplateResponse(
name="map-form.html",
context={
"request": request,
},
media_type="text/html",
)
26 changes: 17 additions & 9 deletions titiler/xarray/main.py → src/titiler/xarray_api/main.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
"""titiler app."""
"""titiler.xarray_api."""

import logging

Expand All @@ -8,19 +8,19 @@
from starlette import status
from starlette.middleware.cors import CORSMiddleware

import titiler.xarray.reader as reader
import titiler.xarray_api.reader as reader
from titiler.core.errors import DEFAULT_STATUS_CODES, add_exception_handlers
from titiler.core.factory import AlgorithmFactory, TMSFactory
from titiler.core.factory import AlgorithmFactory, ColorMapFactory, TMSFactory
from titiler.core.middleware import (
CacheControlMiddleware,
LoggerMiddleware,
TotalTimeMiddleware,
)
from titiler.xarray import __version__ as titiler_version
from titiler.xarray.factory import ZarrTilerFactory
from titiler.xarray.middleware import ServerTimingMiddleware
from titiler.xarray.redis_pool import get_redis
from titiler.xarray.settings import ApiSettings
from titiler.xarray_api import __version__ as titiler_version
from titiler.xarray_api.factory import XarrayTilerFactory
from titiler.xarray_api.middleware import ServerTimingMiddleware
from titiler.xarray_api.redis_pool import get_redis
from titiler.xarray_api.settings import ApiSettings

logging.getLogger("botocore.credentials").disabled = True
logging.getLogger("botocore.utils").disabled = True
Expand All @@ -38,7 +38,7 @@

###############################################################################
# Tiles endpoints
xarray_factory = ZarrTilerFactory()
xarray_factory = XarrayTilerFactory()
app.include_router(xarray_factory.router, tags=["Xarray Tiler API"])

###############################################################################
Expand All @@ -51,6 +51,14 @@
algorithms = AlgorithmFactory()
app.include_router(algorithms.router, tags=["Algorithms"])

###############################################################################
# Colormaps endpoints
cmaps = ColorMapFactory()
app.include_router(
cmaps.router,
tags=["ColorMaps"],
)

error_codes = {
zarr.errors.GroupNotFoundError: status.HTTP_422_UNPROCESSABLE_ENTITY,
}
Expand Down
Loading
Loading