diff --git a/README.md b/README.md index 9bb4865..69fa95d 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,8 @@ Install from sources and run for development: ``` $ git clone https://github.com/developmentseed/titiler-stacapi-mspc.git $ cd titiler-stacapi-mspc +$ virtualenv -p python3 venv +$ source venv/bin/activate $ python -m pip install -e . ``` diff --git a/tests/test_stac_item_tiler_test.py b/tests/test_stac_item_tiler_test.py new file mode 100644 index 0000000..48df165 --- /dev/null +++ b/tests/test_stac_item_tiler_test.py @@ -0,0 +1,14 @@ +"""Test titiler.stacapi.stac_item_tiler endpoints.""" + +import pytest + + +@pytest.mark.skip(reason="To be implemented.") +def test_tile_cog(): + """Test tiling a COG asset.""" + pass + + +def test_tile_netcdf(): + """Test tiling a NetCDF asset.""" + pass diff --git a/tests/test_stac_reader.py b/tests/test_stac_reader.py new file mode 100644 index 0000000..13d7f53 --- /dev/null +++ b/tests/test_stac_reader.py @@ -0,0 +1,34 @@ +"""Test titiler.stacapi.stac_reader functions.""" + +import json +import os + +import pystac +import pytest +from rio_tiler.io import Reader + +from titiler.stacapi.stac_reader import STACReader + +item_file = os.path.join( + os.path.dirname(__file__), "fixtures", "20200307aC0853900w361030.json" +) +item_json = json.loads(open(item_file).read()) + + +@pytest.mark.skip(reason="To be implemented.") +def test_asset_info(): + """Test get_asset_info function""" + pass + + +def test_stac_reader_cog(): + """Test reader is rio_tiler.io.Reader""" + stac_reader = STACReader(pystac.Item.from_dict(item_json)) + asset_info = stac_reader._get_asset_info("cog") + assert stac_reader._get_asset_reader(asset_info) == Reader + + +@pytest.mark.skip(reason="To be implemented.") +def test_stac_reader_netcdf(): + """Test reader attribute is titiler.stacapi.XarrayReader""" + pass diff --git a/tests/test_xarray_reader.py b/tests/test_xarray_reader.py new file mode 100644 index 0000000..c5f7159 --- /dev/null +++ b/tests/test_xarray_reader.py @@ -0,0 +1,9 @@ +"""Test titiler.stacapi.xarray_reader functions.""" + +import pytest + + +@pytest.mark.skip(reason="To be implemented.") +def test_xarray_open_dataset(): + """Test xarray_open_dataset function""" + pass diff --git a/titiler/stacapi/main.py b/titiler/stacapi/main.py index aa5fc30..a944ae2 100644 --- a/titiler/stacapi/main.py +++ b/titiler/stacapi/main.py @@ -12,7 +12,6 @@ from typing_extensions import Annotated from titiler.core.errors import DEFAULT_STATUS_CODES, add_exception_handlers -from titiler.core.factory import AlgorithmFactory, MultiBaseTilerFactory, TMSFactory from titiler.core.middleware import CacheControlMiddleware, LoggerMiddleware from titiler.core.resources.enums import OptionalHeader from titiler.mosaic.errors import MOSAIC_STATUS_CODES @@ -22,6 +21,7 @@ from titiler.stacapi.enums import MediaType from titiler.stacapi.factory import MosaicTilerFactory from titiler.stacapi.settings import ApiSettings, STACAPISettings +from titiler.stacapi.stac_item_tiler import StacItemTiler from titiler.stacapi.stac_reader import STACReader from titiler.stacapi.utils import create_html_response @@ -101,34 +101,18 @@ ############################################################################### # STAC Item Endpoints -# Notes: The `MultiBaseTilerFactory` from titiler.core.factory expect a `URL` as query parameter -# but in this project we use a custom `path_dependency=ItemIdParams`, which define `{collection_id}` and `{item_id}` as -# `Path` dependencies. Then the `ItemIdParams` dependency will fetch the STAC API endpoint to get the STAC Item. The Item -# will then be used in our custom `STACReader`. -stac = MultiBaseTilerFactory( - reader=STACReader, +stac_item_tiler = StacItemTiler( + title=settings.name, path_dependency=ItemIdParams, - optional_headers=optional_headers, + reader=STACReader, router_prefix="/collections/{collection_id}/items/{item_id}", - add_viewer=True, ) app.include_router( - stac.router, - tags=["STAC Item"], + stac_item_tiler.router, + tags=["STAC Items"], prefix="/collections/{collection_id}/items/{item_id}", ) -############################################################################### -# Tiling Schemes Endpoints -tms = TMSFactory() -app.include_router(tms.router, tags=["Tiling Schemes"]) - -############################################################################### -# Algorithms Endpoints -algorithms = AlgorithmFactory() -app.include_router(algorithms.router, tags=["Algorithms"]) - - ############################################################################### # Health Check Endpoint @app.get("/healthz", description="Health Check", tags=["Health Check"]) diff --git a/titiler/stacapi/models.py b/titiler/stacapi/models.py index dfcd3a4..66cf3ef 100644 --- a/titiler/stacapi/models.py +++ b/titiler/stacapi/models.py @@ -6,7 +6,7 @@ """ -from typing import List, Optional +from typing import Dict, List, Optional, Sequence, Tuple, TypedDict from pydantic import BaseModel, Field from typing_extensions import Annotated @@ -82,3 +82,13 @@ class Landing(BaseModel): title: Optional[str] = None description: Optional[str] = None links: List[Link] + + +class AssetInfo(TypedDict, total=False): + """Asset Reader Options.""" + + url: str + type: str + env: Optional[Dict] + metadata: Optional[Dict] + dataset_statistics: Optional[Sequence[Tuple[float, float]]] diff --git a/titiler/stacapi/stac_item_tiler.py b/titiler/stacapi/stac_item_tiler.py new file mode 100644 index 0000000..6faf0fa --- /dev/null +++ b/titiler/stacapi/stac_item_tiler.py @@ -0,0 +1,246 @@ +"""STAC Item Tiler.""" +from dataclasses import dataclass, field +from typing import Any, Callable, List, Literal, Optional, Type, Union + +import numpy +from fastapi import APIRouter, Depends, Path, Query +from morecantile import tms as default_tms +from morecantile.defaults import TileMatrixSets +from pydantic import conint +from rio_tiler.io import BaseReader +from rio_tiler.types import RIOResampling, WarpResampling +from starlette.requests import Request +from starlette.responses import Response +from typing_extensions import Annotated + +from titiler.core import BaseTilerFactory, dependencies +from titiler.core.algorithm import algorithms as available_algorithms +from titiler.core.dependencies import DatasetPathParams +from titiler.core.factory import img_endpoint_params +from titiler.core.resources.enums import ImageType +from titiler.core.utils import render_image +from titiler.stacapi.stac_reader import STACReader + + +@dataclass +class StacItemTiler(BaseTilerFactory): + """STAC Item Tiler.""" + + # Path Dependency + path_dependency: Callable[..., Any] = DatasetPathParams + + # FastAPI router + router: APIRouter = field(default_factory=APIRouter) + + supported_tms: TileMatrixSets = default_tms + + # Router Prefix is needed to find the path for routes when prefixed + # e.g if you mount the route with `/foo` prefix, set router_prefix to foo + router_prefix: str = "" + + title: str = "TiTiler-STACAPI-MSPC" + reader: Type[BaseReader] = STACReader + + def register_routes(self): + """Post Init: register routes.""" + self.register_tiles() + + def register_tiles(self): # noqa: C901 + """Register tileset endpoints.""" + + @self.router.get( + "/tiles/{tileMatrixSetId}/{z}/{x}/{y}", + **img_endpoint_params, + tags=["Raster Tiles"], + ) + @self.router.get( + "/tiles/{tileMatrixSetId}/{z}/{x}/{y}.{format}", + **img_endpoint_params, + tags=["Raster Tiles"], + ) + @self.router.get( + "/tiles/{tileMatrixSetId}/{z}/{x}/{y}@{scale}x", + **img_endpoint_params, + tags=["Raster Tiles"], + ) + @self.router.get( + "/tiles/{tileMatrixSetId}/{z}/{x}/{y}@{scale}x.{format}", + **img_endpoint_params, + tags=["Raster Tiles"], + ) + def tiles_endpoint( + request: Request, + tileMatrixSetId: Annotated[ + Literal[tuple(self.supported_tms.list())], + Path(description="Identifier for a supported TileMatrixSet"), + ], + z: Annotated[ + int, + Path( + description="Identifier (Z) selecting one of the scales defined in the TileMatrixSet and representing the scaleDenominator the tile.", + ), + ], + x: Annotated[ + int, + Path( + description="Column (X) index of the tile on the selected TileMatrix. It cannot exceed the MatrixHeight-1 for the selected TileMatrix.", + ), + ], + y: Annotated[ + int, + Path( + description="Row (Y) index of the tile on the selected TileMatrix. It cannot exceed the MatrixWidth-1 for the selected TileMatrix.", + ), + ], + scale: Annotated[ # type: ignore + conint(gt=0, le=4), "Tile size scale. 1=256x256, 2=512x512..." + ] = 1, + format: Annotated[ + ImageType, + "Default will be automatically defined if the output image needs a mask (png) or not (jpeg).", + ] = None, + asset: Annotated[ + str, + "Asset name to read from (STAC Item asset name)", + ] = "", + ################################################################### + # XarrayReader Options + ################################################################### + variable: Annotated[ + Optional[str], + Query(description="Xarray Variable"), + ] = None, + drop_dim: Annotated[ + Optional[str], + Query(description="Dimension to drop"), + ] = None, + time_slice: Annotated[ + Optional[str], Query(description="Slice of time to read (if available)") + ] = None, + decode_times: Annotated[ + Optional[bool], + Query( + title="decode_times", + description="Whether to decode times", + ), + ] = None, + ################################################################### + # Rasterio Reader Options + ################################################################### + indexes: Annotated[ + Optional[List[int]], + Query( + title="Band indexes", + alias="bidx", + description="Dataset band indexes", + ), + ] = None, + expression: Annotated[ + Optional[str], + Query( + title="Band Math expression", + description="rio-tiler's band math expression", + ), + ] = None, + bands: Annotated[ + Optional[List[str]], + Query( + title="Band names", + description="Band names.", + ), + ] = None, + bands_regex: Annotated[ + Optional[str], + Query( + title="Regex expression to parse dataset links", + description="Regex expression to parse dataset links.", + ), + ] = None, + unscale: Annotated[ + Optional[bool], + Query( + title="Apply internal Scale/Offset", + description="Apply internal Scale/Offset. Defaults to `False`.", + ), + ] = None, + resampling_method: Annotated[ + Optional[RIOResampling], + Query( + alias="resampling", + description="RasterIO resampling algorithm. Defaults to `nearest`.", + ), + ] = None, + ################################################################### + # Reader options + ################################################################### + nodata: Annotated[ + Optional[Union[str, int, float]], + Query( + title="Nodata value", + description="Overwrite internal Nodata value", + ), + ] = None, + reproject_method: Annotated[ + Optional[WarpResampling], + Query( + alias="reproject", + description="WarpKernel resampling algorithm (only used when doing re-projection). Defaults to `nearest`.", + ), + ] = None, + ################################################################### + # Rendering Options + ################################################################### + post_process=Depends(available_algorithms.dependency), + rescale=Depends(dependencies.RescalingParams), + color_formula=Depends(dependencies.ColorFormulaParams), + colormap=Depends(dependencies.ColorMapParams), + render_params=Depends(dependencies.ImageRenderingParams), + item=Depends(self.path_dependency), + ) -> Response: + """Create map tile from a dataset.""" + resampling_method = resampling_method or "nearest" + reproject_method = reproject_method or "nearest" + if nodata is not None: + nodata = numpy.nan if nodata == "nan" else float(nodata) + + asset_url = item.assets[asset].href + stac_item_reader = self.reader(input=asset_url, item=item) + asset_info = stac_item_reader._get_asset_info(asset) + + # Add me - function for parsing reader options specific to the asset type + reader_options = { + "src_path": asset_url, + "variable": variable, + } + + with stac_item_reader._get_asset_reader(asset_info)( + **reader_options + ) as src_dst: + image, _ = src_dst.tile( + x, + y, + z, + tilesize=scale * 256, + nodata=nodata, + reproject_method=reproject_method, + assets=[asset] + # **read_options, + ) + + if post_process: + image = post_process(image) + + if rescale: + image.rescale(rescale) + + if color_formula: + image.apply_color_formula(color_formula) + + content, media_type = render_image( + image, + output_format=format, + colormap=colormap, + **render_params, + ) + + return Response(content, media_type=media_type) diff --git a/titiler/stacapi/stac_reader.py b/titiler/stacapi/stac_reader.py index 10ef9d7..ab64bd4 100644 --- a/titiler/stacapi/stac_reader.py +++ b/titiler/stacapi/stac_reader.py @@ -10,22 +10,41 @@ from rio_tiler.constants import WEB_MERCATOR_TMS, WGS84_CRS from rio_tiler.errors import InvalidAssetName from rio_tiler.io import BaseReader, Reader, stac -from rio_tiler.types import AssetInfo +# Also replace with rio_tiler import when https://github.com/cogeotiff/rio-tiler/pull/711 is merged and released. +from titiler.stacapi.models import AssetInfo from titiler.stacapi.settings import STACSettings +from titiler.stacapi.xarray_reader import XarrayReader stac_config = STACSettings() +valid_types = { + "image/tiff; application=geotiff", + "image/tiff; application=geotiff; profile=cloud-optimized", + "image/tiff; profile=cloud-optimized; application=geotiff", + "image/vnd.stac.geotiff; cloud-optimized=true", + "image/tiff", + "image/x.geotiff", + "image/jp2", + "application/x-hdf5", + "application/x-hdf", + "application/vnd+zarr", + "application/x-netcdf", + "application/netcdf", +} + + @attr.s class STACReader(stac.STACReader): """Custom STAC Reader. - Only accept `pystac.Item` as input (while rio_tiler.io.STACReader accepts url or pystac.Item) + Use STAC item URL to fetch and set the item and determine which backend reader (xarray or rasterio) to use. """ - input: pystac.Item = attr.ib() + input: str = attr.ib() + item: pystac.Item = attr.ib() tms: TileMatrixSet = attr.ib(default=WEB_MERCATOR_TMS) minzoom: int = attr.ib() @@ -36,8 +55,8 @@ class STACReader(stac.STACReader): include_assets: Optional[Set[str]] = attr.ib(default=None) exclude_assets: Optional[Set[str]] = attr.ib(default=None) - include_asset_types: Set[str] = attr.ib(default=stac.DEFAULT_VALID_TYPE) exclude_asset_types: Optional[Set[str]] = attr.ib(default=None) + include_asset_types: Set[str] = attr.ib(default=valid_types) reader: Type[BaseReader] = attr.ib(default=Reader) reader_options: Dict = attr.ib(factory=dict) @@ -46,12 +65,20 @@ class STACReader(stac.STACReader): ctx: Any = attr.ib(default=rasterio.Env) - item: pystac.Item = attr.ib(init=False) + def _get_asset_reader(self, asset_info: AssetInfo) -> Type[BaseReader]: + """Get Asset Reader.""" + asset_type = asset_info.get("type", None) + + if asset_type and asset_type in [ + "application/x-hdf5", + "application/x-hdf", + "application/vnd.zarr", + "application/x-netcdf", + "application/netcdf", + ]: + return XarrayReader - def __attrs_post_init__(self): - """Fetch STAC Item and get list of valid assets.""" - self.item = self.input - super().__attrs_post_init__() + return Reader @minzoom.default def _minzoom(self): @@ -83,10 +110,7 @@ def _get_asset_info(self, asset: str) -> AssetInfo: if alternate := stac_config.alternate_url: url = asset_info.to_dict()["alternate"][alternate]["href"] - info = AssetInfo( - url=url, - metadata=extras, - ) + info = AssetInfo(url=url, metadata=extras, type=asset_info.to_dict()["type"]) if head := extras.get("file:header_size"): info["env"] = {"GDAL_INGESTED_BYTES_AT_OPEN": head} diff --git a/titiler/stacapi/xarray_reader.py b/titiler/stacapi/xarray_reader.py new file mode 100644 index 0000000..a046ab7 --- /dev/null +++ b/titiler/stacapi/xarray_reader.py @@ -0,0 +1,22 @@ +"""Custom XarrayReader for opening data arrays and transforming coordinates.""" + +import attr +import fsspec +import rio_tiler.io.xarray as rio_tiler_xarray +import xarray as xr + + +# Probably want to reimplement https://github.com/developmentseed/titiler-cmr/blob/develop/titiler/cmr/reader.py +@attr.s +class XarrayReader(rio_tiler_xarray.XarrayReader): + """Custom XarrayReader for opening data arrays and transforming coordinates.""" + + input: xr.Dataset = attr.ib(init=False, default=None) + src_path: str = attr.ib(default=None) + variable: str = attr.ib(default=None) + + def __attrs_post_init__(self): + """Post Init.""" + httpfs = fsspec.filesystem("http") + self.input = xr.open_dataset(httpfs.open(self.src_path))[self.variable] + return super().__attrs_post_init__()