diff --git a/CHANGES.md b/CHANGES.md index 90b08ae5..50c852c4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,11 @@ # unreleased +# 6.8.0 (2024-10-23) + +* Enable **Alternate** asset's HREF for STAC by using `RIO_TILER_STAC_ALTERNATE_KEY` environment variable [Backported from `7.0`] + +* Adding support for GDAL VRT Connection string for STAC Assets [Backported from `7.0`] + # 6.7.0 (2024-09-05) * raise `MissingCRS` or `InvalidGeographicBounds` errors when Xarray datasets have wrong geographic metadata diff --git a/rio_tiler/io/stac.py b/rio_tiler/io/stac.py index e5ac2a98..a3a7bc5a 100644 --- a/rio_tiler/io/stac.py +++ b/rio_tiler/io/stac.py @@ -3,7 +3,7 @@ import json import os import warnings -from typing import Any, Dict, Iterator, Optional, Set, Type, Union +from typing import Any, Dict, Iterator, Optional, Set, Tuple, Type, Union from urllib.parse import urlparse import attr @@ -41,6 +41,8 @@ "application/x-hdf", } +STAC_ALTERNATE_KEY = os.environ.get("RIO_TILER_STAC_ALTERNATE_KEY", None) + def aws_get_object( bucket: str, @@ -275,16 +277,34 @@ def _minzoom(self): def _maxzoom(self): return self.tms.maxzoom + def _parse_vrt_asset(self, asset: str) -> Tuple[str, Optional[str]]: + if asset.startswith("vrt://") and asset not in self.assets: + parsed = urlparse(asset) + if not parsed.netloc: + raise InvalidAssetName( + f"'{asset}' is not valid, couldn't find valid asset" + ) + + if parsed.netloc not in self.assets: + raise InvalidAssetName( + f"'{parsed.netloc}' is not valid, should be one of {self.assets}" + ) + + return parsed.netloc, parsed.query + + return asset, None + def _get_asset_info(self, asset: str) -> AssetInfo: - """Validate asset names and return asset's url. + """Validate asset names and return asset's info. Args: asset (str): STAC asset name. Returns: - str: STAC asset href. + AssetInfo: STAC asset info. """ + asset, vrt_options = self._parse_vrt_asset(asset) if asset not in self.assets: raise InvalidAssetName( f"'{asset}' is not valid, should be one of {self.assets}" @@ -295,13 +315,19 @@ def _get_asset_info(self, asset: str) -> AssetInfo: info = AssetInfo( url=asset_info.get_absolute_href() or asset_info.href, - metadata=extras, + metadata=extras if not vrt_options else None, ) + if STAC_ALTERNATE_KEY and extras.get("alternate"): + if alternate := extras["alternate"].get(STAC_ALTERNATE_KEY): + info["url"] = alternate["href"] + + # https://github.com/stac-extensions/file if head := extras.get("file:header_size"): info["env"] = {"GDAL_INGESTED_BYTES_AT_OPEN": head} - if bands := extras.get("raster:bands"): + # https://github.com/stac-extensions/raster + if (bands := extras.get("raster:bands")) and not vrt_options: stats = [ (b["statistics"]["minimum"], b["statistics"]["maximum"]) for b in bands @@ -319,4 +345,7 @@ def _get_asset_info(self, asset: str) -> AssetInfo: "Some statistics data in STAC are invalid, they will be ignored." ) + if vrt_options: + info["url"] = f"vrt://{info['url']}?{vrt_options}" + return info diff --git a/tests/fixtures/gfs.t06z.pgrb2.10p0.f010.grib2 b/tests/fixtures/gfs.t06z.pgrb2.10p0.f010.grib2 new file mode 100644 index 00000000..af97027f Binary files /dev/null and b/tests/fixtures/gfs.t06z.pgrb2.10p0.f010.grib2 differ diff --git a/tests/fixtures/stac_alternate.json b/tests/fixtures/stac_alternate.json new file mode 100644 index 00000000..0bd8f999 --- /dev/null +++ b/tests/fixtures/stac_alternate.json @@ -0,0 +1,85 @@ +{ + "stac_version": "0.9.0", + "stac_extensions": [ + "https://stac-extensions.github.io/file/v2.1.0/schema.json", + "https://stac-extensions.github.io/alternate-assets/v1.2.0/schema.json" + ], + "type": "Feature", + "id": "JQT-123456789", + "bbox": [-81.3085227080129, 32.10817938759764, -78.81735409341113, 34.22870275071835], + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -81.3085227080129, + 32.10817938759764 + ], + [ + -78.81735409341113, + 32.10817938759764 + ], + [ + -78.81735409341113, + 34.22870275071835 + ], + [ + -81.3085227080129, + 34.22870275071835 + ], + [ + -81.3085227080129, + 32.10817938759764 + ] + ] + ] + }, + "properties": { + "datetime": "2016-05-03T13:21:30.040Z", + "collection": "JQT" + }, + "links": [ + { + "rel": "self", + "href": "http://cool-sat.com/catalog/JQT/a-fake-item.json" + }, + { + "rel": "collection", + "href": "http://cool-sat.com/catalog.json" + } + ], + "assets": { + "red": { + "href": "http://somewhere-over-the-rainbow.io/red.tif", + "title": "red", + "file:header_size": 16384, + "alternate:name": "HTTPS", + "alternate": { + "s3": { + "href": "s3://somewhere-over-the-rainbow.io/red.tif", + "alternate:name": "S3" + } + } + }, + "green": { + "href": "http://somewhere-over-the-rainbow.io/green.tif", + "title": "green", + "file:header_size": 30000 + }, + "blue": { + "href": "http://somewhere-over-the-rainbow.io/blue.tif", + "title": "blue", + "file:header_size": 20000 + }, + "lowres": { + "href": "http://somewhere-over-the-rainbow.io/lowres.tif", + "title": "lowres" + }, + "thumbnail": { + "href": "http://cool-sat.com/catalog/a-fake-item/thumbnail.png", + "title": "Thumbnail", + "type": "image/png", + "roles": [ "thumbnail" ] + } + } +} diff --git a/tests/fixtures/stac_grib.json b/tests/fixtures/stac_grib.json new file mode 100644 index 00000000..09b5096d --- /dev/null +++ b/tests/fixtures/stac_grib.json @@ -0,0 +1,304 @@ +{ + "type": "Feature", + "stac_version": "1.0.0", + "id": "gribfile", + "properties": { + "proj:geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -180.125, + -89.875 + ], + [ + 179.875, + -89.875 + ], + [ + 179.875, + 90.125 + ], + [ + -180.125, + 90.125 + ], + [ + -180.125, + -89.875 + ] + ] + ] + }, + "proj:bbox": [ + -180.125, + -89.875, + 179.875, + 90.125 + ], + "proj:shape": [ + 18, + 36 + ], + "proj:transform": [ + 10.0, + 0.0, + -180.125, + 0.0, + -10.0, + 90.125, + 0.0, + 0.0, + 1.0 + ], + "proj:wkt2": "GEOGCS[\"Coordinate System imported from GRIB file\",DATUM[\"unnamed\",SPHEROID[\"Sphere\",6371229,0]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AXIS[\"Latitude\",NORTH],AXIS[\"Longitude\",EAST]]", + "datetime": "2024-09-13T11:08:36.893626Z" + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -180.125, + -89.875 + ], + [ + 179.875, + -89.875 + ], + [ + 179.875, + 90.125 + ], + [ + -180.125, + 90.125 + ], + [ + -180.125, + -89.875 + ] + ] + ] + }, + "links": [], + "assets": { + "asset": { + "href": "gfs.t06z.pgrb2.10p0.f010.grib2", + "raster:bands": [ + { + "data_type": "float64", + "scale": 1.0, + "offset": 0.0, + "statistics": { + "mean": -12.643815520808767, + "minimum": -20.000003814697266, + "maximum": 32.659996032714844, + "stddev": 12.448285782027321, + "valid_percent": 0.15432098765432098 + }, + "histogram": { + "count": 11, + "min": -20.000003814697266, + "max": 32.659996032714844, + "buckets": [ + 449, + 27, + 29, + 31, + 35, + 29, + 15, + 21, + 9, + 3 + ] + } + }, + { + "data_type": "float64", + "scale": 1.0, + "offset": 0.0, + "statistics": { + "mean": -12.609587132580073, + "minimum": -20.000003814697266, + "maximum": 32.66999435424805, + "stddev": 12.450450829609293, + "valid_percent": 0.15432098765432098 + }, + "histogram": { + "count": 11, + "min": -20.000003814697266, + "max": 32.66999435424805, + "buckets": [ + 447, + 27, + 31, + 29, + 39, + 28, + 14, + 21, + 9, + 3 + ] + } + }, + { + "data_type": "float64", + "scale": 1.0, + "offset": 0.0, + "statistics": { + "mean": -10.566084066483503, + "minimum": -20.000003814697266, + "maximum": 32.76999282836914, + "stddev": 13.981093035778388, + "valid_percent": 0.15432098765432098 + }, + "histogram": { + "count": 11, + "min": -20.000003814697266, + "max": 32.76999282836914, + "buckets": [ + 412, + 25, + 31, + 27, + 34, + 49, + 32, + 20, + 11, + 7 + ] + } + }, + { + "data_type": "float64", + "scale": 1.0, + "offset": 0.0, + "statistics": { + "mean": 20302.916205571022, + "minimum": 24.859272003173828, + "maximum": 24134.859375, + "stddev": 7731.471190491879, + "valid_percent": 0.15432098765432098 + }, + "histogram": { + "count": 11, + "min": 24.859272003173828, + "max": 24134.859375, + "buckets": [ + 49, + 24, + 17, + 11, + 8, + 6, + 11, + 11, + 11, + 500 + ] + } + }, + { + "data_type": "float64", + "scale": 1.0, + "offset": 0.0, + "statistics": { + "mean": -0.13451403068101184, + "minimum": -18.402570724487305, + "maximum": 29.097431182861328, + "stddev": 7.346453975073525, + "valid_percent": 0.15432098765432098 + }, + "histogram": { + "count": 11, + "min": -18.402570724487305, + "max": 29.097431182861328, + "buckets": [ + 16, + 47, + 120, + 191, + 153, + 64, + 32, + 15, + 8, + 2 + ] + } + }, + { + "data_type": "float64", + "scale": 1.0, + "offset": 0.0, + "statistics": { + "mean": 0.3850667698676755, + "minimum": -27.066476821899414, + "maximum": 20.933523178100586, + "stddev": 6.63843088754233, + "valid_percent": 0.15432098765432098 + }, + "histogram": { + "count": 11, + "min": -27.066476821899414, + "max": 20.933523178100586, + "buckets": [ + 2, + 7, + 15, + 34, + 106, + 212, + 175, + 64, + 25, + 8 + ] + } + } + ], + "eo:bands": [ + { + "name": "b1", + "description": "1[-] HYBL=\"Hybrid level\"" + }, + { + "name": "b2", + "description": "2[-] HYBL=\"Hybrid level\"" + }, + { + "name": "b3", + "description": "0[-] EATM=\"Entire Atmosphere\"" + }, + { + "name": "b4", + "description": "0[-] SFC=\"Ground or water surface\"" + }, + { + "name": "b5", + "description": "0[-] RESERVED(220) (Reserved for local use)" + }, + { + "name": "b6", + "description": "0[-] RESERVED(220) (Reserved for local use)" + } + ], + "roles": [] + } + }, + "bbox": [ + -180.125, + -89.875, + 179.875, + 90.125 + ], + "stac_extensions": [ + "https://stac-extensions.github.io/projection/v1.1.0/schema.json", + "https://stac-extensions.github.io/raster/v1.1.0/schema.json", + "https://stac-extensions.github.io/eo/v1.1.0/schema.json" + ] +} diff --git a/tests/test_io_stac.py b/tests/test_io_stac.py index e4f1b2cf..2f070000 100644 --- a/tests/test_io_stac.py +++ b/tests/test_io_stac.py @@ -19,6 +19,7 @@ TileOutsideBounds, ) from rio_tiler.io import Reader, STACReader +from rio_tiler.io.stac import DEFAULT_VALID_TYPE from rio_tiler.models import BandStatistics PREFIX = os.path.join(os.path.dirname(__file__), "fixtures") @@ -27,6 +28,8 @@ STAC_GDAL_PATH = os.path.join(PREFIX, "stac_headers.json") STAC_RASTER_PATH = os.path.join(PREFIX, "stac_raster.json") STAC_WRONGSTATS_PATH = os.path.join(PREFIX, "stac_wrong_stats.json") +STAC_ALTERNATE_PATH = os.path.join(PREFIX, "stac_alternate.json") +STAC_GRIB_PATH = os.path.join(PREFIX, "stac_grib.json") with open(STAC_PATH) as f: item = json.loads(f.read()) @@ -890,3 +893,52 @@ def test_expression_with_wrong_stac_stats(rio): expression="where((wrongstat>0.5),1,0)", asset_as_band=True, ) + + +@patch("rio_tiler.io.stac.STAC_ALTERNATE_KEY", "s3") +def test_alternate_assets(): + """Should return the alternate key""" + with STACReader(STAC_ALTERNATE_PATH) as stac: + assert stac._get_asset_info("red")["url"].startswith("s3://") + # fall back to href when alternate doesn't exist + assert stac._get_asset_info("blue")["url"].startswith("http://") + + +def test_vrt_string_assets(): + """Should work with VRT connection string""" + VALID_TYPE = { + *DEFAULT_VALID_TYPE, + "application/wmo-GRIB2", + } + + with STACReader(STAC_GRIB_PATH, include_asset_types=VALID_TYPE) as stac: + assert stac.assets == ["asset"] + info = stac._get_asset_info("asset") + assert info["url"] + + info_vrt = stac._get_asset_info("vrt://asset") + # without any option there is no need to use the vrt prefix + assert info["url"] == info_vrt["url"] + + info_vrt = stac._get_asset_info("vrt://asset?bands=1") + assert not info["url"] == info_vrt["url"] + assert info_vrt["url"].startswith("vrt://") and info_vrt["url"].endswith( + "?bands=1" + ) + + with pytest.raises(InvalidAssetName): + stac._get_asset_info("vrt://somthing?bands=1") + + with pytest.raises(InvalidAssetName): + stac._get_asset_info("vrt://?bands=1") + + info = stac.info(assets="vrt://asset?bands=1") + assert info["vrt://asset?bands=1"] + assert len(info["vrt://asset?bands=1"].band_metadata) == 1 + + info = stac.info(assets="vrt://asset?bands=1,2") + assert info["vrt://asset?bands=1,2"] + assert len(info["vrt://asset?bands=1,2"].band_metadata) == 2 + + img = stac.preview(assets="vrt://asset?bands=1") + assert img.count == 1