From 233df6fc98389cef55aeaf0c1090765424f8afc6 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Fri, 9 Feb 2024 13:10:15 +0800 Subject: [PATCH 1/2] validate /collections endpoint --- README.md | 1 + stac_validator/stac_validator.py | 33 ++++++++++++++- stac_validator/validate.py | 30 +++++++++++++- tests/test_validate_collections.py | 56 ++++++++++++++++++++++++++ tests/test_validate_item_collection.py | 2 +- 5 files changed, 119 insertions(+), 3 deletions(-) create mode 100644 tests/test_validate_collections.py diff --git a/README.md b/README.md index 1f1bf6a9..7778dc6a 100644 --- a/README.md +++ b/README.md @@ -106,6 +106,7 @@ Options: -m, --max-depth INTEGER Maximum depth to traverse when recursing. Omit this argument to get full recursion. Ignored if `recursive == False`. + --collections Validate /collections response. --item-collection Validate item collection response. Can be combined with --pages. Defaults to one page. -p, --pages INTEGER Maximum number of pages to validate via --item- diff --git a/stac_validator/stac_validator.py b/stac_validator/stac_validator.py index 0a2a9cef..6dca8db6 100644 --- a/stac_validator/stac_validator.py +++ b/stac_validator/stac_validator.py @@ -46,6 +46,25 @@ def item_collection_summary(message: List[Dict[str, Any]]) -> None: click.secho(f"valid_items: {valid_count}") +def collections_summary(message: List[Dict[str, Any]]) -> None: + """Prints a summary of the validation results for an item collection response. + + Args: + message (List[Dict[str, Any]]): The validation results for the item collection. + + Returns: + None + """ + valid_count = 0 + for collection in message: + if "valid_stac" in collection and collection["valid_stac"] is True: + valid_count = valid_count + 1 + click.secho() + click.secho("--collections summary", bold=True) + click.secho(f"collections_validated: {len(message)}") + click.secho(f"valid_collections: {valid_count}") + + @click.command() @click.argument("stac_file") @click.option( @@ -80,6 +99,11 @@ def item_collection_summary(message: List[Dict[str, Any]]) -> None: type=int, help="Maximum depth to traverse when recursing. Omit this argument to get full recursion. Ignored if `recursive == False`.", ) +@click.option( + "--collections", + is_flag=True, + help="Validate /collections response.", +) @click.option( "--item-collection", is_flag=True, @@ -102,6 +126,7 @@ def item_collection_summary(message: List[Dict[str, Any]]) -> None: ) def main( stac_file: str, + collections: bool, item_collection: bool, pages: int, recursive: bool, @@ -120,6 +145,7 @@ def main( Args: stac_file (str): Path to the STAC file to be validated. + collections (bool): Validate response from /collections endpoint. item_collection (bool): Whether to validate item collection responses. pages (int): Maximum number of pages to validate via `item_collection`. recursive (bool): Whether to recursively validate all related STAC objects. @@ -143,6 +169,7 @@ def main( valid = True stac = StacValidate( stac_file=stac_file, + collections=collections, item_collection=item_collection, pages=pages, recursive=recursive, @@ -155,8 +182,10 @@ def main( verbose=verbose, log=log_file, ) - if not item_collection: + if not item_collection and not collections: valid = stac.run() + elif collections: + stac.validate_collections() else: stac.validate_item_collection() @@ -169,6 +198,8 @@ def main( if item_collection: item_collection_summary(message) + elif collections: + collections_summary(message) sys.exit(0 if valid else 1) diff --git a/stac_validator/validate.py b/stac_validator/validate.py index e89a6024..ac6ece9a 100644 --- a/stac_validator/validate.py +++ b/stac_validator/validate.py @@ -25,6 +25,7 @@ class StacValidate: Attributes: stac_file (str): The path or URL to the STAC object to be validated. + collections (bool): Validate response from a /collections endpoint. item_collection (bool): Whether the STAC object to be validated is an item collection. pages (int): The maximum number of pages to validate if `item_collection` is True. recursive (bool): Whether to recursively validate related STAC objects. @@ -45,6 +46,7 @@ class StacValidate: def __init__( self, stac_file: Optional[str] = None, + collections: bool = False, item_collection: bool = False, pages: Optional[int] = None, recursive: bool = False, @@ -58,6 +60,7 @@ def __init__( log: str = "", ): self.stac_file = stac_file + self.collections = collections self.item_collection = item_collection self.pages = pages self.message: List = [] @@ -392,6 +395,27 @@ def validate_item_collection_dict(self, item_collection: Dict) -> None: self.schema = "" self.validate_dict(item) + def validate_collections(self) -> None: + """ "Validate STAC collections from a /collections endpoint. + + Raises: + URLError: If there is an issue with the URL used to fetch the item collection. + JSONDecodeError: If the item collection content cannot be parsed as JSON. + ValueError: If the item collection does not conform to the STAC specification. + TypeError: If the item collection content is not a dictionary or JSON object. + FileNotFoundError: If the item collection file cannot be found. + ConnectionError: If there is an issue with the internet connection used to fetch the item collection. + exceptions.SSLError: If there is an issue with the SSL connection used to fetch the item collection. + OSError: If there is an issue with the file system (e.g., read/write permissions) while trying to write to the log file. + + Returns: + None + """ + collections = fetch_and_parse_file(str(self.stac_file)) + for collection in collections["collections"]: + self.schema = "" + self.validate_dict(collection) + def validate_item_collection(self) -> None: """Validate a STAC item collection. @@ -457,7 +481,11 @@ def run(self) -> bool: """ message = {} try: - if self.stac_file is not None and not self.item_collection: + if ( + self.stac_file is not None + and not self.item_collection + and not self.collections + ): self.stac_content = fetch_and_parse_file(self.stac_file) stac_type = get_stac_type(self.stac_content).upper() diff --git a/tests/test_validate_collections.py b/tests/test_validate_collections.py new file mode 100644 index 00000000..12aee812 --- /dev/null +++ b/tests/test_validate_collections.py @@ -0,0 +1,56 @@ +""" +Description: Test stac-validator on --collections (/collections validation). + +""" + + +from stac_validator import stac_validator + + +def test_validate_collections_remote(): + stac_file = "https://earth-search.aws.element84.com/v0/collections" + stac = stac_validator.StacValidate(stac_file, collections=True) + stac.validate_collections() + + assert stac.message == [ + { + "version": "1.0.0-beta.2", + "path": "https://earth-search.aws.element84.com/v0/collections", + "schema": [ + "https://schemas.stacspec.org/v1.0.0-beta.2/collection-spec/json-schema/collection.json" + ], + "valid_stac": True, + "asset_type": "COLLECTION", + "validation_method": "default", + }, + { + "version": "1.0.0-beta.2", + "path": "https://earth-search.aws.element84.com/v0/collections", + "schema": [ + "https://schemas.stacspec.org/v1.0.0-beta.2/collection-spec/json-schema/collection.json" + ], + "valid_stac": True, + "asset_type": "COLLECTION", + "validation_method": "default", + }, + { + "version": "1.0.0-beta.2", + "path": "https://earth-search.aws.element84.com/v0/collections", + "schema": [ + "https://schemas.stacspec.org/v1.0.0-beta.2/collection-spec/json-schema/collection.json" + ], + "valid_stac": True, + "asset_type": "COLLECTION", + "validation_method": "default", + }, + { + "version": "1.0.0-beta.2", + "path": "https://earth-search.aws.element84.com/v0/collections", + "schema": [ + "https://schemas.stacspec.org/v1.0.0-beta.2/collection-spec/json-schema/collection.json" + ], + "valid_stac": True, + "asset_type": "COLLECTION", + "validation_method": "default", + }, + ] diff --git a/tests/test_validate_item_collection.py b/tests/test_validate_item_collection.py index c27a2045..5b6fd439 100644 --- a/tests/test_validate_item_collection.py +++ b/tests/test_validate_item_collection.py @@ -1,5 +1,5 @@ """ -Description: Test the validator +Description: Test stac-validator on item-collection validation. """ From f3510be8b632e8ec5851b5e866053579c842b1e9 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Fri, 9 Feb 2024 15:12:25 +0800 Subject: [PATCH 2/2] update changelog, pre-commit versions --- .pre-commit-config.yaml | 8 ++++---- CHANGELOG.md | 1 + stac_validator/validate.py | 6 +++--- tests/test_validate_collections.py | 1 - tests/test_validate_item_collection.py | 1 - 5 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fec58f10..b4800948 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,20 +1,20 @@ repos: - repo: https://github.com/PyCQA/flake8 - rev: 6.1.0 + rev: 7.0.0 hooks: - id: flake8 - repo: https://github.com/timothycrosley/isort - rev: 5.12.0 + rev: 5.13.2 hooks: - id: isort args: ["--profile", "black"] - repo: https://github.com/psf/black - rev: 23.11.0 + rev: 24.1.1 hooks: - id: black language_version: python3.8 - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.7.0 + rev: v1.8.0 hooks: - id: mypy exclude: /tests/ diff --git a/CHANGELOG.md b/CHANGELOG.md index c75cf505..236997af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is (loosely) based on [Keep a Changelog](http://keepachangelog.com/) ### Added +- Added ability to validate response from a /collections endpoint [#220](https://github.com/stac-utils/stac-validator/issues/220) - Added mypy to pre-commit config ([#229](https://github.com/stac-utils/stac-validator/pull/224)) ## [v3.3.2] - 2023-11-17 diff --git a/stac_validator/validate.py b/stac_validator/validate.py index ac6ece9a..c4a0c94f 100644 --- a/stac_validator/validate.py +++ b/stac_validator/validate.py @@ -453,9 +453,9 @@ def validate_item_collection(self) -> None: break except Exception as e: message = {} - message[ - "pagination_error" - ] = f"Validating the item collection failed on page {page}: {str(e)}" + message["pagination_error"] = ( + f"Validating the item collection failed on page {page}: {str(e)}" + ) self.message.append(message) def run(self) -> bool: diff --git a/tests/test_validate_collections.py b/tests/test_validate_collections.py index 12aee812..fe7ba19e 100644 --- a/tests/test_validate_collections.py +++ b/tests/test_validate_collections.py @@ -3,7 +3,6 @@ """ - from stac_validator import stac_validator diff --git a/tests/test_validate_item_collection.py b/tests/test_validate_item_collection.py index 5b6fd439..c74db64a 100644 --- a/tests/test_validate_item_collection.py +++ b/tests/test_validate_item_collection.py @@ -3,7 +3,6 @@ """ - from stac_validator import stac_validator