From 4e70e589eac1f55373c0fd7f00f67e2c4102a8d5 Mon Sep 17 00:00:00 2001 From: Stefaan Lippens Date: Wed, 13 Sep 2023 13:00:37 +0200 Subject: [PATCH] Verify back-end UDP support before doing request for better error messages --- CHANGELOG.md | 2 + openeo/rest/__init__.py | 4 + openeo/rest/_testing.py | 61 +++++++++++ openeo/rest/connection.py | 21 ++-- openeo/rest/udp.py | 5 +- tests/rest/datacube/conftest.py | 19 +++- tests/rest/datacube/test_datacube100.py | 7 +- tests/rest/test_connection.py | 128 ++++++++++++++---------- tests/rest/test_udp.py | 3 +- 9 files changed, 181 insertions(+), 69 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 62afcf4b0..52e825aba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Before doing user-defined process (UDP) listing/creation: verify that back-end supports that (through openEO capabilities document) to improve error message. + ### Removed - Bumped minimal supported Python version to 3.7 ([#460](https://github.com/Open-EO/openeo-python-client/issues/460)) diff --git a/openeo/rest/__init__.py b/openeo/rest/__init__.py index 6738c507b..ffeddf10e 100644 --- a/openeo/rest/__init__.py +++ b/openeo/rest/__init__.py @@ -6,6 +6,10 @@ class OpenEoClientException(BaseOpenEoException): pass +class CapabilitiesException(OpenEoClientException): + """Back-end does not support certain openEO feature or endpoint.""" + + class JobFailedException(OpenEoClientException): """A synchronous batch job failed. This exception references its corresponding job so the client can e.g. retrieve its logs. diff --git a/openeo/rest/_testing.py b/openeo/rest/_testing.py index de37bd213..c87c56017 100644 --- a/openeo/rest/_testing.py +++ b/openeo/rest/_testing.py @@ -123,3 +123,64 @@ def execute(self, cube: Union[DataCube, VectorCube], process_id: Optional[str] = """ cube.execute() return self.get_pg(process_id=process_id) + + +def build_capabilities( + *, + api_version: str = "1.0.0", + stac_version: str = "0.9.0", + basic_auth: bool = True, + oidc_auth: bool = True, + collections: bool = True, + processes: bool = True, + sync_processing: bool = True, + validation: bool = False, + batch_jobs: bool = True, + udp: bool = False, +) -> dict: + """Build a dummy capabilities document for testing purposes.""" + + endpoints = [] + if basic_auth: + endpoints.append({"path": "/credentials/basic", "methods": ["GET"]}) + if oidc_auth: + endpoints.append({"path": "/credentials/oidc", "methods": ["GET"]}) + if basic_auth or oidc_auth: + endpoints.append({"path": "/me", "methods": ["GET"]}) + + if collections: + endpoints.append({"path": "/collections", "methods": ["GET"]}) + endpoints.append({"path": "/collections/{collection_id}", "methods": ["GET"]}) + if processes: + endpoints.append({"path": "/processes", "methods": ["GET"]}) + if sync_processing: + endpoints.append({"path": "/result", "methods": ["POST"]}) + if validation: + endpoints.append({"path": "/validation", "methods": ["POST"]}) + if batch_jobs: + endpoints.extend( + [ + {"path": "/jobs", "methods": ["GET", "POST"]}, + {"path": "/jobs/{job_id}", "methods": ["GET", "DELETE"]}, + {"path": "/jobs/{job_id}/results", "methods": ["GET", "POST", "DELETE"]}, + {"path": "/jobs/{job_id}/logs", "methods": ["GET"]}, + ] + ) + if udp: + endpoints.extend( + [ + {"path": "/process_graphs", "methods": ["GET"]}, + {"path": "/process_graphs/{process_graph_id", "methods": ["GET", "PUT", "DELETE"]}, + ] + ) + + capabilities = { + "api_version": api_version, + "stac_version": stac_version, + "id": "dummy", + "title": "Dummy openEO back-end", + "description": "Dummy openeEO back-end", + "endpoints": endpoints, + "links": [], + } + return capabilities diff --git a/openeo/rest/connection.py b/openeo/rest/connection.py index 8e5e1f0ad..b765a5309 100644 --- a/openeo/rest/connection.py +++ b/openeo/rest/connection.py @@ -27,14 +27,8 @@ from openeo.internal.jupyter import VisualDict, VisualList from openeo.internal.processes.builder import ProcessBuilderBase from openeo.internal.warnings import deprecated, legacy_alias -from openeo.metadata import ( - Band, - BandDimension, - CollectionMetadata, - SpatialDimension, - TemporalDimension, -) -from openeo.rest import OpenEoApiError, OpenEoClientException, OpenEoRestError +from openeo.metadata import Band, BandDimension, CollectionMetadata, SpatialDimension, TemporalDimension +from openeo.rest import CapabilitiesException, OpenEoApiError, OpenEoClientException, OpenEoRestError from openeo.rest._datacube import build_child_callback from openeo.rest.auth.auth import BasicBearerAuth, BearerAuth, NullAuth, OidcBearerAuth from openeo.rest.auth.config import AuthConfig, RefreshTokenStore @@ -984,6 +978,15 @@ def list_jobs(self) -> List[dict]: jobs = resp["jobs"] return VisualList("data-table", data=jobs, parameters={'columns': 'jobs'}) + def assert_user_defined_process_support(self): + """ + Capabilities document based verification that back-end supports user-defined processes. + + .. versionadded:: 0.23.0 + """ + if not self.capabilities().supports_endpoint("/process_graphs"): + raise CapabilitiesException("Backend does not support user-defined processes.") + def save_user_defined_process( self, user_defined_process_id: str, process_graph: Union[dict, ProcessBuilderBase], @@ -1011,6 +1014,7 @@ def save_user_defined_process( :param links: A list of links. :return: a RESTUserDefinedProcess instance """ + self.assert_user_defined_process_support() if user_defined_process_id in set(p["id"] for p in self.list_processes()): warnings.warn("Defining user-defined process {u!r} with same id as a pre-defined process".format( u=user_defined_process_id)) @@ -1028,6 +1032,7 @@ def list_user_defined_processes(self) -> List[dict]: """ Lists all user-defined processes of the authenticated user. """ + self.assert_user_defined_process_support() data = self.get("/process_graphs", expected_status=200).json()["processes"] return VisualList("processes", data=data, parameters={'show-graph': True, 'provide-download': False}) diff --git a/openeo/rest/udp.py b/openeo/rest/udp.py index b390ed761..0ccd20135 100644 --- a/openeo/rest/udp.py +++ b/openeo/rest/udp.py @@ -66,6 +66,7 @@ class RESTUserDefinedProcess: def __init__(self, user_defined_process_id: str, connection: Connection): self.user_defined_process_id = user_defined_process_id self._connection = connection + self._connection.assert_user_defined_process_support() def _repr_html_(self): process = self.describe() @@ -93,7 +94,9 @@ def store( # TODO: this "public" flag is not standardized yet EP-3609, https://github.com/Open-EO/openeo-api/issues/310 process["public"] = public - self._connection.put(path="/process_graphs/{}".format(self.user_defined_process_id), json=process) + self._connection.put( + path="/process_graphs/{}".format(self.user_defined_process_id), json=process, expected_status=200 + ) @deprecated( "Use `store` instead. Method `update` is misleading: OpenEO API does not provide (partial) updates" diff --git a/tests/rest/datacube/conftest.py b/tests/rest/datacube/conftest.py index 26b6eee4e..b8bebf0e0 100644 --- a/tests/rest/datacube/conftest.py +++ b/tests/rest/datacube/conftest.py @@ -1,8 +1,9 @@ -from typing import List +from typing import List, Optional import pytest import openeo +from openeo.rest._testing import build_capabilities from openeo.rest.connection import Connection from openeo.rest.datacube import DataCube @@ -26,9 +27,11 @@ } -def _setup_connection(api_version, requests_mock) -> Connection: +def _setup_connection(api_version, requests_mock, build_capabilities_kwargs: Optional[dict] = None) -> Connection: # TODO: make this more reusable? - requests_mock.get(API_URL + "/", json={"api_version": api_version}) + requests_mock.get( + API_URL + "/", json=build_capabilities(api_version=api_version, **(build_capabilities_kwargs or {})) + ) # Classic Sentinel2 collection requests_mock.get(API_URL + "/collections/SENTINEL2_RADIOMETRY_10M", json=DEFAULT_S2_METADATA) # Alias for quick tests @@ -64,6 +67,12 @@ def setup_collection_metadata(requests_mock, cid: str, bands: List[str]): }) +@pytest.fixture +def support_udp() -> bool: + """Per-test overridable `build_capabilities_kwargs(udp=...)` value for connection fixtures""" + return False + + @pytest.fixture def connection(api_version, requests_mock) -> Connection: """Connection fixture to a backend of given version with some image collections.""" @@ -71,9 +80,9 @@ def connection(api_version, requests_mock) -> Connection: @pytest.fixture -def con100(requests_mock) -> Connection: +def con100(requests_mock, support_udp) -> Connection: """Connection fixture to a 1.0.0 backend with some image collections.""" - return _setup_connection("1.0.0", requests_mock) + return _setup_connection("1.0.0", requests_mock, build_capabilities_kwargs={"udp": support_udp}) @pytest.fixture diff --git a/tests/rest/datacube/test_datacube100.py b/tests/rest/datacube/test_datacube100.py index 29f49ce4e..34892c2ee 100644 --- a/tests/rest/datacube/test_datacube100.py +++ b/tests/rest/datacube/test_datacube100.py @@ -25,10 +25,11 @@ from openeo.internal.graph_building import PGNode from openeo.internal.process_graph_visitor import ProcessGraphVisitException from openeo.internal.warnings import UserDeprecationWarning +from openeo.processes import ProcessBuilder from openeo.rest import OpenEoClientException +from openeo.rest._testing import build_capabilities from openeo.rest.connection import Connection from openeo.rest.datacube import THIS, UDF, DataCube -from openeo.processes import ProcessBuilder from openeo.rest.vectorcube import VectorCube from ... import load_json_resource @@ -1793,7 +1794,9 @@ def test_custom_process_arguments_namespacd(con100: Connection): assert res.flat_graph() == expected +@pytest.mark.parametrize("support_udp", [True]) def test_save_user_defined_process(con100, requests_mock): + requests_mock.get(API_URL + "/", json=build_capabilities(udp=True)) requests_mock.get(API_URL + "/processes", json={"processes": [{"id": "add"}]}) expected_body = load_json_resource("data/1.0.0/save_user_defined_process.json") @@ -1815,7 +1818,9 @@ def check_body(request): assert adapter.called +@pytest.mark.parametrize("support_udp", [True]) def test_save_user_defined_process_public(con100, requests_mock): + requests_mock.get(API_URL + "/", json=build_capabilities(udp=True)) requests_mock.get(API_URL + "/processes", json={"processes": [{"id": "add"}]}) expected_body = load_json_resource("data/1.0.0/save_user_defined_process.json") diff --git a/tests/rest/test_connection.py b/tests/rest/test_connection.py index d852c55a6..b2d88fa78 100644 --- a/tests/rest/test_connection.py +++ b/tests/rest/test_connection.py @@ -18,7 +18,8 @@ from openeo.capabilities import ApiVersionException, ComparableVersion from openeo.internal.compat import nullcontext from openeo.internal.graph_building import FlatGraphableMixin, PGNode -from openeo.rest import OpenEoApiError, OpenEoClientException, OpenEoRestError +from openeo.rest import CapabilitiesException, OpenEoApiError, OpenEoClientException, OpenEoRestError +from openeo.rest._testing import build_capabilities from openeo.rest.auth.auth import BearerAuth, NullAuth from openeo.rest.auth.oidc import OidcException from openeo.rest.auth.testing import ABSENT, OidcMock @@ -2656,80 +2657,101 @@ def test_execute_100(requests_mock, pg): ] -def test_create_udp(requests_mock): - requests_mock.get(API_URL, json={"api_version": "1.0.0"}) - requests_mock.get(API_URL + "processes", json={"processes": [{"id": "add"}]}) - conn = Connection(API_URL) +class TestUserDefinedProcesses: + """Test for UDP features""" - new_udp = load_json_resource("data/1.0.0/udp_details.json") + def test_create_udp(self, requests_mock): + requests_mock.get(API_URL, json=build_capabilities(udp=True)) + requests_mock.get(API_URL + "processes", json={"processes": [{"id": "add"}]}) + conn = Connection(API_URL) - def check_body(request): - body = request.json() - assert body['process_graph'] == new_udp['process_graph'] - assert body['parameters'] == new_udp['parameters'] - assert body['public'] is False - return True + new_udp = load_json_resource("data/1.0.0/udp_details.json") - adapter = requests_mock.put(API_URL + "process_graphs/evi", additional_matcher=check_body) + def check_body(request): + body = request.json() + assert body["process_graph"] == new_udp["process_graph"] + assert body["parameters"] == new_udp["parameters"] + assert body["public"] is False + return True - conn.save_user_defined_process( - user_defined_process_id='evi', - process_graph=new_udp['process_graph'], - parameters=new_udp['parameters'] - ) + adapter = requests_mock.put(API_URL + "process_graphs/evi", additional_matcher=check_body) - assert adapter.called + conn.save_user_defined_process( + user_defined_process_id="evi", process_graph=new_udp["process_graph"], parameters=new_udp["parameters"] + ) + assert adapter.called -def test_create_public_udp(requests_mock): - requests_mock.get(API_URL, json={"api_version": "1.0.0"}) - requests_mock.get(API_URL + "processes", json={"processes": [{"id": "add"}]}) - conn = Connection(API_URL) + def test_create_udp_public(self, requests_mock): + requests_mock.get(API_URL, json=build_capabilities(udp=True)) + requests_mock.get(API_URL + "processes", json={"processes": [{"id": "add"}]}) + conn = Connection(API_URL) - new_udp = load_json_resource("data/1.0.0/udp_details.json") + new_udp = load_json_resource("data/1.0.0/udp_details.json") - def check_body(request): - body = request.json() - assert body['process_graph'] == new_udp['process_graph'] - assert body['parameters'] == new_udp['parameters'] - assert body['public'] is True - return True + def check_body(request): + body = request.json() + assert body["process_graph"] == new_udp["process_graph"] + assert body["parameters"] == new_udp["parameters"] + assert body["public"] is True + return True - adapter = requests_mock.put(API_URL + "process_graphs/evi", additional_matcher=check_body) + adapter = requests_mock.put(API_URL + "process_graphs/evi", additional_matcher=check_body) - conn.save_user_defined_process( - user_defined_process_id='evi', - process_graph=new_udp['process_graph'], - parameters=new_udp['parameters'], - public=True - ) + conn.save_user_defined_process( + user_defined_process_id="evi", + process_graph=new_udp["process_graph"], + parameters=new_udp["parameters"], + public=True, + ) - assert adapter.called + assert adapter.called + def test_create_udp_unsupported(self, requests_mock): + requests_mock.get(API_URL, json=build_capabilities(udp=False)) + conn = Connection(API_URL) -def test_list_udps(requests_mock): - requests_mock.get(API_URL, json={"api_version": "1.0.0"}) - conn = Connection(API_URL) + new_udp = load_json_resource("data/1.0.0/udp_details.json") - udp = load_json_resource("data/1.0.0/udp_details.json") + with pytest.raises(CapabilitiesException, match="Backend does not support user-defined processes."): + _ = conn.save_user_defined_process( + user_defined_process_id="evi", process_graph=new_udp["process_graph"], parameters=new_udp["parameters"] + ) - requests_mock.get(API_URL + "process_graphs", json={ - 'processes': [udp] - }) + def test_list_udps(self, requests_mock): + requests_mock.get(API_URL, json=build_capabilities(udp=True)) + conn = Connection(API_URL) - user_udps = conn.list_user_defined_processes() + udp = load_json_resource("data/1.0.0/udp_details.json") - assert len(user_udps) == 1 - assert user_udps[0] == udp + requests_mock.get(API_URL + "process_graphs", json={"processes": [udp]}) + user_udps = conn.list_user_defined_processes() + + assert len(user_udps) == 1 + assert user_udps[0] == udp -def test_get_udp(requests_mock): - requests_mock.get(API_URL, json={"api_version": "1.0.0"}) - conn = Connection(API_URL) - udp = conn.user_defined_process('evi') + def test_list_udps_unsupported(self, requests_mock): + requests_mock.get(API_URL, json=build_capabilities(udp=False)) + conn = Connection(API_URL) + with pytest.raises(CapabilitiesException, match="Backend does not support user-defined processes."): + _ = conn.list_user_defined_processes() - assert udp.user_defined_process_id == 'evi' + + def test_get_udp(self, requests_mock): + requests_mock.get(API_URL, json=build_capabilities(udp=True)) + conn = Connection(API_URL) + + udp = conn.user_defined_process("evi") + + assert udp.user_defined_process_id == "evi" + + def test_get_udp_unsupported(self, requests_mock): + requests_mock.get(API_URL, json=build_capabilities(udp=False)) + conn = Connection(API_URL) + with pytest.raises(CapabilitiesException, match="Backend does not support user-defined processes."): + _ = conn.user_defined_process("evi") def _gzip_compress(data: bytes) -> bytes: diff --git a/tests/rest/test_udp.py b/tests/rest/test_udp.py index b793fcb64..58ba7ae7f 100644 --- a/tests/rest/test_udp.py +++ b/tests/rest/test_udp.py @@ -4,6 +4,7 @@ import openeo from openeo.api.process import Parameter +from openeo.rest._testing import build_capabilities from openeo.rest.udp import RESTUserDefinedProcess, build_process_dict from .. import load_json_resource @@ -13,7 +14,7 @@ @pytest.fixture def con100(requests_mock): - requests_mock.get(API_URL + "/", json={"api_version": "1.0.0"}) + requests_mock.get(API_URL + "/", json=build_capabilities(udp=True)) con = openeo.connect(API_URL) return con