Verify back-end UDP support before doing request for better error mes…

…sages

CHANGELOG.md

@@ -12,6 +12,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
+- Before doing user-defined process (UDP) listing/creation: verify that back-end supports that (through openEO capabilities document) to improve error message.
+
 ### Removed
 
 - Bumped minimal supported Python version to 3.7 ([#460](https://github.com/Open-EO/openeo-python-client/issues/460))

openeo/rest/__init__.py

@@ -6,6 +6,10 @@ class OpenEoClientException(BaseOpenEoException):
     pass
 
 
+class CapabilitiesException(OpenEoClientException):
+    """Back-end does not support certain openEO feature or endpoint."""
+
+
 class JobFailedException(OpenEoClientException):
     """A synchronous batch job failed. This exception references its corresponding job so the client can e.g.
     retrieve its logs.

openeo/rest/_testing.py

@@ -123,3 +123,64 @@ def execute(self, cube: Union[DataCube, VectorCube], process_id: Optional[str] =
         """
         cube.execute()
         return self.get_pg(process_id=process_id)
+
+
+def build_capabilities(
+    *,
+    api_version: str = "1.0.0",
+    stac_version: str = "0.9.0",
+    basic_auth: bool = True,
+    oidc_auth: bool = True,
+    collections: bool = True,
+    processes: bool = True,
+    sync_processing: bool = True,
+    validation: bool = False,
+    batch_jobs: bool = True,
+    udp: bool = False,
+) -> dict:
+    """Build a dummy capabilities document for testing purposes."""
+
+    endpoints = []
+    if basic_auth:
+        endpoints.append({"path": "/credentials/basic", "methods": ["GET"]})
+    if oidc_auth:
+        endpoints.append({"path": "/credentials/oidc", "methods": ["GET"]})
+    if basic_auth or oidc_auth:
+        endpoints.append({"path": "/me", "methods": ["GET"]})
+
+    if collections:
+        endpoints.append({"path": "/collections", "methods": ["GET"]})
+        endpoints.append({"path": "/collections/{collection_id}", "methods": ["GET"]})
+    if processes:
+        endpoints.append({"path": "/processes", "methods": ["GET"]})
+    if sync_processing:
+        endpoints.append({"path": "/result", "methods": ["POST"]})
+    if validation:
+        endpoints.append({"path": "/validation", "methods": ["POST"]})
+    if batch_jobs:
+        endpoints.extend(
+            [
+                {"path": "/jobs", "methods": ["GET", "POST"]},
+                {"path": "/jobs/{job_id}", "methods": ["GET", "DELETE"]},
+                {"path": "/jobs/{job_id}/results", "methods": ["GET", "POST", "DELETE"]},
+                {"path": "/jobs/{job_id}/logs", "methods": ["GET"]},
+            ]
+        )
+    if udp:
+        endpoints.extend(
+            [
+                {"path": "/process_graphs", "methods": ["GET"]},
+                {"path": "/process_graphs/{process_graph_id", "methods": ["GET", "PUT", "DELETE"]},
+            ]
+        )
+
+    capabilities = {
+        "api_version": api_version,
+        "stac_version": stac_version,
+        "id": "dummy",
+        "title": "Dummy openEO back-end",
+        "description": "Dummy openeEO back-end",
+        "endpoints": endpoints,
+        "links": [],
+    }
+    return capabilities

openeo/rest/connection.py

@@ -27,14 +27,8 @@
 from openeo.internal.jupyter import VisualDict, VisualList
 from openeo.internal.processes.builder import ProcessBuilderBase
 from openeo.internal.warnings import deprecated, legacy_alias
-from openeo.metadata import (
-    Band,
-    BandDimension,
-    CollectionMetadata,
-    SpatialDimension,
-    TemporalDimension,
-)
-from openeo.rest import OpenEoApiError, OpenEoClientException, OpenEoRestError
+from openeo.metadata import Band, BandDimension, CollectionMetadata, SpatialDimension, TemporalDimension
+from openeo.rest import CapabilitiesException, OpenEoApiError, OpenEoClientException, OpenEoRestError
 from openeo.rest._datacube import build_child_callback
 from openeo.rest.auth.auth import BasicBearerAuth, BearerAuth, NullAuth, OidcBearerAuth
 from openeo.rest.auth.config import AuthConfig, RefreshTokenStore
@@ -984,6 +978,15 @@ def list_jobs(self) -> List[dict]:
         jobs = resp["jobs"]
         return VisualList("data-table", data=jobs, parameters={'columns': 'jobs'})
 
+    def assert_user_defined_process_support(self):
+        """
+        Capabilities document based verification that back-end supports user-defined processes.
+
+        .. versionadded:: 0.23.0
+        """
+        if not self.capabilities().supports_endpoint("/process_graphs"):
+            raise CapabilitiesException("Backend does not support user-defined processes.")
+
     def save_user_defined_process(
             self, user_defined_process_id: str,
             process_graph: Union[dict, ProcessBuilderBase],
@@ -1011,6 +1014,7 @@ def save_user_defined_process(
         :param links: A list of links.
         :return: a RESTUserDefinedProcess instance
         """
+        self.assert_user_defined_process_support()
         if user_defined_process_id in set(p["id"] for p in self.list_processes()):
             warnings.warn("Defining user-defined process {u!r} with same id as a pre-defined process".format(
                 u=user_defined_process_id))
@@ -1028,6 +1032,7 @@ def list_user_defined_processes(self) -> List[dict]:
         """
         Lists all user-defined processes of the authenticated user.
         """
+        self.assert_user_defined_process_support()
         data = self.get("/process_graphs", expected_status=200).json()["processes"]
         return VisualList("processes", data=data, parameters={'show-graph': True, 'provide-download': False})
 

openeo/rest/udp.py

@@ -66,6 +66,7 @@ class RESTUserDefinedProcess:
     def __init__(self, user_defined_process_id: str, connection: Connection):
         self.user_defined_process_id = user_defined_process_id
         self._connection = connection
+        self._connection.assert_user_defined_process_support()
 
     def _repr_html_(self):
         process = self.describe()
@@ -93,7 +94,9 @@ def store(
         # TODO: this "public" flag is not standardized yet EP-3609, https://github.com/Open-EO/openeo-api/issues/310
         process["public"] = public
 
-        self._connection.put(path="/process_graphs/{}".format(self.user_defined_process_id), json=process)
+        self._connection.put(
+            path="/process_graphs/{}".format(self.user_defined_process_id), json=process, expected_status=200
+        )
 
     @deprecated(
         "Use `store` instead. Method `update` is misleading: OpenEO API does not provide (partial) updates"

tests/rest/datacube/conftest.py

@@ -1,8 +1,9 @@
-from typing import List
+from typing import List, Optional
 
 import pytest
 
 import openeo
+from openeo.rest._testing import build_capabilities
 from openeo.rest.connection import Connection
 from openeo.rest.datacube import DataCube
 
@@ -26,9 +27,11 @@
 }
 
 
-def _setup_connection(api_version, requests_mock) -> Connection:
+def _setup_connection(api_version, requests_mock, build_capabilities_kwargs: Optional[dict] = None) -> Connection:
     # TODO: make this more reusable?
-    requests_mock.get(API_URL + "/", json={"api_version": api_version})
+    requests_mock.get(
+        API_URL + "/", json=build_capabilities(api_version=api_version, **(build_capabilities_kwargs or {}))
+    )
     # Classic Sentinel2 collection
     requests_mock.get(API_URL + "/collections/SENTINEL2_RADIOMETRY_10M", json=DEFAULT_S2_METADATA)
     # Alias for quick tests
@@ -64,16 +67,22 @@ def setup_collection_metadata(requests_mock, cid: str, bands: List[str]):
     })
 
 
+@pytest.fixture
+def support_udp() -> bool:
+    """Per-test overridable `build_capabilities_kwargs(udp=...)` value for connection fixtures"""
+    return False
+
+
 @pytest.fixture
 def connection(api_version, requests_mock) -> Connection:
     """Connection fixture to a backend of given version with some image collections."""
     return _setup_connection(api_version, requests_mock)
 
 
 @pytest.fixture
-def con100(requests_mock) -> Connection:
+def con100(requests_mock, support_udp) -> Connection:
     """Connection fixture to a 1.0.0 backend with some image collections."""
-    return _setup_connection("1.0.0", requests_mock)
+    return _setup_connection("1.0.0", requests_mock, build_capabilities_kwargs={"udp": support_udp})
 
 
 @pytest.fixture

tests/rest/datacube/test_datacube100.py

@@ -25,10 +25,11 @@
 from openeo.internal.graph_building import PGNode
 from openeo.internal.process_graph_visitor import ProcessGraphVisitException
 from openeo.internal.warnings import UserDeprecationWarning
+from openeo.processes import ProcessBuilder
 from openeo.rest import OpenEoClientException
+from openeo.rest._testing import build_capabilities
 from openeo.rest.connection import Connection
 from openeo.rest.datacube import THIS, UDF, DataCube
-from openeo.processes import ProcessBuilder
 from openeo.rest.vectorcube import VectorCube
 
 from ... import load_json_resource
@@ -1793,7 +1794,9 @@ def test_custom_process_arguments_namespacd(con100: Connection):
     assert res.flat_graph() == expected
 
 
+@pytest.mark.parametrize("support_udp", [True])
 def test_save_user_defined_process(con100, requests_mock):
+    requests_mock.get(API_URL + "/", json=build_capabilities(udp=True))
     requests_mock.get(API_URL + "/processes", json={"processes": [{"id": "add"}]})
 
     expected_body = load_json_resource("data/1.0.0/save_user_defined_process.json")
@@ -1815,7 +1818,9 @@ def check_body(request):
     assert adapter.called
 
 
+@pytest.mark.parametrize("support_udp", [True])
 def test_save_user_defined_process_public(con100, requests_mock):
+    requests_mock.get(API_URL + "/", json=build_capabilities(udp=True))
     requests_mock.get(API_URL + "/processes", json={"processes": [{"id": "add"}]})
 
     expected_body = load_json_resource("data/1.0.0/save_user_defined_process.json")

tests/rest/test_connection.py

@@ -18,7 +18,8 @@
 from openeo.capabilities import ApiVersionException, ComparableVersion
 from openeo.internal.compat import nullcontext
 from openeo.internal.graph_building import FlatGraphableMixin, PGNode
-from openeo.rest import OpenEoApiError, OpenEoClientException, OpenEoRestError
+from openeo.rest import CapabilitiesException, OpenEoApiError, OpenEoClientException, OpenEoRestError
+from openeo.rest._testing import build_capabilities
 from openeo.rest.auth.auth import BearerAuth, NullAuth
 from openeo.rest.auth.oidc import OidcException
 from openeo.rest.auth.testing import ABSENT, OidcMock
@@ -2656,80 +2657,101 @@ def test_execute_100(requests_mock, pg):
     ]
 
 
-def test_create_udp(requests_mock):
-    requests_mock.get(API_URL, json={"api_version": "1.0.0"})
-    requests_mock.get(API_URL + "processes", json={"processes": [{"id": "add"}]})
-    conn = Connection(API_URL)
+class TestUserDefinedProcesses:
+    """Test for UDP features"""
 
-    new_udp = load_json_resource("data/1.0.0/udp_details.json")
+    def test_create_udp(self, requests_mock):
+        requests_mock.get(API_URL, json=build_capabilities(udp=True))
+        requests_mock.get(API_URL + "processes", json={"processes": [{"id": "add"}]})
+        conn = Connection(API_URL)
 
-    def check_body(request):
-        body = request.json()
-        assert body['process_graph'] == new_udp['process_graph']
-        assert body['parameters'] == new_udp['parameters']
-        assert body['public'] is False
-        return True
+        new_udp = load_json_resource("data/1.0.0/udp_details.json")
 
-    adapter = requests_mock.put(API_URL + "process_graphs/evi", additional_matcher=check_body)
+        def check_body(request):
+            body = request.json()
+            assert body["process_graph"] == new_udp["process_graph"]
+            assert body["parameters"] == new_udp["parameters"]
+            assert body["public"] is False
+            return True
 
-    conn.save_user_defined_process(
-        user_defined_process_id='evi',
-        process_graph=new_udp['process_graph'],
-        parameters=new_udp['parameters']
-    )
+        adapter = requests_mock.put(API_URL + "process_graphs/evi", additional_matcher=check_body)
 
-    assert adapter.called
+        conn.save_user_defined_process(
+            user_defined_process_id="evi", process_graph=new_udp["process_graph"], parameters=new_udp["parameters"]
+        )
 
+        assert adapter.called
 
-def test_create_public_udp(requests_mock):
-    requests_mock.get(API_URL, json={"api_version": "1.0.0"})
-    requests_mock.get(API_URL + "processes", json={"processes": [{"id": "add"}]})
-    conn = Connection(API_URL)
+    def test_create_udp_public(self, requests_mock):
+        requests_mock.get(API_URL, json=build_capabilities(udp=True))
+        requests_mock.get(API_URL + "processes", json={"processes": [{"id": "add"}]})
+        conn = Connection(API_URL)
 
-    new_udp = load_json_resource("data/1.0.0/udp_details.json")
+        new_udp = load_json_resource("data/1.0.0/udp_details.json")
 
-    def check_body(request):
-        body = request.json()
-        assert body['process_graph'] == new_udp['process_graph']
-        assert body['parameters'] == new_udp['parameters']
-        assert body['public'] is True
-        return True
+        def check_body(request):
+            body = request.json()
+            assert body["process_graph"] == new_udp["process_graph"]
+            assert body["parameters"] == new_udp["parameters"]
+            assert body["public"] is True
+            return True
 
-    adapter = requests_mock.put(API_URL + "process_graphs/evi", additional_matcher=check_body)
+        adapter = requests_mock.put(API_URL + "process_graphs/evi", additional_matcher=check_body)
 
-    conn.save_user_defined_process(
-        user_defined_process_id='evi',
-        process_graph=new_udp['process_graph'],
-        parameters=new_udp['parameters'],
-        public=True
-    )
+        conn.save_user_defined_process(
+            user_defined_process_id="evi",
+            process_graph=new_udp["process_graph"],
+            parameters=new_udp["parameters"],
+            public=True,
+        )
 
-    assert adapter.called
+        assert adapter.called
 
+    def test_create_udp_unsupported(self, requests_mock):
+        requests_mock.get(API_URL, json=build_capabilities(udp=False))
+        conn = Connection(API_URL)
 
-def test_list_udps(requests_mock):
-    requests_mock.get(API_URL, json={"api_version": "1.0.0"})
-    conn = Connection(API_URL)
+        new_udp = load_json_resource("data/1.0.0/udp_details.json")
 
-    udp = load_json_resource("data/1.0.0/udp_details.json")
+        with pytest.raises(CapabilitiesException, match="Backend does not support user-defined processes."):
+            _ = conn.save_user_defined_process(
+                user_defined_process_id="evi", process_graph=new_udp["process_graph"], parameters=new_udp["parameters"]
+            )
 
-    requests_mock.get(API_URL + "process_graphs", json={
-        'processes': [udp]
-    })
+    def test_list_udps(self, requests_mock):
+        requests_mock.get(API_URL, json=build_capabilities(udp=True))
+        conn = Connection(API_URL)
 
-    user_udps = conn.list_user_defined_processes()
+        udp = load_json_resource("data/1.0.0/udp_details.json")
 
-    assert len(user_udps) == 1
-    assert user_udps[0] == udp
+        requests_mock.get(API_URL + "process_graphs", json={"processes": [udp]})
 
+        user_udps = conn.list_user_defined_processes()
+
+        assert len(user_udps) == 1
+        assert user_udps[0] == udp
 
-def test_get_udp(requests_mock):
-    requests_mock.get(API_URL, json={"api_version": "1.0.0"})
-    conn = Connection(API_URL)
 
-    udp = conn.user_defined_process('evi')
+    def test_list_udps_unsupported(self, requests_mock):
+        requests_mock.get(API_URL, json=build_capabilities(udp=False))
+        conn = Connection(API_URL)
+        with pytest.raises(CapabilitiesException, match="Backend does not support user-defined processes."):
+            _ = conn.list_user_defined_processes()
 
-    assert udp.user_defined_process_id == 'evi'
+
+    def test_get_udp(self, requests_mock):
+        requests_mock.get(API_URL, json=build_capabilities(udp=True))
+        conn = Connection(API_URL)
+
+        udp = conn.user_defined_process("evi")
+
+        assert udp.user_defined_process_id == "evi"
+
+    def test_get_udp_unsupported(self, requests_mock):
+        requests_mock.get(API_URL, json=build_capabilities(udp=False))
+        conn = Connection(API_URL)
+        with pytest.raises(CapabilitiesException, match="Backend does not support user-defined processes."):
+            _ = conn.user_defined_process("evi")
 
 
 def _gzip_compress(data: bytes) -> bytes: