Update backend calls to match changes to the CDS API

test/mock_results/test_client_url_and_key_environment_variables/request.json

@@ -0,0 +1 @@
+{"_dataset_name": "reanalysis-era5-single-levels-monthly-means", "variable": "2m_temperature", "area": [89.875, -179.875, -89.875, 179.875], "grid": [0.25, 0.25], "format": "netcdf", "product_type": "monthly_averaged_reanalysis", "time": "00:00", "day": "15", "month": "10", "year": "2015"}

test/mock_results/test_client_url_and_key_parameters/request.json

@@ -0,0 +1 @@
+{"_dataset_name": "reanalysis-era5-single-levels-monthly-means", "variable": "2m_temperature", "area": [89.875, -179.875, -89.875, 179.875], "grid": [0.25, 0.25], "format": "netcdf", "product_type": "monthly_averaged_reanalysis", "time": "00:00", "day": "15", "month": "10", "year": "2015"}

test/mock_results/test_client_url_and_key_rc_file/request.json

@@ -0,0 +1 @@
+{"_dataset_name": "reanalysis-era5-single-levels-monthly-means", "variable": "2m_temperature", "area": [89.875, -179.875, -89.875, 179.875], "grid": [0.25, 0.25], "format": "netcdf", "product_type": "monthly_averaged_reanalysis", "time": "00:00", "day": "15", "month": "10", "year": "2015"}

test/mock_results/test_copy_on_open/request.json

@@ -0,0 +1 @@
+{"_dataset_name": "satellite-soil-moisture", "variable": "volumetric_surface_soil_moisture", "type_of_sensor": "combined_passive_and_active", "time_aggregation": "month_average", "type_of_record": "cdr", "version": "v202012", "format": "tgz", "day": "01", "month": ["01", "02"], "year": "2015"}

test/mock_results/test_sea_ice_cryosat-2/request.json → test/mock_results/test_open_cryosat_2/request.json

@@ -1 +1 @@
-{"_dataset_name": "satellite-sea-ice-thickness", "satellite": "cryosat-2", "cdr_type": "cdr", "version": "2_0", "variable": "all", "format": "tgz", "month": ["03", "04"], "year": "2016"}
+{"_dataset_name": "satellite-sea-ice-thickness", "satellite": "cryosat_2", "cdr_type": "cdr", "version": "2_0", "variable": "all", "format": "tgz", "month": ["03", "04"], "year": "2016"}

test/mock_results/test_soil_moisture_saturation_10_day/request.json

@@ -1 +1 @@
-{"_dataset_name": "satellite-soil-moisture", "variable": "soil_moisture_saturation", "type_of_sensor": "active", "time_aggregation": "10_day_average", "type_of_record": "cdr", "version": "v201912.0.0", "format": "tgz", "day": ["01", "11"], "month": "04", "year": "2015"}
+{"_dataset_name": "satellite-soil-moisture", "variable": "surface_soil_moisture", "type_of_sensor": "active", "time_aggregation": "10_day_average", "type_of_record": "cdr", "version": "v202012", "format": "tgz", "day": ["01", "11"], "month": "04", "year": "2015"}

test/mock_results/test_soil_moisture_saturation_daily/request.json

@@ -1 +1 @@
-{"_dataset_name": "satellite-soil-moisture", "variable": "soil_moisture_saturation", "type_of_sensor": "active", "time_aggregation": "day_average", "type_of_record": "cdr", "version": "v201912.0.0", "format": "tgz", "day": ["01", "02", "03", "04"], "month": "03", "year": "2016"}
+{"_dataset_name": "satellite-soil-moisture", "variable": "surface_soil_moisture", "type_of_sensor": "active", "time_aggregation": "day_average", "type_of_record": "cdr", "version": "v202012", "format": "tgz", "day": ["01", "02", "03", "04"], "month": "03", "year": "2016"}

test/mock_results/test_soil_moisture_volumetric_minimal_params/request.json

@@ -1 +1 @@
-{"_dataset_name": "satellite-soil-moisture", "variable": "volumetric_surface_soil_moisture", "type_of_sensor": "combined_passive_and_active", "time_aggregation": "month_average", "type_of_record": "cdr", "version": "v201912.0.0", "format": "tgz", "day": "01", "month": ["01", "02"], "year": "2015"}
+{"_dataset_name": "satellite-soil-moisture", "variable": "volumetric_surface_soil_moisture", "type_of_sensor": "combined_passive_and_active", "time_aggregation": "month_average", "type_of_record": "cdr", "version": "v202012", "format": "tgz", "day": "01", "month": ["01", "02"], "year": "2015"}

test/mock_results/test_soil_moisture_volumetric_monthly_2_years/request.json

@@ -1 +1 @@
-{"_dataset_name": "satellite-soil-moisture", "variable": "volumetric_surface_soil_moisture", "type_of_sensor": "combined_passive_and_active", "time_aggregation": "month_average", "type_of_record": "cdr", "version": "v201912.0.0", "format": "tgz", "day": "01", "month": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"], "year": ["2015", "2016"]}
+{"_dataset_name": "satellite-soil-moisture", "variable": "volumetric_surface_soil_moisture", "type_of_sensor": "combined_passive_and_active", "time_aggregation": "month_average", "type_of_record": "cdr", "version": "v202012", "format": "tgz", "day": "01", "month": ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"], "year": ["2015", "2016"]}

test/mock_results/test_soil_moisture_volumetric_optional_params/request.json

@@ -0,0 +1 @@
+{"_dataset_name": "satellite-soil-moisture", "variable": "volumetric_surface_soil_moisture", "type_of_sensor": "combined_passive_and_active", "time_aggregation": "month_average", "type_of_record": "cdr", "version": "v202012", "format": "tgz", "day": "01", "month": ["01", "02"], "year": "2015"}

test/mocks.py

@@ -1,12 +1,49 @@
 import json
+import pathlib
 import shutil
 import os
+import inspect
+import enum
 
 import cdsapi
 
 
-class CDSClientMock:
+class _Behaviour(enum.Enum):
+    MOCK = enum.auto()
+    REAL_CLIENT = enum.auto()
+    SAVE_RESULTS = enum.auto()
+
+# MOCK uses a mock CDS API client returning pre-generated, saved results
+# REAL_CLIENT uses the real CDS client
+# SAVE_RESULTS uses the real CDS client and saves results for future mocking
+# REAL_CLIENT and SAVE_results require the credentials to be set
+_BEHAVIOUR = _Behaviour.MOCK
+
+
+class _SessionMock:
+    def close(self):
+        pass
+
 
+def _get_url_and_key(url, key):
+    if url is None:
+        url = os.environ.get('CDSAPI_URL')
+    if key is None:
+        key = os.environ.get('CDSAPI_KEY')
+    dotrc = os.environ.get('CDSAPI_RC', os.path.expanduser('~/.cdsapirc'))
+    if url is None or key is None:
+        if os.path.exists(dotrc):
+            config = cdsapi.api.read_config(dotrc)
+            if key is None:
+                key = config.get('key')
+            if url is None:
+                url = config.get('url')
+    if url is None or key is None:
+        raise Exception(f'Missing/incomplete configuration file: {dotrc}')
+    return url, key
+
+
+class CDSClientMock:
     """A simple mock of the cdsapi.Client class
 
     This mock class uses predefined requests from on-disk JSON files. When
@@ -26,39 +63,20 @@ class CDSClientMock:
     """
 
     def __init__(self, url=None, key=None):
-        class Session:
-            def close(self):
-                pass
-        self.session = Session()
-
-        if url is None:
-            url = os.environ.get('CDSAPI_URL')
-        if key is None:
-            key = os.environ.get('CDSAPI_KEY')
-        dotrc = os.environ.get('CDSAPI_RC', os.path.expanduser('~/.cdsapirc'))
-        if url is None or key is None:
-            if os.path.exists(dotrc):
-                config = cdsapi.api.read_config(dotrc)
-                if key is None:
-                    key = config.get('key')
-                if url is None:
-                    url = config.get('url')
-        if url is None or key is None:
-            raise Exception(f'Missing/incomplete configuration file: {dotrc}')
-
-        self.url = url
-        self.key = key
-
-        resource_path = os.path.join(os.path.dirname(__file__),
-                                     'mock_results')
+        self.session = _SessionMock()
+        self.url, self.key = _get_url_and_key(url, key)
+
+        resource_path = os.path.join(os.path.dirname(__file__), 'mock_results')
         # request_map is a list because dicts can't be hashed in Python, and
         # it's not worth introducing a dependency on frozendict just for this.
         self.request_map = []
         for d in os.listdir(resource_path):
             dir_path = os.path.join(resource_path, d)
             with open(os.path.join(dir_path, 'request.json'), 'r') as fh:
                 request = json.load(fh)
-            self.request_map.append((request, os.path.join(dir_path, 'result')))
+            self.request_map.append(
+                (request, os.path.join(dir_path, 'result'))
+            )
 
     def _get_result(self, request):
         for canned_request, canned_result in self.request_map:
@@ -67,6 +85,52 @@ def _get_result(self, request):
         raise KeyError('Request not recognized')
 
     def retrieve(self, dataset_name, params, file_path):
-        params_with_name = {**dict(_dataset_name=dataset_name),
-                            **params}
+        params_with_name = {**dict(_dataset_name=dataset_name), **params}
         shutil.copy2(self._get_result(params_with_name), file_path)
+
+
+class CDSClientWrapper:
+    def __init__(self, url=None, key=None):
+        self.session = _SessionMock()
+        self.real_client = cdsapi.Client()
+        self.url, self.key = _get_url_and_key(url, key)
+
+    def retrieve(self, dataset_name, params, file_path):
+        self.real_client.retrieve(dataset_name, params, file_path)
+
+
+def get_cds_client(dirname=None):
+    if _BEHAVIOUR is _Behaviour.MOCK:
+        # Use pre-generated response data for known requests.
+        return CDSClientMock
+
+    elif _BEHAVIOUR is _Behaviour.REAL_CLIENT:
+        # Use the real cdsapi client, but wrap it to ignore the passed-in
+        # credentials (which will probably be dummy values) and fall back
+        # to environment variables.
+        return CDSClientWrapper
+
+    elif _BEHAVIOUR is _Behaviour.SAVE_RESULTS:
+        # Wrap the real client and save the requests and responses
+        # for future mocking. As above, ignore passed-in credentials.
+        if dirname is None:
+            # Default directory name is name of calling function.
+            dirname = inspect.currentframe().f_back.f_code.co_name
+        resource_path = os.path.join(os.path.dirname(__file__), 'mock_results')
+        path = os.path.join(resource_path, dirname)
+
+        class ResultSavingClientWrapper(CDSClientWrapper):
+            def retrieve(self, dataset_name, params, file_path):
+                params_with_name = {
+                    **dict(_dataset_name=dataset_name),
+                    **params,
+                }
+                pathlib.Path(path).mkdir(parents=True, exist_ok=True)
+                with open(os.path.join(path, 'request.json'), 'w') as fh:
+                    json.dump(params_with_name, fh)
+                self.real_client.retrieve(dataset_name, params, file_path)
+                shutil.copy2(file_path, os.path.join(path, 'result'))
+
+        return ResultSavingClientWrapper
+    else:
+        raise Exception(f'Unknown behaviour {_BEHAVIOUR}')

test/test_era5.py

@@ -31,7 +31,7 @@
 
 import xcube
 import xcube.core
-from test.mocks import CDSClientMock
+from test.mocks import get_cds_client
 from xcube.core.store import DATASET_TYPE
 from xcube.core.store import VariableDescriptor
 from xcube_cds.store import CDSDataOpener
@@ -44,7 +44,7 @@
 class CDSEra5Test(unittest.TestCase):
 
     def test_open(self):
-        opener = CDSDataOpener(client_class=CDSClientMock,
+        opener = CDSDataOpener(client_class=get_cds_client(),
                                endpoint_url=_CDS_API_URL,
                                cds_api_key=_CDS_API_KEY)
         dataset = opener.open_data(
@@ -62,7 +62,7 @@ def test_open(self):
         self.assertEqual(10, len(dataset.variables['time']))
 
     def test_normalize_variable_names(self):
-        store = CDSDataStore(client_class=CDSClientMock, normalize_names=True,
+        store = CDSDataStore(client_class=get_cds_client(), normalize_names=True,
                              endpoint_url=_CDS_API_URL,
                              cds_api_key=_CDS_API_KEY)
         dataset = store.open_data(
@@ -90,7 +90,7 @@ def test_request_parameter_out_of_range(self):
             )
 
     def test_era5_land_monthly(self):
-        store = CDSDataStore(client_class=CDSClientMock,
+        store = CDSDataStore(client_class=get_cds_client(),
                              endpoint_url=_CDS_API_URL,
                              cds_api_key=_CDS_API_KEY)
         dataset = store.open_data(
@@ -106,7 +106,7 @@ def test_era5_land_monthly(self):
         self.assertTrue('u10' in dataset.variables)
 
     def test_era5_single_levels_hourly(self):
-        store = CDSDataStore(client_class=CDSClientMock,
+        store = CDSDataStore(client_class=get_cds_client(),
                              endpoint_url=_CDS_API_URL,
                              cds_api_key=_CDS_API_KEY)
         dataset = store.open_data(
@@ -123,7 +123,7 @@ def test_era5_single_levels_hourly(self):
         self.assertEqual(48, len(dataset.variables['time']))
 
     def test_era5_land_hourly(self):
-        store = CDSDataStore(client_class=CDSClientMock,
+        store = CDSDataStore(client_class=get_cds_client(),
                              endpoint_url=_CDS_API_URL,
                              cds_api_key=_CDS_API_KEY)
         dataset = store.open_data(
@@ -139,7 +139,7 @@ def test_era5_land_hourly(self):
         self.assertEqual(48, len(dataset.variables['time']))
 
     def test_era5_bounds(self):
-        opener = CDSDataOpener(client_class=CDSClientMock,
+        opener = CDSDataOpener(client_class=get_cds_client(),
                                endpoint_url=_CDS_API_URL,
                                cds_api_key=_CDS_API_KEY)
         dataset = opener.open_data(
@@ -176,7 +176,7 @@ def test_era5_open_data_empty_variables_list(self):
         self.assertEqual(361, len(dataset.variables['lon']))
 
     def test_open_data_null_variables_list(self):
-        store = CDSDataStore(client_class=CDSClientMock,
+        store = CDSDataStore(client_class=get_cds_client(),
                              endpoint_url=_CDS_API_URL,
                              cds_api_key=_CDS_API_KEY)
         data_id = 'reanalysis-era5-single-levels-monthly-means:' \

test/test_sea_ice_thickness.py

@@ -29,7 +29,7 @@
 from typing import Optional
 import unittest
 
-from test.mocks import CDSClientMock
+from test.mocks import get_cds_client
 from xcube_cds.store import CDSDataStore
 from xcube_cds.datasets.satellite_sea_ice_thickness import SeaIceThicknessHandler
 
@@ -174,7 +174,7 @@ class CdsSeaIceThicknessStoreTest(unittest.TestCase):
 
     def setUp(self) -> None:
         self.store = CDSDataStore(
-            client_class=CDSClientMock,
+            client_class=get_cds_client(dirname=self._testMethodName),
             endpoint_url=_CDS_API_URL,
             cds_api_key=_CDS_API_KEY
         )

test/test_soil_moisture.py

@@ -29,7 +29,7 @@
 import tempfile
 import unittest
 
-from test.mocks import CDSClientMock
+from test.mocks import get_cds_client
 from xcube_cds.store import CDSDataStore
 
 _CDS_API_URL = 'dummy'
@@ -40,7 +40,7 @@ class CDSSoilMoistureTest(unittest.TestCase):
 
     def test_soil_moisture_volumetric_minimal_params(self):
         store = CDSDataStore(
-            client_class=CDSClientMock,
+            client_class=get_cds_client(),
             endpoint_url=_CDS_API_URL,
             cds_api_key=_CDS_API_KEY
         )
@@ -61,7 +61,7 @@ def test_soil_moisture_volumetric_minimal_params(self):
 
     def test_soil_moisture_volumetric_monthly_2_years(self):
         store = CDSDataStore(
-            client_class=CDSClientMock,
+            client_class=get_cds_client(),
             endpoint_url=_CDS_API_URL,
             cds_api_key=_CDS_API_KEY
         )
@@ -83,7 +83,7 @@ def test_soil_moisture_volumetric_monthly_2_years(self):
 
     def test_soil_moisture_saturation_daily(self):
         store = CDSDataStore(
-            client_class=CDSClientMock,
+            client_class=get_cds_client(),
             endpoint_url=_CDS_API_URL,
             cds_api_key=_CDS_API_KEY
         )
@@ -97,15 +97,15 @@ def test_soil_moisture_saturation_daily(self):
         self.assertEqual(4, len(dataset.variables['time']))
         self.assertEqual('19910805T000000Z',
                          dataset.attrs['time_coverage_start'])
-        self.assertEqual('20191231T235959Z',
+        self.assertEqual('20201231T235959Z',
                          dataset.attrs['time_coverage_end'])
         description = store.describe_data(data_id)
         self.assertCountEqual(description.data_vars.keys(),
                               map(str, dataset.data_vars))
 
     def test_soil_moisture_saturation_10_day(self):
         store = CDSDataStore(
-            client_class=CDSClientMock,
+            client_class=get_cds_client(),
             endpoint_url=_CDS_API_URL,
             cds_api_key=_CDS_API_KEY
         )
@@ -127,7 +127,7 @@ def test_soil_moisture_saturation_10_day(self):
 
     def test_soil_moisture_volumetric_optional_params(self):
         store = CDSDataStore(
-            client_class=CDSClientMock,
+            client_class=get_cds_client(),
             endpoint_url=_CDS_API_URL,
             cds_api_key=_CDS_API_KEY
         )
@@ -161,7 +161,7 @@ def test_soil_moisture_empty_variables_list(self):
         self.assertEqual(1441, len(dataset.variables['lon']))
 
     def test_copy_on_open(self):
-        store = CDSDataStore(client_class=CDSClientMock,
+        store = CDSDataStore(client_class=get_cds_client(),
                              endpoint_url=_CDS_API_URL,
                              cds_api_key=_CDS_API_KEY)
         data_id = 'satellite-soil-moisture:volumetric:monthly'
@@ -182,7 +182,7 @@ def test_copy_on_open(self):
             self.assertTrue(os.path.isdir(zarr_path))
 
     def test_soil_moisture_get_open_params_schema(self):
-        store = CDSDataStore(client_class=CDSClientMock,
+        store = CDSDataStore(client_class=get_cds_client(),
                              endpoint_url=_CDS_API_URL,
                              cds_api_key=_CDS_API_KEY)
         data_id = 'satellite-soil-moisture:volumetric:monthly'