Update for new API (#31)

* Update for new API

* Update readme

* Update ci

* Fix test

* Update test

.github/workflows/ci.yml

@@ -67,6 +67,8 @@ jobs:
           fi
           pip install -e .[testing]
       - name: Run all tests
+        env:
+          CDS_APIKEY: ${{ secrets.CDS_APIKEY }}
         shell: bash -l {0}
         run: |      
           pytest

CHANGELOG.rst

@@ -7,6 +7,10 @@ Unreleased Changes
 
 -
 
+Version 0.3.1
+=============
+- Fixing CDS API access (new token and API url)
+
 Version 0.3.0
 =============
 - Added CLI module

README.rst

@@ -1,6 +1,6 @@
-============
+======
 c3s_sm
-============
+======
 
 |ci| |cov| |pip| |doc|
 
@@ -36,6 +36,47 @@ might be necessary to install the pykdtree package from conda-forge
 
     conda install -c conda-forge pykdtree
 
+API Key
+-------
+In order to download C3S soil moisture data from CDS, this package uses the
+CDS API (https://pypi.org/project/cdsapi/). You can
+either pass your credentials directly on the command line (which might be
+unsafe) or set up a `.cdsapirc` file in your home directory (recommended).
+Please see the description at https://cds.climate.copernicus.eu/how-to-api.
+
+Quickstart
+==========
+Download image data from CDS using the c3s_sm shell command
+
+.. code-block:: shell
+
+    c3s_sm download /tmp/c3s/img -s 2023-09-01 -e 2023-10-31 -v v202212
+
+... and convert them to time series
+
+.. code-block:: shell
+
+    c3s_sm reshuffle /tmp/c3s/img /tmp/c3s/ts
+
+Finally, in python, read the time series data for a location as pandas
+DataFrame.
+
+.. code-block:: python
+
+    >> from c3s_sm.interface import C3STs
+    >> ds = C3STs('/tmp/c3s/ts')
+    >> ts = ds.read(18, 48)
+
+                      sm  sm_uncertainty  flag  ...  mode  sensor            t0
+    2023-09-01  0.222125        0.014661     0  ...     2     544  19601.100348
+    2023-09-02  0.213480        0.011166     0  ...     3   38432  19602.051628
+    2023-09-03  0.197324        0.014661     0  ...     3   33312  19602.945730
+                  ...             ...   ...  ...   ...     ...           ...
+    2023-10-29  0.265275        0.013192     0  ...     3   37408  19658.955236
+    2023-10-30  0.256964        0.011166     0  ...     3   38432  19660.085144
+    2023-10-31  0.241187        0.014661     0  ...     3   33312  19660.945730
+
+
 Tutorials
 =========
 
@@ -56,6 +97,10 @@ with a spatial sampling of 0.25 degrees.
 
 Build Docker image
 ==================
+
+For operational implementations, this package and be installed in a
+docker container.
+
 - Check out the repo at the branch/tag/commit you want build
 - Make sure you have docker installed and run the command (replace the tag `latest`
   with something more meaningful, e.g. a matching version number)

docs/pages/download.md

@@ -16,13 +16,13 @@ since the last update (e.g. you can set up a cron job to keep your records
 up-to-date)
 
 Before any of the 2 scripts can be used, you must provide your CDS API key. 
-Follow this guide: https://cds.climate.copernicus.eu/api-how-to#install-the-cds-api-key
+Follow this guide: https://cds.climate.copernicus.eu/how-to-api
 
 Make sure that 
 - On Linux: You have your credentials stored in `$HOME/.cdsapirc`
-- On Windows: Your have your credentials stored in `%USERPROFILE%\.cdsapirc`,
+- On Windows: You have your credentials stored in `%USERPROFILE%\.cdsapirc`,
 %USERPROFILE% is usually located at C:\Users\Username folder
-- On MacOS: Your have your credentials stored in `~/.cdsapirc`
+- On MacOS: Your have your credentials stored in `/Users/<USERNAME>/.cdsapirc`
 
 Alternatively you can pass your UID and API Key (that you get from your CDS
 profile page) directly with the download command (but the .cdsapirc option
@@ -40,7 +40,7 @@ Example command to download the daily passive product v202212 in the period from
 E.g.
 ```console
 c3s_sm download /target/path -s 2019-05-01 -e 2019-05-10 --product passive 
---freq daily -v v202212 --cds_token XXXX:xxxx-xxxxxx-xxxx-xxxx
+--freq daily -v v202212 --cds_token xxxxxxx-xxxx-xxxx-xxxxxxxxxx
 ```
 
 `--product` can be one of `active`, `combined` or `passive`. `--freq` is either
@@ -50,6 +50,8 @@ page.
 This will create a subfolder for each year in the target directory and store 
 downloaded images there.
 
+Note: You don't have to provide your token if you have set up a .cdsapirc file.
+
 ```
 /target/path/
 ├── 2019/
@@ -69,9 +71,11 @@ archive.
 
 E.g.
 ```console
-c3s_sm update_img /target/path --cds_token XXXX:xxxx-xxxxxx-xxxx-xxxx
+c3s_sm update_img /target/path --cds_token xxxxxxx-xxxx-xxxx-xxxxxxxxxx
 ```
 
 requires that some (previously downloaded) files are available in /target/path.
 It will then check for matching new data online and download those.
 
+Note: You don't have to provide your token if you have set up a .cdsapirc file.
+

setup.cfg

@@ -44,7 +44,7 @@ install_requires =
     smecv_grid>=0.3
     more_itertools
     click
-    cdsapi>=0.7.0
+    cdsapi>=0.7.3
 
 # The usage of test_requires is discouraged, see `Dependency Management` docs
 # tests_require = pytest; pytest-cov

src/c3s_sm/cli.py

@@ -61,9 +61,9 @@
     "--cds_token",
     type=click.STRING,
     default=None,
-    help="To identify with the CDS, required if no .cdsapi file exists. "
-    "Consists of your UID and API Key <UID:APIKEY>. Both can be "
-    "found on your CDS User profile page.")
+    help="To identify with the CDS. Required if no .cdsapirc file exists in "
+         "the home directory (see documentation). You can find your token/key "
+         "on your CDS user profile page.")
 def cli_download(path,
                  startdate,
                  enddate,
@@ -75,26 +75,21 @@ def cli_download(path,
     """
     Download C3S SM data within a chosen period. NOTE: Before using this
     program, create a CDS account and set up a `.cdsapirc` file as described
-    here: https://cds.climate.copernicus.eu/api-how-to
+    here: https://cds.climate.copernicus.eu/how-to-api
 
     \b
     Required Parameters
     -------------------
     PATH: string (required)
         Path where the downloaded C3S SM images are stored.
         Make sure to set up the CDS API for your account as describe in
-        https://cds.climate.copernicus.eu/api-how-to
+        https://cds.climate.copernicus.eu/how-to-api
     """
     # The docstring above is slightly different to the normal python one to
     # display it properly on the command line.
 
-    url = os.environ.get('CDSAPI_URL',
-                         "https://cds.climate.copernicus.eu/api/v2")
-    os.environ['CDSAPI_URL'] = url
-
     if cds_token is not None:
         os.environ["CDSAPI_KEY"] = cds_token
-
     check_api_read()
 
     startdate = pd.to_datetime(startdate)
@@ -130,10 +125,9 @@ def cli_download(path,
     "--cds_token",
     type=click.STRING,
     default=None,
-    help="To identify with the CDS. Required if no .cdsapi file exists. "
-    "In the home directory (see documentation)."
-    "Consists of your UID and API Key <UID:APIKEY>. Both can be "
-    "found under your CDS User profile page.")
+    help="To identify with the CDS. Required if no .cdsapirc file exists in "
+         "the home directory (see documentation). You can find your token/key "
+         "on your CDS user profile page.")
 def cli_update_img(path, fntempl, cds_token=None):
     """
     Extend a locally existing C3S SM record by downloading new files that
@@ -149,18 +143,13 @@ def cli_update_img(path, fntempl, cds_token=None):
     PATH: string
         Path where previously downloaded C3S SM images are stored.
         Make sure to set up the CDS API for your account as describe in
-        https://cds.climate.copernicus.eu/api-how-to
+        https://cds.climate.copernicus.eu/how-to-api
     """
     # The docstring above is slightly different to the normal python one to
     # display it properly on the command line.
 
-    # if not set, use URL from const
-    if 'CDSAPI_URL' not in os.environ:
-        os.environ['CDSAPI_URL'] = cds_api_url
-
     if cds_token is not None:
         os.environ["CDSAPI_KEY"] = cds_token
-
     check_api_read()
 
     props = img_infer_file_props(path, fntempl=fntempl, start_from='last')
@@ -296,7 +285,7 @@ def cli_reshuffle(img_path, ts_path, startdate, enddate, parameters, land,
     if startdate is None:
         startdate = get_first_image_date(img_path, fntempl)
     if enddate is None:
-        enddate = get_last_image_date(ts_path, fntempl)
+        enddate = get_last_image_date(img_path, fntempl)
 
     startdate = pd.to_datetime(startdate)
     enddate = pd.to_datetime(enddate)

src/c3s_sm/const.py

@@ -3,33 +3,33 @@
 from pathlib import Path
 from datetime import datetime
 
-cds_api_url = "https://cds.climate.copernicus.eu/api/v2"
+# This can be overridden in the .cdsapirc file
+cds_api_url = "https://cds.climate.copernicus.eu/api"
 
 # CDSAPI_RC variable must be set or we use home dir
 dotrc = os.environ.get('CDSAPI_RC', os.path.join(Path.home(), '.cdsapirc'))
 
 
 def check_api_read() -> bool:
     if not os.path.isfile(dotrc):
-        url = os.environ.get('CDSAPI_URL')
         key = os.environ.get('CDSAPI_KEY')
-        if url is None or key is None:
-            ValueError('CDS API KEY or .cdsapirc file not found, '
-                       'download will not work! '
-                       'Please create a .cdsapirc file with your credentials'
-                       'or pass your uid/key to the command line tool '
-                       'See: '
-                       'https://cds.climate.copernicus.eu/api-how-to')
-            api_ready = False
-        elif ":" not in key:
+        if "CDSAPI_URL" not in os.environ:
+            os.environ['CDSAPI_URL'] = cds_api_url
+
+        if key is None:
             raise ValueError(
-                'Your CDS token is not valid. It must be in the format '
-                '<UID>:<APIKEY>, both of which are found on your CDS'
-                'profile page.')
+                'Neither CDSAPI_KEY variable nor .cdsapirc file found, '
+                'download will not work! '
+                'Please create a .cdsapirc file with your API key. '
+                'See: https://cds.climate.copernicus.eu/how-to-api'
+            )
         else:
             api_ready = True
     else:
+        if "CDSAPI_URL" in os.environ:
+            os.environ.pop("CDSAPI_URL")   # Use URL from file
         api_ready = True
+
     return api_ready
 
 

src/c3s_sm/download.py

@@ -16,7 +16,7 @@
 import traceback
 
 from c3s_sm.const import variable_lut, freq_lut, check_api_read
-from c3s_sm.misc import update_image_summary_file
+from c3s_sm.misc import update_image_summary_file, delete_empty_directories
 
 
 def logger(fname, level=logging.DEBUG, verbose=False):
@@ -97,7 +97,7 @@ def download_c3ssm(c,
             raise ValueError(
                 "Cannot establish connection to CDS. Please set up"
                 "your CDS API key as described at "
-                "https://cds.climate.copernicus.eu/api-how-to")
+                "https://cds.climate.copernicus.eu/how-to-api")
 
         os.makedirs(target_dir, exist_ok=True)
 
@@ -321,6 +321,8 @@ def download_and_extract(target_path,
         logger_name='dl_logger',
         show_progress_bars=True)
 
+    delete_empty_directories(target_path)
+
     try:
         update_image_summary_file(target_path)
     except ValueError as _:
@@ -361,4 +363,4 @@ def first_missing_date(last_date: str, freq: str = 'daily') -> datetime:
             next_date = last_date + relativedelta(months=1)
             next_date = datetime(next_date.year, next_date.month, 1)
 
-    return next_date
+    return next_date

src/c3s_sm/misc.py

@@ -256,3 +256,15 @@ def update_ts_summary_file(data_path, props=None, collect_cov=False, **kwargs):
         yaml.dump(props, f, default_flow_style=False, sort_keys=False)
 
     return props
+
+
+def delete_empty_directories(path: str):
+    """
+    Delete empty dirs in path
+    """
+    for root, dirs, files in os.walk(path, topdown=False):
+        for dir_name in dirs:
+            dir_path = os.path.join(root, dir_name)
+            # Check if the directory is empty
+            if not os.listdir(dir_path):
+                os.rmdir(dir_path)

tests/test_download.py

@@ -5,6 +5,7 @@
 import subprocess
 from c3s_sm.download import download_and_extract
 from c3s_sm.misc import read_summary_yml
+from c3s_sm.const import dotrc
 
 def test_download_dry_run():
     with TemporaryDirectory() as outpath:
@@ -31,9 +32,10 @@ def test_download_dry_run():
         assert queries[1]['icdr']['request']['day'] == ['01', '02', '03', '04', '05']
         assert queries[1]['icdr']['request']['version'] == 'v202212'
 
-
-@pytest.mark.skipif("CDS_APIKEY" not in os.environ,
-                    reason="No environment variable CDS_APIKEY key found")
+@pytest.mark.skipif(("CDS_APIKEY" not in os.environ) and not os.path.exists(dotrc),
+                    # To run this test on Github, the CDS_APIKEY env secret must be set (also in ci.yml!)
+                    reason="No environment variable CDS_APIKEY or "
+                           ".cdsapirc file found.")
 def test_download_with_token():
     with TemporaryDirectory() as outpath:
         args = [outpath] \
@@ -42,6 +44,10 @@ def test_download_with_token():
                + ['--product', 'combined'] \
                + ['--freq', 'monthly'] \
                + ['--version', 'v202212']
+
+        if not os.path.exists(dotrc):
+           args += ['--cds_token', os.environ['CDS_APIKEY']]
+
         subprocess.call(['c3s_sm', 'download', *args])
         files = os.listdir(os.path.join(outpath, '2022'))
         assert len(files) == 2