From ad986a021b479eaab5ebf009567f807b59ecc547 Mon Sep 17 00:00:00 2001 From: Wolfgang Preimesberger Date: Mon, 21 Oct 2024 16:54:14 +0200 Subject: [PATCH] Update for new API (#31) * Update for new API * Update readme * Update ci * Fix test * Update test --- .github/workflows/ci.yml | 2 ++ CHANGELOG.rst | 4 ++++ README.rst | 49 ++++++++++++++++++++++++++++++++++++++-- docs/pages/download.md | 14 ++++++++---- setup.cfg | 2 +- src/c3s_sm/cli.py | 31 ++++++++----------------- src/c3s_sm/const.py | 28 +++++++++++------------ src/c3s_sm/download.py | 8 ++++--- src/c3s_sm/misc.py | 12 ++++++++++ tests/test_download.py | 12 +++++++--- 10 files changed, 113 insertions(+), 49 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d5e190b..62bf3a4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,6 +67,8 @@ jobs: fi pip install -e .[testing] - name: Run all tests + env: + CDS_APIKEY: ${{ secrets.CDS_APIKEY }} shell: bash -l {0} run: | pytest diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3df12a5..5df986b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,10 @@ Unreleased Changes - +Version 0.3.1 +============= +- Fixing CDS API access (new token and API url) + Version 0.3.0 ============= - Added CLI module diff --git a/README.rst b/README.rst index 18273f6..8441929 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,6 @@ -============ +====== c3s_sm -============ +====== |ci| |cov| |pip| |doc| @@ -36,6 +36,47 @@ might be necessary to install the pykdtree package from conda-forge conda install -c conda-forge pykdtree +API Key +------- +In order to download C3S soil moisture data from CDS, this package uses the +CDS API (https://pypi.org/project/cdsapi/). You can +either pass your credentials directly on the command line (which might be +unsafe) or set up a `.cdsapirc` file in your home directory (recommended). +Please see the description at https://cds.climate.copernicus.eu/how-to-api. + +Quickstart +========== +Download image data from CDS using the c3s_sm shell command + +.. code-block:: shell + + c3s_sm download /tmp/c3s/img -s 2023-09-01 -e 2023-10-31 -v v202212 + +... and convert them to time series + +.. code-block:: shell + + c3s_sm reshuffle /tmp/c3s/img /tmp/c3s/ts + +Finally, in python, read the time series data for a location as pandas +DataFrame. + +.. code-block:: python + + >> from c3s_sm.interface import C3STs + >> ds = C3STs('/tmp/c3s/ts') + >> ts = ds.read(18, 48) + + sm sm_uncertainty flag ... mode sensor t0 + 2023-09-01 0.222125 0.014661 0 ... 2 544 19601.100348 + 2023-09-02 0.213480 0.011166 0 ... 3 38432 19602.051628 + 2023-09-03 0.197324 0.014661 0 ... 3 33312 19602.945730 + ... ... ... ... ... ... ... + 2023-10-29 0.265275 0.013192 0 ... 3 37408 19658.955236 + 2023-10-30 0.256964 0.011166 0 ... 3 38432 19660.085144 + 2023-10-31 0.241187 0.014661 0 ... 3 33312 19660.945730 + + Tutorials ========= @@ -56,6 +97,10 @@ with a spatial sampling of 0.25 degrees. Build Docker image ================== + +For operational implementations, this package and be installed in a +docker container. + - Check out the repo at the branch/tag/commit you want build - Make sure you have docker installed and run the command (replace the tag `latest` with something more meaningful, e.g. a matching version number) diff --git a/docs/pages/download.md b/docs/pages/download.md index e0fbb7d..6871d33 100644 --- a/docs/pages/download.md +++ b/docs/pages/download.md @@ -16,13 +16,13 @@ since the last update (e.g. you can set up a cron job to keep your records up-to-date) Before any of the 2 scripts can be used, you must provide your CDS API key. -Follow this guide: https://cds.climate.copernicus.eu/api-how-to#install-the-cds-api-key +Follow this guide: https://cds.climate.copernicus.eu/how-to-api Make sure that - On Linux: You have your credentials stored in `$HOME/.cdsapirc` -- On Windows: Your have your credentials stored in `%USERPROFILE%\.cdsapirc`, +- On Windows: You have your credentials stored in `%USERPROFILE%\.cdsapirc`, %USERPROFILE% is usually located at C:\Users\Username folder -- On MacOS: Your have your credentials stored in `~/.cdsapirc` +- On MacOS: Your have your credentials stored in `/Users//.cdsapirc` Alternatively you can pass your UID and API Key (that you get from your CDS profile page) directly with the download command (but the .cdsapirc option @@ -40,7 +40,7 @@ Example command to download the daily passive product v202212 in the period from E.g. ```console c3s_sm download /target/path -s 2019-05-01 -e 2019-05-10 --product passive ---freq daily -v v202212 --cds_token XXXX:xxxx-xxxxxx-xxxx-xxxx +--freq daily -v v202212 --cds_token xxxxxxx-xxxx-xxxx-xxxxxxxxxx ``` `--product` can be one of `active`, `combined` or `passive`. `--freq` is either @@ -50,6 +50,8 @@ page. This will create a subfolder for each year in the target directory and store downloaded images there. +Note: You don't have to provide your token if you have set up a .cdsapirc file. + ``` /target/path/ ├── 2019/ @@ -69,9 +71,11 @@ archive. E.g. ```console -c3s_sm update_img /target/path --cds_token XXXX:xxxx-xxxxxx-xxxx-xxxx +c3s_sm update_img /target/path --cds_token xxxxxxx-xxxx-xxxx-xxxxxxxxxx ``` requires that some (previously downloaded) files are available in /target/path. It will then check for matching new data online and download those. +Note: You don't have to provide your token if you have set up a .cdsapirc file. + diff --git a/setup.cfg b/setup.cfg index 148c2b4..b83ccc2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -44,7 +44,7 @@ install_requires = smecv_grid>=0.3 more_itertools click - cdsapi>=0.7.0 + cdsapi>=0.7.3 # The usage of test_requires is discouraged, see `Dependency Management` docs # tests_require = pytest; pytest-cov diff --git a/src/c3s_sm/cli.py b/src/c3s_sm/cli.py index dc40e81..2a67c8f 100644 --- a/src/c3s_sm/cli.py +++ b/src/c3s_sm/cli.py @@ -61,9 +61,9 @@ "--cds_token", type=click.STRING, default=None, - help="To identify with the CDS, required if no .cdsapi file exists. " - "Consists of your UID and API Key . Both can be " - "found on your CDS User profile page.") + help="To identify with the CDS. Required if no .cdsapirc file exists in " + "the home directory (see documentation). You can find your token/key " + "on your CDS user profile page.") def cli_download(path, startdate, enddate, @@ -75,7 +75,7 @@ def cli_download(path, """ Download C3S SM data within a chosen period. NOTE: Before using this program, create a CDS account and set up a `.cdsapirc` file as described - here: https://cds.climate.copernicus.eu/api-how-to + here: https://cds.climate.copernicus.eu/how-to-api \b Required Parameters @@ -83,18 +83,13 @@ def cli_download(path, PATH: string (required) Path where the downloaded C3S SM images are stored. Make sure to set up the CDS API for your account as describe in - https://cds.climate.copernicus.eu/api-how-to + https://cds.climate.copernicus.eu/how-to-api """ # The docstring above is slightly different to the normal python one to # display it properly on the command line. - url = os.environ.get('CDSAPI_URL', - "https://cds.climate.copernicus.eu/api/v2") - os.environ['CDSAPI_URL'] = url - if cds_token is not None: os.environ["CDSAPI_KEY"] = cds_token - check_api_read() startdate = pd.to_datetime(startdate) @@ -130,10 +125,9 @@ def cli_download(path, "--cds_token", type=click.STRING, default=None, - help="To identify with the CDS. Required if no .cdsapi file exists. " - "In the home directory (see documentation)." - "Consists of your UID and API Key . Both can be " - "found under your CDS User profile page.") + help="To identify with the CDS. Required if no .cdsapirc file exists in " + "the home directory (see documentation). You can find your token/key " + "on your CDS user profile page.") def cli_update_img(path, fntempl, cds_token=None): """ Extend a locally existing C3S SM record by downloading new files that @@ -149,18 +143,13 @@ def cli_update_img(path, fntempl, cds_token=None): PATH: string Path where previously downloaded C3S SM images are stored. Make sure to set up the CDS API for your account as describe in - https://cds.climate.copernicus.eu/api-how-to + https://cds.climate.copernicus.eu/how-to-api """ # The docstring above is slightly different to the normal python one to # display it properly on the command line. - # if not set, use URL from const - if 'CDSAPI_URL' not in os.environ: - os.environ['CDSAPI_URL'] = cds_api_url - if cds_token is not None: os.environ["CDSAPI_KEY"] = cds_token - check_api_read() props = img_infer_file_props(path, fntempl=fntempl, start_from='last') @@ -296,7 +285,7 @@ def cli_reshuffle(img_path, ts_path, startdate, enddate, parameters, land, if startdate is None: startdate = get_first_image_date(img_path, fntempl) if enddate is None: - enddate = get_last_image_date(ts_path, fntempl) + enddate = get_last_image_date(img_path, fntempl) startdate = pd.to_datetime(startdate) enddate = pd.to_datetime(enddate) diff --git a/src/c3s_sm/const.py b/src/c3s_sm/const.py index e1dd699..a011404 100644 --- a/src/c3s_sm/const.py +++ b/src/c3s_sm/const.py @@ -3,7 +3,8 @@ from pathlib import Path from datetime import datetime -cds_api_url = "https://cds.climate.copernicus.eu/api/v2" +# This can be overridden in the .cdsapirc file +cds_api_url = "https://cds.climate.copernicus.eu/api" # CDSAPI_RC variable must be set or we use home dir dotrc = os.environ.get('CDSAPI_RC', os.path.join(Path.home(), '.cdsapirc')) @@ -11,25 +12,24 @@ def check_api_read() -> bool: if not os.path.isfile(dotrc): - url = os.environ.get('CDSAPI_URL') key = os.environ.get('CDSAPI_KEY') - if url is None or key is None: - ValueError('CDS API KEY or .cdsapirc file not found, ' - 'download will not work! ' - 'Please create a .cdsapirc file with your credentials' - 'or pass your uid/key to the command line tool ' - 'See: ' - 'https://cds.climate.copernicus.eu/api-how-to') - api_ready = False - elif ":" not in key: + if "CDSAPI_URL" not in os.environ: + os.environ['CDSAPI_URL'] = cds_api_url + + if key is None: raise ValueError( - 'Your CDS token is not valid. It must be in the format ' - ':, both of which are found on your CDS' - 'profile page.') + 'Neither CDSAPI_KEY variable nor .cdsapirc file found, ' + 'download will not work! ' + 'Please create a .cdsapirc file with your API key. ' + 'See: https://cds.climate.copernicus.eu/how-to-api' + ) else: api_ready = True else: + if "CDSAPI_URL" in os.environ: + os.environ.pop("CDSAPI_URL") # Use URL from file api_ready = True + return api_ready diff --git a/src/c3s_sm/download.py b/src/c3s_sm/download.py index 8be027e..a46f570 100644 --- a/src/c3s_sm/download.py +++ b/src/c3s_sm/download.py @@ -16,7 +16,7 @@ import traceback from c3s_sm.const import variable_lut, freq_lut, check_api_read -from c3s_sm.misc import update_image_summary_file +from c3s_sm.misc import update_image_summary_file, delete_empty_directories def logger(fname, level=logging.DEBUG, verbose=False): @@ -97,7 +97,7 @@ def download_c3ssm(c, raise ValueError( "Cannot establish connection to CDS. Please set up" "your CDS API key as described at " - "https://cds.climate.copernicus.eu/api-how-to") + "https://cds.climate.copernicus.eu/how-to-api") os.makedirs(target_dir, exist_ok=True) @@ -321,6 +321,8 @@ def download_and_extract(target_path, logger_name='dl_logger', show_progress_bars=True) + delete_empty_directories(target_path) + try: update_image_summary_file(target_path) except ValueError as _: @@ -361,4 +363,4 @@ def first_missing_date(last_date: str, freq: str = 'daily') -> datetime: next_date = last_date + relativedelta(months=1) next_date = datetime(next_date.year, next_date.month, 1) - return next_date + return next_date \ No newline at end of file diff --git a/src/c3s_sm/misc.py b/src/c3s_sm/misc.py index bca2016..3c7c897 100644 --- a/src/c3s_sm/misc.py +++ b/src/c3s_sm/misc.py @@ -256,3 +256,15 @@ def update_ts_summary_file(data_path, props=None, collect_cov=False, **kwargs): yaml.dump(props, f, default_flow_style=False, sort_keys=False) return props + + +def delete_empty_directories(path: str): + """ + Delete empty dirs in path + """ + for root, dirs, files in os.walk(path, topdown=False): + for dir_name in dirs: + dir_path = os.path.join(root, dir_name) + # Check if the directory is empty + if not os.listdir(dir_path): + os.rmdir(dir_path) diff --git a/tests/test_download.py b/tests/test_download.py index 3b043e6..11b8f15 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -5,6 +5,7 @@ import subprocess from c3s_sm.download import download_and_extract from c3s_sm.misc import read_summary_yml +from c3s_sm.const import dotrc def test_download_dry_run(): with TemporaryDirectory() as outpath: @@ -31,9 +32,10 @@ def test_download_dry_run(): assert queries[1]['icdr']['request']['day'] == ['01', '02', '03', '04', '05'] assert queries[1]['icdr']['request']['version'] == 'v202212' - -@pytest.mark.skipif("CDS_APIKEY" not in os.environ, - reason="No environment variable CDS_APIKEY key found") +@pytest.mark.skipif(("CDS_APIKEY" not in os.environ) and not os.path.exists(dotrc), + # To run this test on Github, the CDS_APIKEY env secret must be set (also in ci.yml!) + reason="No environment variable CDS_APIKEY or " + ".cdsapirc file found.") def test_download_with_token(): with TemporaryDirectory() as outpath: args = [outpath] \ @@ -42,6 +44,10 @@ def test_download_with_token(): + ['--product', 'combined'] \ + ['--freq', 'monthly'] \ + ['--version', 'v202212'] + + if not os.path.exists(dotrc): + args += ['--cds_token', os.environ['CDS_APIKEY']] + subprocess.call(['c3s_sm', 'download', *args]) files = os.listdir(os.path.join(outpath, '2022')) assert len(files) == 2