CMORizer for JRA-55 (#3141)

Co-authored-by: Rémi Kazeroni <[email protected]> Co-authored-by: Rémi Kazeroni <[email protected]> Co-authored-by: Romain Beucher <[email protected]> Co-authored-by: Felicity Chun <[email protected]>
ESMValGroup · Oct 1, 2024 · 6155acf · 6155acf
1 parent ba94011
commit 6155acf
Show file tree

Hide file tree

Showing 11 changed files with 456 additions and 0 deletions.
diff --git a/doc/sphinx/source/input.rst b/doc/sphinx/source/input.rst
@@ -366,6 +366,8 @@ A list of the datasets for which a CMORizers is available is provided in the fol
 +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+
 | JRA-25                       | clt, hus, prw, rlut, rlutcs, rsut, rsutcs (Amon)                                                     |   2  | Python          |
 +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+
+| JRA-55                       | cli, clivi, clw, clwvi, clt, prw, rlus, rlut, rlutcs, rsus, rsuscs, rsut, rsutcs, ta, tas, wap (Amon)|   2  | Python          |
++------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+
 | Kadow2020                    | tasa (Amon)                                                                                          |   2  | Python          |
 +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+
 | LAI3g                        | lai (Lmon)                                                                                           |   3  | Python          |

diff --git a/environment.yml b/environment.yml
@@ -14,6 +14,7 @@ dependencies:
   - cdo >=2.3.0
   - cdsapi
   - cf-units
+  - cfgrib
   - cftime
   - cmocean
   - curl <8.10

diff --git a/environment_osx.yml b/environment_osx.yml
@@ -14,6 +14,7 @@ dependencies:
   - cdo >=2.3.0
   - cdsapi
   - cf-units
+  - cfgrib
   - cftime
   - cmocean
   - cython

diff --git a/esmvaltool/cmorizers/data/cmor_config/JRA-55.yml b/esmvaltool/cmorizers/data/cmor_config/JRA-55.yml
@@ -0,0 +1,103 @@
+---
+# Common global attributes for Cmorizer output
+attributes:
+  dataset_id: JRA-55
+  version: '1'
+  tier: 2
+  modeling_realm: reanaly
+  project_id: OBS6
+  source: 'https://rda.ucar.edu/datasets/ds628.1/'
+  reference: 'jra_55'
+  comment: |
+    '' 
+
+# Variables to cmorize
+variables:
+  cli:
+    short_name: cli
+    mip: Amon
+    file: fcst_p125.229_ciwc.{year}01_{year}12.grb
+
+  clivi:
+    short_name: clivi
+    mip: Amon
+    file: fcst_column125.058_cice.{year}01_{year}12.grb
+
+  clw:
+    short_name: clw
+    mip: Amon
+    file: fcst_p125.228_clwc.{year}01_{year}12.grb
+
+  clwvi:
+    short_name: clwvi
+    mip: Amon
+    operator: sum
+    files:
+      - 'fcst_column125.058_cice.{year}01_{year}12.grb'
+      - 'fcst_column125.227_cw.{year}01_{year}12.grb'
+
+  clt:
+    short_name: clt
+    mip: Amon
+    file: fcst_surf125.071_tcdc.{year}01_{year}12.grb
+
+  prw:
+    short_name: prw
+    mip: Amon
+    file: fcst_column125.054_pwat.{year}01_{year}12.grb
+
+  rlus:
+    short_name: rlus
+    mip: Amon
+    typeOfLevel: surface
+    file: fcst_phy2m125.212_ulwrf.{year}01_{year}12.grb
+
+  rlut:
+    short_name: rlut
+    mip: Amon
+    typeOfLevel: nominalTop
+    file: fcst_phy2m125.212_ulwrf.{year}01_{year}12.grb
+
+  rlutcs:
+    short_name: rlutcs
+    mip: Amon
+    file: fcst_phy2m125.162_csulf.{year}01_{year}12.grb
+
+  rsus:
+    short_name: rsus
+    mip: Amon
+    typeOfLevel: surface
+    file: fcst_phy2m125.211_uswrf.{year}01_{year}12.grb
+
+  rsuscs:
+    short_name: rsuscs
+    mip: Amon
+    typeOfLevel: surface
+    file: fcst_phy2m125.160_csusf.{year}01_{year}12.grb
+
+  rsut:
+    short_name: rsut
+    mip: Amon
+    typeOfLevel: nominalTop
+    file: fcst_phy2m125.211_uswrf.{year}01_{year}12.grb
+
+  rsutcs:
+    short_name: rsutcs
+    mip: Amon
+    typeOfLevel: nominalTop
+    file: fcst_phy2m125.160_csusf.{year}01_{year}12.grb
+
+  ta:
+    short_name: ta
+    mip: Amon
+    file: anl_p125.011_tmp.{year}01_{year}12.grb
+
+  tas:
+    short_name: tas
+    mip: Amon
+    file: anl_surf125.011_tmp.{year}01_{year}12.grb
+
+  wap:
+    short_name: wap
+    mip: Amon
+    file: anl_p125.039_vvel.{year}01_{year}12.grb
diff --git a/esmvaltool/cmorizers/data/datasets.yml b/esmvaltool/cmorizers/data/datasets.yml
@@ -808,6 +808,15 @@ datasets:
         mon/atmos/rsut/rsut_Amon_reanalysis_JRA-25_197901-201312.nc
         mon/atmos/rsutcs/rsutcs_Amon_reanalysis_JRA-25_197901-201312.nc
 
+  JRA-55:
+    tier: 2
+    source: https://rda.ucar.edu/datasets/ds628.1/
+    last_access: 2023-03-22
+    info: |
+      Create an account on the research data archive (RDA) in order to be able
+      to download the data (1.25 degree, pressure levels). See
+      https://rda.ucar.edu/login/register/ for more details.
+
   Kadow2020:
     tier: 2
     source: http://users.met.fu-berlin.de/~ChristopherKadow/

diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/jra_55.py b/esmvaltool/cmorizers/data/downloaders/datasets/jra_55.py
@@ -0,0 +1,115 @@
+"""Script to download JRA-55 from RDA."""
+import logging
+import os
+
+from datetime import datetime
+
+from dateutil import relativedelta
+
+from esmvaltool.cmorizers.data.downloaders.wget import WGetDownloader
+
+
+logger = logging.getLogger(__name__)
+
+
+def download_dataset(config, dataset, dataset_info, start_date, end_date,
+                     overwrite):
+    """Download dataset.
+
+    Parameters
+    ----------
+    config : dict
+        ESMValTool's user configuration
+    dataset : str
+        Name of the dataset
+    dataset_info : dict
+         Dataset information from the datasets.yml file
+    start_date : datetime
+        Start of the interval to download
+    end_date : datetime
+        End of the interval to download
+    overwrite : bool
+        Overwrite already downloaded files
+    """
+    downloader = WGetDownloader(
+        config=config,
+        dataset=dataset,
+        dataset_info=dataset_info,
+        overwrite=overwrite,
+    )
+
+    os.makedirs(downloader.local_folder, exist_ok=True)
+
+    user = os.environ.get("rda-user")
+    if user is None:
+        user = str(input("RDA user name? "))
+        if user == "":
+            errmsg = ("A RDA account is required to download JRA-55 data."
+                      " Please visit https://rda.ucar.edu/login/register/"
+                      " to create an account at the Research Data Archive"
+                      " (RDA) if needed.")
+            logger.error(errmsg)
+            raise ValueError
+
+    passwd = os.environ.get("rda-passwd")
+    if passwd is None:
+        passwd = str(input("RDA password? "))
+
+    if start_date is None:
+        start_date = datetime(1958, 1, 1)
+    if end_date is None:
+        end_date = datetime(2022, 12, 31)
+    loop_date = start_date
+
+    options = ["-O", "Authentication.log", "--save-cookies=auth.rda_ucar_edu",
+               f"--post-data=\"email={user}&passwd={passwd}&action=login\""]
+
+    # login to Research Data Archive (RDA)
+
+    downloader.login("https://rda.ucar.edu/cgi-bin/login", options)
+
+    # download files
+
+    url = "https://data.rda.ucar.edu/ds628.1"
+    download_options = ["--load-cookies=auth.rda_ucar_edu"]
+
+    # define variables to download
+
+    var = [["011_tmp", "anl_p125"],
+           ["011_tmp", "anl_surf125"],
+           ["039_vvel", "anl_p125"],
+           ["071_tcdc", "fcst_surf125"],
+           ["054_pwat", "fcst_column125"],
+           ["058_cice", "fcst_column125"],
+           ["160_csusf", "fcst_phy2m125"],
+           ["162_csulf", "fcst_phy2m125"],
+           ["211_uswrf", "fcst_phy2m125"],
+           ["212_ulwrf", "fcst_phy2m125"],
+           ["227_cw", "fcst_column125"],
+           ["228_clwc", "fcst_p125"],
+           ["229_ciwc", "fcst_p125"]]
+
+    # download data
+
+    while loop_date <= end_date:
+        year = loop_date.year
+
+        for item in var:
+            varname = item[0]
+            channel = item[1]
+            fname = f"{channel}.{varname}.{year}01_{year}12"
+            # download file
+            downloader.download_file(url + f"/{channel}/{year}/" +
+                                     fname, download_options)
+            # add file extension ".grb"
+            os.rename(downloader.local_folder + "/" + fname,
+                      downloader.local_folder + "/" + fname + ".grb")
+
+        loop_date += relativedelta.relativedelta(years=1)
+
+    # clean up temporary files
+
+    if os.path.exists("Authentication.log"):
+        os.remove("Authentication.log")
+    if os.path.exists("auth.rda_ucar_edu"):
+        os.remove("auth.rda_ucar_edu")
diff --git a/esmvaltool/cmorizers/data/downloaders/wget.py b/esmvaltool/cmorizers/data/downloaders/wget.py
@@ -54,6 +54,20 @@ def download_file(self, server_path, wget_options):
         logger.debug(command)
         subprocess.check_output(command)
 
+    def login(self, server_path, wget_options):
+        """Login.
+
+        Parameters
+        ----------
+        server_path: str
+            Path to remote file
+        wget_options: list(str)
+            Extra options for wget
+        """
+        command = ['wget'] + wget_options + [server_path]
+        logger.debug(command)
+        subprocess.check_output(command)
+
     @property
     def overwrite_options(self):
         """Get overwrite options as configured in downloader."""
-Original file line number
+Diff line change
@@ Expand Up / @@ -14,6 +14,7 @@ dependencies: @@
       - cdo >=2.3.0
       - cdsapi
       - cf-units
+      - cfgrib
       - cftime
       - cmocean
       - curl <8.10
@@ Expand Down @@