Open multiple files using xcdat

.github/workflows/build_workflow.yml

@@ -31,19 +31,19 @@ jobs:
         with:
           node-version: '16'
       - name: Checkout Code Repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           node-version: '16'
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v2
+      - name: Set up Python
+        uses: actions/setup-python@v4
         with:
           node-version: '16'
-          python-version: 3.9
+          python-version: '3.10'
       # Run all pre-commit hooks on all the files.
       # Getting only staged files can be tricky in case a new PR is opened
       # since the action is run on a branch in detached head state
       - name: Install and Run Pre-commit
-        uses: pre-commit/action@v2.0.3
+        uses: pre-commit/action@v3.0.0
         with:
           node-version: '16'
 
@@ -59,26 +59,48 @@ jobs:
       - uses: actions/setup-node@v3
         with:
           node-version: '16'
-      - uses: actions/checkout@v2
-      - name: Cache Conda
-        uses: actions/cache@v2
-        env:
-          # Increase this value to reset cache if conda/dev.yml has not changed in the workflow
-          CACHE_NUMBER: 0
-        with:
-          node-version: '16'
-          path: ~/conda_pkgs_dir
-          key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-publish
+      - uses: actions/checkout@v3
 
       - name: Set up Conda Environment
         uses: conda-incubator/setup-miniconda@v2
         with:
           activate-environment: "pcmdi_metrics_dev"
-          environment-file: conda-env/dev.yml
+          miniforge-variant: Mambaforge
+          miniforge-version: latest
+          use-mamba: true
+          mamba-version: "*"
           channel-priority: strict
           auto-update-conda: true
-          # IMPORTANT: This needs to be set for caching to work properly!
-          use-only-tar-bz2: true
+
+      # Used for refreshing the cache every 24 hours to avoid inconsistencies of package
+      # versions between the CI pipeline and local installations.
+      - name: Get Date
+        id: get-date
+        run: echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT
+        shell: bash
+
+      - name: Cache Conda
+        uses: actions/cache@v3
+        with:
+          node-version: '16'
+          path: ${{ env.CONDA }}/envs
+          key:
+            conda-${{ runner.os }}--${{ runner.arch }}--${{
+            steps.get-date.outputs.today }}-${{
+            hashFiles('conda-env/dev.yml') }}-${{ env.CACHE_NUMBER}}
+        env:
+          # Increase this value to reset cache if conda/dev.yml has not changed in the workflow
+          CACHE_NUMBER: 0
+
+      - name: Update environment
+        run:
+          mamba env update -n pcmdi_metrics_dev -f conda-env/dev.yml
+        if: steps.cache.outputs.cache-hit != 'true'
+
+      - name: Install pcmdi_metrics
+        # Source: https://github.com/conda/conda-build/issues/4251#issuecomment-1053460542
+        run: |
+            python -m pip install --no-build-isolation --no-deps -e .
 
       - name: Run Tests
         run: |

conda-env/dev.yml

@@ -6,34 +6,34 @@ channels:
 dependencies:
   # Base
   # ==================
-  - python=3.9.7
-  - pip=21.2.4
-  - numpy=1.21.3
-  - cartopy=0.20.1
-  - matplotlib=3.4.3
+  - python=3.10.10
+  - pip=23.1.2
+  - numpy=1.23.5
+  - cartopy=0.21.1
+  - matplotlib=3.7.1
   - cdat_info=8.2.1
   - cdms2=3.1.5
   - genutil=8.2.1
   - cdutil=8.2.1
   - cdp=1.7.0
   - eofs=1.4.0
-  - seaborn=0.11.1
+  - seaborn=0.12.2
   - enso_metrics=1.1.1
-  - xcdat=0.4.0
+  - xcdat=0.5.0
   - xmltodict=0.13.0
-  - setuptools=65.5.0
-  - netcdf4=1.6.0
+  - setuptools=67.7.2
+  - netcdf4=1.6.3
   - regionmask=0.9.0
-  - rasterio=1.2.10
-  - shapely=1.8.0
+  - rasterio=1.3.6
+  - shapely=2.0.1
   # Testing
   # ==================
-  - pre_commit=2.20.0
-  - pytest=6.2.5
-  - pytest-cov=3.0.0
+  - pre_commit=3.2.2
+  - pytest=7.3.1
+  - pytest-cov=4.0.0
   # Developer Tools
   # ==================
-  - jupyterlab=3.2.1
+  - jupyterlab=3.6.3
   - nb_conda=2.2.1
   - nb_conda_kernels=2.3.1
 

pcmdi_metrics/io/base.py

@@ -18,7 +18,8 @@
 
 import pcmdi_metrics
 from pcmdi_metrics import LOG_LEVEL
-from pcmdi_metrics.io import xcdat_open
+
+import xcdat as xc
 
 value = 0
 cdms2.setNetcdfShuffleFlag(value)  # where value is either 0 or 1
@@ -371,9 +372,9 @@ def extract_var_from_file(self, var, var_in_file, *args, **kwargs):
             var_in_file = var
 
         try:
-            ds = xcdat_open(self(), data_var=var_in_file, decode_times=True)
+            ds = xc.open_mfdataset(self(), data_var=var_in_file, decode_times=True)
         except Exception:
-            ds = xcdat_open(self(), data_var=var_in_file, decode_times=False)  # Temporary part to read in cdms written obs4MIP AC files
+            ds = xc.open_mfdataset(self(), data_var=var_in_file, decode_times=False)  # Temporary part to read in cdms written obs4MIP AC files
 
         if 'level' in list(kwargs.keys()):
             level = kwargs['level']

pcmdi_metrics/mean_climate/lib/calculate_climatology.py

@@ -4,7 +4,7 @@
 import dask
 from genutil import StringConstructor
 
-from pcmdi_metrics.io import xcdat_open
+import xcdat as xc
 
 
 def calculate_climatology(
@@ -21,7 +21,7 @@ def calculate_climatology(
     print("infilename:", infilename)
 
     # open file
-    d = xcdat_open(infile, data_var=var)   # wrapper of xcdat open functions to enable using xml
+    d = xc.open_mfdataset(infile, data_var=var)
     atts = d.attrs
 
     print("type(d):", type(d))

pcmdi_metrics/mean_climate/lib/load_and_regrid.py

@@ -1,4 +1,3 @@
-from pcmdi_metrics.io import xcdat_open
 import cftime
 import xcdat as xc
 import numpy as np
@@ -23,7 +22,7 @@ def load_and_regrid(data_path, varname, varname_in_file=None, level=None, t_grid
         varname_in_file = varname
 
     # load data
-    ds = xcdat_open(data_path, data_var=varname_in_file, decode_times=decode_times)  # NOTE: decode_times=False will be removed once obs4MIP written using xcdat
+    ds = xc.open_mfdataset(data_path, data_var=varname_in_file, decode_times=decode_times)  # NOTE: decode_times=False will be removed once obs4MIP written using xcdat
 
     # calendar quality check
     if "calendar" in list(ds.time.attrs.keys()):

setup.py

@@ -11,7 +11,7 @@
 else:
     install_dev = False
 
-release_version = "3.0.2"
+release_version = "3.1"
 
 p = subprocess.Popen(
     ("git", "describe", "--tags"),
@@ -20,34 +20,6 @@
     stderr=subprocess.PIPE,
 )
 
-"""
-try:
-    descr = p.stdout.readlines()[0].strip().decode("utf-8")
-    Version = "-".join(descr.split("-")[:-2])
-    if Version == "":
-        Version = descr
-except Exception:
-    descr = release_version
-    Version = release_version
-
-p = subprocess.Popen(
-    ("git", "log", "-n1", "--pretty=short"),
-    stdin=subprocess.PIPE,
-    stdout=subprocess.PIPE,
-    stderr=subprocess.PIPE,
-)
-try:
-    commit = p.stdout.readlines()[0].split()[1].decode("utf-8")
-except Exception:
-    commit = ""
-
-f = open("pcmdi_metrics/version.py", "w")
-print("__version__ = '%s'" % Version, file=f)
-print("__git_tag_describe__ = '%s'" % descr, file=f)
-print("__git_sha1__ = '%s'" % commit, file=f)
-f.close()
-"""
-
 # Generate and install default arguments
 p = subprocess.Popen(["python", "setup_default_args.py"], cwd="share")
 p.communicate()