Point to right directory where Hycom data is stored

.gitignore

@@ -52,6 +52,7 @@ coverage.xml
 *.cover
 .hypothesis/
 .pytest_cache/
+oceanspy/tests/Data
 oceanspy/tests/Data/
 oceanspy/tests/.ipynb_checkpoints/
 oceanspy/tests/.nfs*

.pre-commit-config.yaml

@@ -9,7 +9,7 @@ repos:
       - id: isort
 
   - repo: https://github.com/psf/black
-    rev: 21.10b0
+    rev: 21.12b0
     hooks:
       - id: black
 
@@ -19,7 +19,7 @@ repos:
        - id: flake8
 
   - repo: https://github.com/nbQA-dev/nbQA
-    rev: 1.1.1
+    rev: 1.2.2
     hooks:
       - id: nbqa-black
         additional_dependencies: [black]

oceanspy/tests/Data

@@ -0,0 +1 @@
+/noc/users/malmans/.cache/pooch/c627406d4df65814883c5a0f99418257-Data.tar.gz.untar/Data

oceanspy/tests/conftest.py

@@ -1,24 +1,26 @@
 # Import modules
-import subprocess
+import os
 
+import pooch
+from pooch import Untar
 
-# Download data
-def pytest_configure():
-    print("\nHello! I'm downloading test data.")
 
-    # Directory
-    Datadir = "./oceanspy/tests/"
+# Download data if necessary
+def pytest_configure():
 
-    # Download xmitgcm test
-    commands = [
-        "cd {}".format(Datadir),
-        "rm -fr Data",
-        "wget -v -O Data.tar.gz -L "
-        "https://livejohnshopkins-my.sharepoint.com/"
-        ":u:/g/personal/malmans2_jh_edu/"
-        "EVtxjAQL13tCt7dFHNrzsrwBuqdjDe3zrvrJ625YrjrF0g"
-        "?download=1",
-        "tar xvzf Data.tar.gz",
-        "rm -f Data.tar.gz",
-    ]
-    subprocess.call("&&".join(commands), shell=True)
+    fnames = pooch.retrieve(
+        url="https://zenodo.org/record/5825166/files/Data.tar.gz?download=1",
+        processor=Untar(),
+        known_hash="165bb4c0459a3b776efe9f94e8f7843140bc7e90b9d686af574cdb8c11006ba2",
+    )
+    symlink_args = dict(
+        src=f"{os.path.commonpath(fnames)}",
+        dst="./oceanspy/tests/Data",
+        target_is_directory=True,
+    )
+    try:
+        print(f"Linking {symlink_args['src']!r} to {symlink_args['dst']!r}")
+        os.symlink(**symlink_args)
+    except FileExistsError:
+        os.unlink("./oceanspy/tests/Data")
+        os.symlink(**symlink_args)

oceanspy/tests/test_open_oceandataset.py

@@ -24,6 +24,7 @@
 xmitgcm_url = "{}catalog_xmitgcm.yaml".format(Datadir)
 xarray_url = "{}catalog_xarray.yaml".format(Datadir)
 ECCO_url = "{}catalog_ECCO.yaml".format(Datadir)
+hycom_url = "{}hycom_test.yaml".format(Datadir)
 
 
 # Test SciServer
@@ -43,6 +44,7 @@ def test_find_entries(names):
         ("grd_rect", xarray_url),
         ("grd_curv", xarray_url),
         ("LLC", ECCO_url),
+        ("HYCOM", hycom_url),
     ],
 )
 def test_opening_and_saving(name, catalog_url):
@@ -55,13 +57,15 @@ def test_opening_and_saving(name, catalog_url):
         od1 = from_catalog(name, catalog_url)
 
         # Check dimensions
-        if name != "xarray":
+        if name not in ["xarray", "HYCOM"]:
             dimsList = ["X", "Y", "Xp1", "Yp1"]
             assert set(dimsList).issubset(set(od1.dataset.dims))
 
             # Check coordinates
             if name == "LLC":
                 coordsList = ["XC", "YC", "XG", "YG"]
+            elif name == "HYCOM":
+                coordsList = ["XC", "YC"]
             else:
                 coordsList = ["XC", "YC", "XG", "YG", "XU", "YU", "XV", "YV"]
             assert set(coordsList).issubset(set(od1.dataset.coords))

sciserver_catalogs/catalog_xarray.yaml

@@ -1442,32 +1442,21 @@ sources:
         Justin et al (2014), A new synoptic scale resolving global climate simulatoin using the Community Earth System Model. J. Adv. Model. Earth Syst., 6, 1065-1094. https:doi.org/10.1002/2014MS000363
       original_output: Monthly means.
 
+
 # ======================================================================
 # HyCOM data test, interpolated onto lat-lon-depth coords
   HYCOM_test:
-    description: Simulation off the Northern California Coast with HyCOM ocean mdoel
-    driver: netcdf
+    description: Simulation off the Northern California Coast with HyCOM ocean model
+    driver: zarr
     model: HyCOM
     args:
-      urlpath: /sciserver/oceanography/HyCOM/swot+smode/GLBy0.04_*.nc
-      xarray_kwargs:
-        engine: netcdf4
-        drop_variables: ['bottom_salinity','bottom_temp','bottom_u','bottom_v']
-        concat_dim: time
-        combine: by_coords
-        parallel: True
-        decode_cf: False
+      urlpath: /home/idies/workspace/poseidon/data10_01/HyCOM
+    xarray_kwargs:
+        engine: zarr
     metadata:
       rename:
-        water_temp: Temp
-        water_u: U
-        water_v: V
-        lon: X
-        lat: Y
-        depth: Z
-      manipulate_coords:
-        fillna: False
-        coords1Dfrom2D: False
+        lon: XC
+        lat: YC
       grid_coords:
         add_midp: True
         grid_coords:
@@ -1484,19 +1473,11 @@ sources:
             Zl: -0.5
           time: 
             time: -0.5
-      parameters:
-        temp_offset: 20
-        temp_scale: 0.001
-        temp_fill: -30000
-        sal_offset: 20
-        sal_scale: 0.001
-        sal_fill: -30000
-        ssh_fill: 1.2676e30
       name: California_Coast
       description: |
         High-resolution grid of the (1/25 deg) numerical simulation covering the California Coast using the HyCOM model. The original grid has 41 (isopycnal) layers but these files were interpolated to 101 vertical levels.
       mates: | 
         Sample (test) files
       projection: Mercator
-      original_output: snapshot.
+      original_output: snapshot