update dataset url (#690)

* update dataset url * added download tests for cds and cmems
Deltares · Dec 6, 2023 · 4b7440e · 4b7440e
1 parent 9a527af
commit 4b7440e
Show file tree

Hide file tree

Showing 2 changed files with 38 additions and 2 deletions.
diff --git a/dfm_tools/download.py b/dfm_tools/download.py
@@ -108,7 +108,7 @@ def download_CMEMS(varkey,
     global product #set product as global variable, so it only has to be retreived once per download run (otherwise once per variable)
     if 'product' not in globals():
         print('retrieving time range of CMEMS reanalysis and forecast products') #assuming here that physchem and bio reanalyisus/multiyear datasets have the same enddate, this seems safe
-        reanalysis_tstart, reanalysis_tstop = get_OPeNDAP_xr_ds_timerange(dataset_url='https://my.cmems-du.eu/thredds/dodsC/cmems_mod_glo_phy_my_0.083_P1D-m')
+        reanalysis_tstart, reanalysis_tstop = get_OPeNDAP_xr_ds_timerange(dataset_url='https://my.cmems-du.eu/thredds/dodsC/cmems_mod_glo_phy_my_0.083deg_P1D-m')
         forecast_tstart, forecast_tstop = get_OPeNDAP_xr_ds_timerange(dataset_url='https://nrt.cmems-du.eu/thredds/dodsC/cmems_mod_glo_phy_anfc_0.083deg_P1D-m')
         if (date_min >= reanalysis_tstart) & (date_max <= reanalysis_tstop):
             product = 'reanalysis'
@@ -131,7 +131,7 @@ def download_CMEMS(varkey,
                 varkey_name = 'phy'
             dataset_url = f'https://nrt.cmems-du.eu/thredds/dodsC/cmems_mod_glo_{varkey_name}_anfc_0.083deg_P1D-m'#.html' #TODO: also PT6H-i timeresolution available but not for all variables and not for reanalysis
         else: #reanalysis: https://data.marine.copernicus.eu/product/GLOBAL_MULTIYEAR_PHY_001_030/description
-            dataset_url = 'https://my.cmems-du.eu/thredds/dodsC/cmems_mod_glo_phy_my_0.083_P1D-m'
+            dataset_url = 'https://my.cmems-du.eu/thredds/dodsC/cmems_mod_glo_phy_my_0.083deg_P1D-m'
     else: #for bio
         buffer = 2/4 # resolution is 1/4 degrees
         if product == 'analysisforecast': #forecast: https://data.marine.copernicus.eu/product/GLOBAL_ANALYSIS_FORECAST_BIO_001_028/description

diff --git a/tests/test_download.py b/tests/test_download.py
@@ -10,6 +10,7 @@
 import pandas as pd
 import numpy as np
 from dfm_tools.download import round_timestamp_to_outer_noon, cds_credentials, copernicusmarine_credentials
+import dfm_tools as dfmt
 
 
 @pytest.mark.unittest
@@ -47,3 +48,38 @@ def test_cds_credentials():
 @pytest.mark.unittest
 def test_copernicusmarine_credentials():
     copernicusmarine_credentials()
+
+
+#TODO: properly set environment variables in github would prevent localness
+@pytest.mark.requireslocaldata
+@pytest.mark.unittest
+def test_download_era5():
+    date_min = '2010-01-01'
+    date_max = '2010-01-02'
+    longitude_min, longitude_max, latitude_min, latitude_max =    2,   3,  51, 52 #test domain
+    variables_era5 = ['msl']#'v10n'] # check variables_dict in dfmt.download_ERA5() for valid names
+    dir_output = './era5_temp'
+    for varkey in variables_era5:
+        os.makedirs(dir_output, exist_ok=True)
+
+        dfmt.download_ERA5(varkey, 
+                           longitude_min=longitude_min, longitude_max=longitude_max, latitude_min=latitude_min, latitude_max=latitude_max,
+                           date_min=date_min, date_max=date_max,
+                           dir_output=dir_output, overwrite=True)
+
+
+#TODO: properly set environment variables in github would prevent localness
+@pytest.mark.requireslocaldata
+@pytest.mark.unittest
+def test_download_cmems():
+    date_min = '2010-01-01'
+    date_max = '2010-01-02'
+    longitude_min, longitude_max, latitude_min, latitude_max =    2,   3,  51, 52 #test domain
+    varlist_cmems = ['bottomT','no3'] # avaliable variables differ per product, examples are ['bottomT','mlotst','siconc','sithick','so','thetao','uo','vo','usi','vsi','zos','no3']. More info on https://data.marine.copernicus.eu/products
+    dir_output = './cmems_temp'
+    for varkey in varlist_cmems:
+        file_prefix = 'cmems_'
+        dfmt.download_CMEMS(varkey=varkey,
+                            longitude_min=longitude_min, longitude_max=longitude_max, latitude_min=latitude_min, latitude_max=latitude_max,
+                            date_min=date_min, date_max=date_max,
+                            dir_output=dir_output, file_prefix=file_prefix, overwrite=True)