#326, #546 - ODP dataset fetch and open, excluding problematic variables

CCI-Tools · Mar 9, 2018 · f3bb1d8 · f3bb1d8
1 parent ea9564f
commit f3bb1d8
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 15 deletions.
diff --git a/cate/core/ds.py b/cate/core/ds.py
@@ -555,11 +555,11 @@ def open_xarray_dataset(paths, concat_dim='time', **kwargs) -> xr.Dataset:
 
     # Find number of chunks as the closest larger squared number (1,4,9,..)
     try:
-        temp_ds = xr.open_dataset(paths[0])
+        temp_ds = xr.open_dataset(paths[0], **kwargs)
     except (OSError, RuntimeError):
         # netcdf4 >=1.2.2 raises RuntimeError
         # We have a glob not a list
-        temp_ds = xr.open_dataset(glob.glob(paths)[0])
+        temp_ds = xr.open_dataset(glob.glob(paths)[0], **kwargs)
 
     n_chunks = ceil(sqrt(temp_ds.nbytes / threshold)) ** 2
 

diff --git a/cate/ds/esa_cci_odp.py b/cate/ds/esa_cci_odp.py
@@ -678,7 +678,7 @@ def _make_local(self,
         region = PolygonLike.convert(region)
         var_names = VarNamesLike.convert(var_names)
 
-        time_range, region, var_names = self._apply_make_local_fixes(time_range, region, var_names)
+        exclude_var_names = self._apply_make_local_fixes()
 
         compression_level = get_config_value('NETCDF_COMPRESSION_LEVEL', NETCDF_COMPRESSION_LEVEL)
         compression_enabled = True if compression_level > 0 else False
@@ -725,7 +725,7 @@ def _make_local(self,
 
                     child_monitor.start(label=file_name, total_work=1)
 
-                    remote_dataset = xr.open_dataset(dataset_uri)
+                    remote_dataset = xr.open_dataset(dataset_uri, drop_variables=exclude_var_names)
 
                     if var_names:
                         remote_dataset = remote_dataset.drop(
@@ -814,12 +814,10 @@ def reporthook(block_number, read_size, total_file_size):
             raise DataAccessError("Copying remote data source failed: {}".format(e), source=self) from e
         local_ds.meta_info['temporal_coverage_start'] = TimeLike.format(verified_time_coverage_start)
         local_ds.meta_info['temporal_coverage_end'] = TimeLike.format(verified_time_coverage_end)
+        local_ds.meta_info['exclude_var_names'] = exclude_var_names
         local_ds.save(True)
 
-    def _apply_make_local_fixes(self,
-                                time_range: Optional[TimeRange],
-                                region: Optional[Polygon],
-                                var_names: Optional[VarNames]):
+    def _apply_make_local_fixes(self):
         """
         This method applies fixes to the parameters of a 'make_local' invocation.
         """
@@ -833,11 +831,8 @@ def _apply_make_local_fixes(self,
             # can not be decoded by xarray and lead to an unusable dataset
             # see: https://github.com/CCI-Tools/cate/issues/326
             if self.id in SOILMOISTURE_DS:
-                if not var_names:
-                    var_names = self._json_dict.get('variable', [])
-                if 't0' in var_names:
-                    var_names.remove('t0')
-        return time_range, region, var_names
+                return ['t0']
+            return []
 
     def make_local(self,
                    local_name: str,

diff --git a/cate/ds/local.py b/cate/ds/local.py
@@ -169,14 +169,14 @@ def open_dataset(self,
         if paths:
             paths = sorted(set(paths))
             try:
-                ds = open_xarray_dataset(paths)
+                ds = open_xarray_dataset(paths, drop_variables=self._meta_info.get('exclude_var_names', []))
                 if region:
                     ds = normalize_impl(ds)
                     ds = subset_spatial_impl(ds, region)
                 if var_names:
                     ds = ds.drop([var_name for var_name in ds.data_vars.keys() if var_name not in var_names])
                 return ds
-            except OSError as e:
+            except (OSError, ValueError) as e:
                 if time_range:
                     raise DataAccessError("Cannot open local dataset for time range {}:\n"
                                           "{}"