diff --git a/cate/core/ds.py b/cate/core/ds.py index 2443d6642..3908d33a9 100644 --- a/cate/core/ds.py +++ b/cate/core/ds.py @@ -555,11 +555,11 @@ def open_xarray_dataset(paths, concat_dim='time', **kwargs) -> xr.Dataset: # Find number of chunks as the closest larger squared number (1,4,9,..) try: - temp_ds = xr.open_dataset(paths[0]) + temp_ds = xr.open_dataset(paths[0], **kwargs) except (OSError, RuntimeError): # netcdf4 >=1.2.2 raises RuntimeError # We have a glob not a list - temp_ds = xr.open_dataset(glob.glob(paths)[0]) + temp_ds = xr.open_dataset(glob.glob(paths)[0], **kwargs) n_chunks = ceil(sqrt(temp_ds.nbytes / threshold)) ** 2 diff --git a/cate/ds/esa_cci_odp.py b/cate/ds/esa_cci_odp.py index c060d1914..3d31b21cd 100644 --- a/cate/ds/esa_cci_odp.py +++ b/cate/ds/esa_cci_odp.py @@ -678,7 +678,7 @@ def _make_local(self, region = PolygonLike.convert(region) var_names = VarNamesLike.convert(var_names) - time_range, region, var_names = self._apply_make_local_fixes(time_range, region, var_names) + exclude_var_names = self._apply_make_local_fixes() compression_level = get_config_value('NETCDF_COMPRESSION_LEVEL', NETCDF_COMPRESSION_LEVEL) compression_enabled = True if compression_level > 0 else False @@ -725,7 +725,7 @@ def _make_local(self, child_monitor.start(label=file_name, total_work=1) - remote_dataset = xr.open_dataset(dataset_uri) + remote_dataset = xr.open_dataset(dataset_uri, drop_variables=exclude_var_names) if var_names: remote_dataset = remote_dataset.drop( @@ -814,12 +814,10 @@ def reporthook(block_number, read_size, total_file_size): raise DataAccessError("Copying remote data source failed: {}".format(e), source=self) from e local_ds.meta_info['temporal_coverage_start'] = TimeLike.format(verified_time_coverage_start) local_ds.meta_info['temporal_coverage_end'] = TimeLike.format(verified_time_coverage_end) + local_ds.meta_info['exclude_var_names'] = exclude_var_names local_ds.save(True) - def _apply_make_local_fixes(self, - time_range: Optional[TimeRange], - region: Optional[Polygon], - var_names: Optional[VarNames]): + def _apply_make_local_fixes(self): """ This method applies fixes to the parameters of a 'make_local' invocation. """ @@ -833,11 +831,8 @@ def _apply_make_local_fixes(self, # can not be decoded by xarray and lead to an unusable dataset # see: https://github.com/CCI-Tools/cate/issues/326 if self.id in SOILMOISTURE_DS: - if not var_names: - var_names = self._json_dict.get('variable', []) - if 't0' in var_names: - var_names.remove('t0') - return time_range, region, var_names + return ['t0'] + return [] def make_local(self, local_name: str, diff --git a/cate/ds/local.py b/cate/ds/local.py index f7b65deaf..b297edf3c 100644 --- a/cate/ds/local.py +++ b/cate/ds/local.py @@ -169,14 +169,14 @@ def open_dataset(self, if paths: paths = sorted(set(paths)) try: - ds = open_xarray_dataset(paths) + ds = open_xarray_dataset(paths, drop_variables=self._meta_info.get('exclude_var_names', [])) if region: ds = normalize_impl(ds) ds = subset_spatial_impl(ds, region) if var_names: ds = ds.drop([var_name for var_name in ds.data_vars.keys() if var_name not in var_names]) return ds - except OSError as e: + except (OSError, ValueError) as e: if time_range: raise DataAccessError("Cannot open local dataset for time range {}:\n" "{}"