From 349e7db5ef5cd7fb20e4aeec047e9d55d683909e Mon Sep 17 00:00:00 2001 From: "Phillip J. Wolfram" Date: Wed, 11 Jan 2017 09:33:36 -0700 Subject: [PATCH] Fixes too many files open error: h5netcdf backend --- xarray/backends/h5netcdf_.py | 44 +++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/xarray/backends/h5netcdf_.py b/xarray/backends/h5netcdf_.py index 76582cfd72e..0679b928859 100644 --- a/xarray/backends/h5netcdf_.py +++ b/xarray/backends/h5netcdf_.py @@ -2,6 +2,7 @@ from __future__ import division from __future__ import print_function import functools +import contextlib from .. import Variable from ..core import indexing @@ -36,6 +37,11 @@ def _read_attributes(h5netcdf_var): _extract_h5nc_encoding = functools.partial(_extract_nc4_encoding, lsd_okay=False, backend='h5netcdf') +def _close_ds(ds): + # netCDF4 only allows closing the root group + while ds.parent is not None: + ds = ds.parent + ds.close() def _open_h5netcdf_group(filename, mode, group): import h5netcdf.legacyapi @@ -54,12 +60,20 @@ def __init__(self, filename, mode='r', format=None, group=None, opener = functools.partial(_open_h5netcdf_group, filename, mode=mode, group=group) self.ds = opener() + self._isopen = True self.format = format self._opener = opener self._filename = filename self._mode = mode super(H5NetCDFStore, self).__init__(writer) + @contextlib.contextmanager + def ensure_open(self): + if not self._isopen: + self.ds = self._opener() + self._isopen = True + yield + def open_store_variable(self, name, var): dimensions = var.dimensions data = indexing.LazilyIndexedArray(BaseNetCDF4Array(name, self)) @@ -77,20 +91,25 @@ def open_store_variable(self, name, var): return Variable(dimensions, data, attrs, encoding) def get_variables(self): - return FrozenOrderedDict((k, self.open_store_variable(k, v)) - for k, v in iteritems(self.ds.variables)) + with self.ensure_open(): + return FrozenOrderedDict((k, self.open_store_variable(k, v)) + for k, v in iteritems(self.ds.variables)) def get_attrs(self): - return Frozen(_read_attributes(self.ds)) + with self.ensure_open(): + return Frozen(_read_attributes(self.ds)) def get_dimensions(self): - return self.ds.dimensions + with self.ensure_open(): + return self.ds.dimensions def set_dimension(self, name, length): - self.ds.createDimension(name, size=length) + with self.ensure_open(): + self.ds.createDimension(name, size=length) def set_attribute(self, key, value): - self.ds.setncattr(key, value) + with self.ensure_open(): + self.ds.setncattr(key, value) def prepare_variable(self, name, variable, check_encoding=False): import h5py @@ -123,12 +142,11 @@ def prepare_variable(self, name, variable, check_encoding=False): return nc4_var, variable.data def sync(self): - super(H5NetCDFStore, self).sync() - self.ds.sync() + with self.ensure_open(): + super(H5NetCDFStore, self).sync() + self.ds.sync() def close(self): - ds = self.ds - # netCDF4 only allows closing the root group - while ds.parent is not None: - ds = ds.parent - ds.close() + if self._isopen: + self._isopen = False + _close_ds(self.ds)