Solve incompatibilities with newest numpy and xarray

docs/whats_new.rst

@@ -1,5 +1,35 @@
 What's New
 ==========
+v3.14.2 (2024/11/12)
+--------------------
+New Features
+~~~~~~~~~~~~
+
+Breaking changes
+~~~~~~~~~~~~~~~~
+
+Deprecations
+~~~~~~~~~~~~
+
+Bug fixes
+~~~~~~~~~
+
+Documentation
+~~~~~~~~~~~~~
+
+Performance
+~~~~~~~~~~~
+
+Internal Changes
+~~~~~~~~~~~~~~~~
+- Several changes to allow compatibility with latest versions of :py:mod:`numpy` and :py:mod:`xarray`:
+
+  - Ensure that :py:class:`float` objects evaluated with :py:mod:`numpy` are converted to :py:class:`float`.
+
+  - Ensure proper shape of inputs when loading external data.
+
+- Avoid :py:class:`DeprecationWarning` raised in `invert_matrix`` test.
+
 v3.14.1 (2024/07/18)
 --------------------
 New Features

pysd/_version.py

@@ -1 +1 @@
-__version__ = "3.14.1"
+__version__ = "3.14.2"

pysd/builders/python/python_expressions_builder.py

@@ -16,13 +16,13 @@
 from pysd.py_backend.utils import compute_shape
 
 from pysd.translators.structures.abstract_expressions import\
-    AbstractSyntax, AllocateAvailableStructure, AllocateByPriorityStructure,\
-    ArithmeticStructure, CallStructure, DataStructure, DelayFixedStructure,\
-    DelayStructure, DelayNStructure, ForecastStructure, GameStructure,\
-    GetConstantsStructure, GetDataStructure, GetLookupsStructure,\
-    InitialStructure, InlineLookupsStructure, IntegStructure,\
-    LogicStructure, LookupsStructure, ReferenceStructure,\
-    SampleIfTrueStructure, SmoothNStructure, SmoothStructure,\
+    AbstractSyntax, AllocateAvailableStructure, AllocateByPriorityStructure, \
+    ArithmeticStructure, CallStructure, DataStructure, DelayFixedStructure, \
+    DelayStructure, DelayNStructure, ForecastStructure, GameStructure, \
+    GetConstantsStructure, GetDataStructure, GetLookupsStructure, \
+    InitialStructure, InlineLookupsStructure, IntegStructure, \
+    LogicStructure, LookupsStructure, ReferenceStructure, \
+    SampleIfTrueStructure, SmoothNStructure, SmoothStructure, \
     SubscriptsReferenceStructure, TrendStructure
 
 from .python_functions import functionspace
@@ -694,6 +694,10 @@ def build_function_call(self, arguments: dict) -> BuildAST:
                         self.section.subscripts, final_subscripts, True)
                      for i in ["0", "1", "2"]]
 
+        # ensure numpy outputs of floats being floats
+        if "np." in expression and not final_subscripts:
+            expression = "float(" + expression + ")"
+
         return BuildAST(
             expression=expression % arguments,
             calls=calls,

pysd/py_backend/external.py

@@ -502,10 +502,9 @@ def _initialize_data(self, element_type):
             if self.interp != "raw":
                 self._fill_missing(series, data)
 
+        # reshape the data to fit in the xarray.DataArray
         reshape_dims = tuple([len(series)] + utils.compute_shape(self.coords))
-
-        if len(reshape_dims) > 1:
-            data = self._reshape(data, reshape_dims)
+        data = self._reshape(data, reshape_dims)
 
         if element_type == "lookup":
             dim_name = "lookup_dim"
@@ -977,10 +976,7 @@ def _initialize(self):
         # Create only an xarray if the data is not 0 dimensional
         if len(self.coords) > 0:
             reshape_dims = tuple(utils.compute_shape(self.coords))
-
-            if len(reshape_dims) > 1:
-                data = self._reshape(data, reshape_dims)
-
+            data = self._reshape(data, reshape_dims)
             data = xr.DataArray(
                 data=data, coords=self.coords, dims=list(self.coords)
             )

pysd/py_backend/model.py

@@ -1169,14 +1169,14 @@ def _timeseries_component(self, series, dims):
         elif dims:
             # the interpolation will be time dependent
             return lambda: utils.rearrange(
-                np.interp(self.time(), series.index, series.values),
+                float(np.interp(self.time(), series.index, series.values)),
                 dims, self._subscript_dict), {'time': 1}
 
         else:
             # the interpolation will be time dependent
-            return lambda: np.interp(
+            return lambda: float(np.interp(
                 self.time(), series.index, series.values
-                ), {'time': 1}
+                )), {'time': 1}
 
     def _constant_component(self, value, dims):
         """ Internal function for creating a constant model element """

tests/data/expected_data.py

@@ -38,7 +38,7 @@
 coords_2dl = {'ABC': ['A', 'B', 'C'], 'XY': ['X', 'Y']}
 dims_2dl = ['XY', 'ABC']
 
-xpts = np.arange(-0.5, 8.6, 0.5)
+xpts = np.arange(-0.5, 8.6, 0.5).tolist()
 
 # 1d lookup/data
 
@@ -246,4 +246,3 @@
 
 forward_3d = [xr.DataArray(data, coords_2dl, dims_2dl)
               for data in forward_3dl]
-

tests/pytest.ini

@@ -4,4 +4,4 @@ filterwarnings =
     error
     always:numpy.ndarray size changed, may indicate binary incompatibility.:RuntimeWarning
     always::DeprecationWarning
-    always::PendingDeprecationWarning
+    always::PendingDeprecationWarning

tests/pytest_pysd/pytest_functions.py

@@ -445,27 +445,29 @@ def test_invert_matrix(self):
         for data in [data1, data2, data3]:
             datai = invert_matrix(data)
             assert data.dims == datai.dims
+            data_v = data.values
+            datai_v = datai.values
 
             if len(data.shape) == 2:
                 # two dimensions xarrays
                 assert (
-                    abs(np.dot(data, datai) - np.dot(datai, data))
+                    abs(np.dot(data_v, datai_v) - np.dot(datai_v, data_v))
                     < 1e-14
                     ).all()
                 assert (
-                    abs(np.dot(data, datai) - np.identity(data.shape[-1]))
+                    abs(np.dot(data_v, datai_v) - np.identity(data.shape[-1]))
                     < 1e-14
                     ).all()
             else:
                 # three dimensions xarrays
                 for i in range(data.shape[0]):
                     assert (
-                        abs(np.dot(data[i], datai[i])
-                            - np.dot(datai[i], data[i]))
+                        abs(np.dot(data_v[i], datai_v[i])
+                            - np.dot(datai_v[i], data_v[i]))
                         < 1e-14
                         ).all()
                     assert (
-                        abs(np.dot(data[i], datai[i])
+                        abs(np.dot(data_v[i], datai_v[i])
                             - np.identity(data.shape[-1]))
                         < 1e-14
                         ).all()

tests/pytest_pysd/pytest_pysd.py

@@ -561,7 +561,7 @@ def test_set_subscripted_timeseries_parameter_with_constant(self, model):
 
         timeseries = list(range(10))
         val_series = [50 + rd for rd in np.random.rand(len(timeseries)
-                                                       ).cumsum()]
+                                                       ).cumsum().tolist()]
         xr_series = [xr.DataArray(val, coords, dims) for val in val_series]
 
         temp_timeseries = pd.Series(index=timeseries, data=val_series)

tests/pytest_pysd/pytest_random.py

@@ -99,6 +99,9 @@ def data_python(self, data_raw, fake_component, random_size):
             }
             expr = builder.build(args).expression
             expr = expr.replace('()', str(random_size))
+            if expr.startswith('float'):
+                # remove float conversion as size is set bigger than 1
+                expr = expr.replace('float(', '')[:-1]
             out[col] = eval(expr)
 
         return pd.DataFrame(out)