Hyperparameter optimisation diverges for mean-models in cross checks

[Moyoxkit]

When using the new new cross check analysis tool with a mean model:

crosscheck_bins.build_emulators(hide_progress=False,fit_model="polynomial")
crosscheck.build_emulators(hide_progress=False,fit_model="linear")

This leads to the following error:

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-33-9f97797a4d66> in <module>
----> 1 crosscheck_bins.build_emulators(hide_progress=False,fit_model="polynomial")
      2 crosscheck.build_emulators(hide_progress=False,fit_model="linear")

/opt/anaconda3/lib/python3.7/site-packages/swiftemulator/sensitivity/cross_check_bins.py in build_emulators(self, kernel, fit_model, lasso_model_alpha, polynomial_degree, hide_progress)
    119                 fit_model=fit_model,
    120                 lasso_model_alpha=lasso_model_alpha,
--> 121                 polynomial_degree=polynomial_degree,
    122             )
    123 

/opt/anaconda3/lib/python3.7/site-packages/swiftemulator/emulators/gaussian_process_bins.py in fit_model(self, kernel, fit_model, lasso_model_alpha, polynomial_degree)
    261                 fun=negative_log_likelihood,
    262                 x0=gaussian_process.get_parameter_vector(),
--> 263                 jac=grad_negative_log_likelihood,
    264             )
    265 

/opt/anaconda3/lib/python3.7/site-packages/scipy/optimize/_minimize.py in minimize(fun, x0, args, method, jac, hess, hessp, bounds, constraints, tol, callback, options)
    610         return _minimize_cg(fun, x0, args, jac, callback, **options)
    611     elif meth == 'bfgs':
--> 612         return _minimize_bfgs(fun, x0, args, jac, callback, **options)
    613     elif meth == 'newton-cg':
    614         return _minimize_newtoncg(fun, x0, args, jac, hess, hessp, callback,

/opt/anaconda3/lib/python3.7/site-packages/scipy/optimize/optimize.py in _minimize_bfgs(fun, x0, args, jac, callback, gtol, norm, eps, maxiter, disp, return_all, finite_diff_rel_step, **unknown_options)
   1134             alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = \
   1135                      _line_search_wolfe12(f, myfprime, xk, pk, gfk,
-> 1136                                           old_fval, old_old_fval, amin=1e-100, amax=1e100)
   1137         except _LineSearchError:
   1138             # Line search failed to find a better solution.

/opt/anaconda3/lib/python3.7/site-packages/scipy/optimize/optimize.py in _line_search_wolfe12(f, fprime, xk, pk, gfk, old_fval, old_old_fval, **kwargs)
    934     ret = line_search_wolfe1(f, fprime, xk, pk, gfk,
    935                              old_fval, old_old_fval,
--> 936                              **kwargs)
    937 
    938     if ret[0] is not None and extra_condition is not None:

/opt/anaconda3/lib/python3.7/site-packages/scipy/optimize/linesearch.py in line_search_wolfe1(f, fprime, xk, pk, gfk, old_fval, old_old_fval, args, c1, c2, amax, amin, xtol)
     96     stp, fval, old_fval = scalar_search_wolfe1(
     97             phi, derphi, old_fval, old_old_fval, derphi0,
---> 98             c1=c1, c2=c2, amax=amax, amin=amin, xtol=xtol)
     99 
    100     return stp, fc[0], gc[0], fval, old_fval, gval[0]

/opt/anaconda3/lib/python3.7/site-packages/scipy/optimize/linesearch.py in scalar_search_wolfe1(phi, derphi, phi0, old_phi0, derphi0, c1, c2, amax, amin, xtol)
    170         if task[:2] == b'FG':
    171             alpha1 = stp
--> 172             phi1 = phi(stp)
    173             derphi1 = derphi(stp)
    174         else:

/opt/anaconda3/lib/python3.7/site-packages/scipy/optimize/linesearch.py in phi(s)
     82     def phi(s):
     83         fc[0] += 1
---> 84         return f(xk + s*pk, *args)
     85 
     86     def derphi(s):

/opt/anaconda3/lib/python3.7/site-packages/scipy/optimize/_differentiable_functions.py in fun(self, x)
    180         if not np.array_equal(x, self.x):
    181             self._update_x_impl(x)
--> 182         self._update_fun()
    183         return self.f
    184 

/opt/anaconda3/lib/python3.7/site-packages/scipy/optimize/_differentiable_functions.py in _update_fun(self)
    164     def _update_fun(self):
    165         if not self.f_updated:
--> 166             self._update_fun_impl()
    167             self.f_updated = True
    168 

/opt/anaconda3/lib/python3.7/site-packages/scipy/optimize/_differentiable_functions.py in update_fun()
     71 
     72         def update_fun():
---> 73             self.f = fun_wrapped(self.x)
     74 
     75         self._update_fun_impl = update_fun

/opt/anaconda3/lib/python3.7/site-packages/scipy/optimize/_differentiable_functions.py in fun_wrapped(x)
     68         def fun_wrapped(x):
     69             self.nfev += 1
---> 70             return fun(x, *args)
     71 
     72         def update_fun():

/opt/anaconda3/lib/python3.7/site-packages/swiftemulator/emulators/gaussian_process_bins.py in negative_log_likelihood(p)
    251             def negative_log_likelihood(p):
    252                 gaussian_process.set_parameter_vector(p)
--> 253                 return -gaussian_process.log_likelihood(dependent_variables)
    254 
    255             def grad_negative_log_likelihood(p):

/opt/anaconda3/lib/python3.7/site-packages/george/gp.py in log_likelihood(self, y, quiet)
    358 
    359         """
--> 360         if not self.recompute(quiet=quiet):
    361             return -np.inf
    362         try:

/opt/anaconda3/lib/python3.7/site-packages/george/gp.py in recompute(self, quiet, **kwargs)
    330                 # Update the model making sure that we store the original
    331                 # ordering of the points.
--> 332                 self.compute(self._x, np.sqrt(self._yerr2), **kwargs)
    333             except (ValueError, LinAlgError):
    334                 if quiet:

/opt/anaconda3/lib/python3.7/site-packages/george/gp.py in compute(self, x, yerr, **kwargs)
    307         # Include the white noise term.
    308         yerr = np.sqrt(self._yerr2 + np.exp(self._call_white_noise(self._x)))
--> 309         self.solver.compute(self._x, yerr, **kwargs)
    310 
    311         self._const = -0.5 * (len(self._x) * np.log(2 * np.pi) +

/opt/anaconda3/lib/python3.7/site-packages/george/solvers/basic.py in compute(self, x, yerr)
     66 
     67         # Factor the matrix and compute the log-determinant.
---> 68         self._factor = (cholesky(K, overwrite_a=True, lower=False), False)
     69         self.log_determinant = 2 * np.sum(np.log(np.diag(self._factor[0])))
     70         self.computed = True

/opt/anaconda3/lib/python3.7/site-packages/scipy/linalg/decomp_cholesky.py in cholesky(a, lower, overwrite_a, check_finite)
     87     """
     88     c, lower = _cholesky(a, lower=lower, overwrite_a=overwrite_a, clean=True,
---> 89                          check_finite=check_finite)
     90     return c
     91 

/opt/anaconda3/lib/python3.7/site-packages/scipy/linalg/decomp_cholesky.py in _cholesky(a, lower, overwrite_a, clean, check_finite)
     15     """Common code for cholesky() and cho_factor()."""
     16 
---> 17     a1 = asarray_chkfinite(a) if check_finite else asarray(a)
     18     a1 = atleast_2d(a1)
     19 

/opt/anaconda3/lib/python3.7/site-packages/numpy/lib/function_base.py in asarray_chkfinite(a, dtype, order)
    484     if a.dtype.char in typecodes['AllFloat'] and not np.isfinite(a).all():
    485         raise ValueError(
--> 486             "array must not contain infs or NaNs")
    487     return a
    488 

ValueError: array must not contain infs or NaNs

This usually happens on the first iteration of the cross check loop. When excluding the first simulation it sometimes get's a bit farther but it always get stuck at some point. This is true for both the binned and non-binned case.