Add switches for some functions to compile differently for host vs device

CHANGELOG.md

@@ -25,6 +25,7 @@ Release is in preparation for JOSS publication.
 - [[PR330]](https://github.com/lanl/singularity-eos/pull/330) Piecewise grids for Spiner EOS.
 
 ### Fixed (Repair bugs, etc)
+- [[PR434]](https://github.com/lanl/singularity-eos/pull/434) Fix failure of eospac to build on HIP and segfaults with Evalaute
 - [[PR424]](https://github.com/lanl/singularity-eos/pull/424) Fix for variant patch: point to correct patch file
 - [[PR420]](https://github.com/lanl/singularity-eos/pull/420) Fix broken test_get_sg_eos
 - [[PR417]](https://github.com/lanl/singularity-eos/pull/417) Bugs in shared memory related to eospac resolved

CMakeLists.txt

@@ -32,6 +32,7 @@ list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake")
 set(SINGULARITY_GOLDFILES_VERSION "goldfiles-1.8.0")
 set(SINGULARITY_GOLDFILE_HASH
     249772f3314c4b6b9386aa08895280698ae905ef188f4383b819f28c1484603b)
+set(DOWNLOAD_EXTRACT_TIMESTAMP ON)
 
 # ------------------------------------------------------------------------------#
 # Options

doc/sphinx/src/using-eos.rst

@@ -36,9 +36,10 @@ conditions. The type of parallelism used depends on how
 ``singularity-eos`` is compiled. If the ``Kokkos`` backend is used,
 any parallel dispatch supported by ``Kokkos`` is supported.
 
-A more generic version of the vector calls exists in the ``Evaluate``
-method, which allows the user to specify arbitrary parallel dispatch
-models by writing their own loops. See the relevant section below.
+A more generic version of the vector calls exists in the
+``EvaluateHost`` and ``EvaluateDevice`` methods, which allows the user
+to specify arbitrary parallel dispatch models by writing their own
+loops. See the relevant section below.
 
 Serialization and shared memory
 --------------------------------
@@ -494,31 +495,38 @@ available member function.
    eos.TemperatureFromDensityInternalEnergy(density.data(), energy.data(), temperature.data(),
                                             scratch.data(), density.size());
 
-The Evaluate Method
-~~~~~~~~~~~~~~~~~~~
+The Evaluate Methods
+~~~~~~~~~~~~~~~~~~~~~~
 
-A special call related to the vector calls is the ``Evaluate``
-method. The ``Evaluate`` method requests the EOS object to evaluate
-almost arbitrary code, but in a way where the type of the underlying
-EOS object is resolved *before* this arbitrary code is evaluated. This
-means the code required to resolve the type of the variant is only
-executed *once* per ``Evaluate`` call. This can enable composite EOS
-calls, non-standard vector calls, and vector calls with non-standard
-loop structure.
+A pair of special call related to the vector calls are the
+``EvaluateHost`` and ``EvaluateDevice`` methods. These methods request
+the EOS object to evaluate almost arbitrary code, but in a way where
+the type of the underlying EOS object is resolved *before* this
+arbitrary code is evaluated. This means the code required to resolve
+the type of the variant is only executed *once* per ``Evaluate``
+call. This can enable composite EOS calls, non-standard vector calls,
+and vector calls with non-standard loop structure.
 
-The ``Evaluate`` call has the signature
+The ``EvaluateHost`` call has the signature
+
+.. code-block:: cpp
+
+  template<typename Functor_t>
+  void Evaluate(Functor_t f);
+
+and the ``EvaluateDevice`` method has the signature
 
 .. code-block:: cpp
 
   template<typename Functor_t>
   PORTABLE_INLINE_FUNCTION
   void Evaluate(Functor_t f);
 
+
 where a ``Functor_t`` is a class that *must* provide a ``void
-operator() const`` method templated on EOS type. ``Evaluate`` is
-decorated so that it may be evaluated on either host or device,
-depending on desired use-case. Alternatively, you may use an anonymous
-function with an `auto` argument as the input, e.g.,
+operator() const`` method templated on EOS type. Alternatively, you
+may use an anonymous function with an `auto` argument as the input,
+e.g.,
 
 .. code-block:: cpp
 
@@ -531,7 +539,9 @@ function with an `auto` argument as the input, e.g.,
   with GPUs it can produce very unintuitive behaviour. We recommend
   you only make the ``operator()`` non-const if you really know what
   you're doing. And in the anonymous function case, we recommend you
-  capture by value, not reference.
+  capture by value, not reference. ``EvaluateDevice`` does not support
+  side effects at all and you must pass your functors in by value in
+  that case.
 
 To see the utlity of the ``Evaluate`` function, it's probably just
 easiest to provide an example. The following code evaluates the EOS on
@@ -583,17 +593,17 @@ is summed using the ``Kokkos::parallel_reduce`` functionality in the
   CheckPofRE my_op(P, rho, sie, N);
 
   // Here we call the evaluate function
-  eos.Evaluate(my_op);
+  eos.EvaluateHost(my_op);
 
   // The above two lines could have been called "in-one" with:
-  // eos.Evaluate(CheckPofRE(P, rho, sie, N));
+  // eos.EvaluateHost(CheckPofRE(P, rho, sie, N));
 
 Alternatively, you could eliminate the functor and use an anonymous
 function with:
 
 .. code-block:: cpp
 
-  eos.Evaluate([=](auto eos) {
+  eos.EvaluateHost([=](auto eos) {
     Real tot_diff;
     Kokkos::parallel_reduce(
         "MyCheckPofRE", N_,

singularity-eos/eos/eos_base.hpp

@@ -148,8 +148,13 @@ class EosBase {
 
   // Generic evaluator
   template <typename Functor_t>
-  constexpr void Evaluate(Functor_t &f) const {
-    CRTP copy = *(static_cast<CRTP const *>(this));
+  PORTABLE_INLINE_FUNCTION void EvaluateDevice(const Functor_t f) const {
+    const CRTP copy = *(static_cast<CRTP const *>(this));
+    f(copy);
+  }
+  template <typename Functor_t>
+  void EvaluateHost(Functor_t &f) const {
+    const CRTP copy = *(static_cast<CRTP const *>(this));
     f(copy);
   }
 

singularity-eos/eos/eos_eospac.hpp

@@ -1274,43 +1274,52 @@ inline std::size_t EOSPAC::SetDynamicMemory(char *src, const SharedMemSettings &
 }
 inline std::size_t EOSPAC::SharedMemorySizeInBytes() const { return shared_size_; }
 
-SG_PIF_NOWARN
 template <typename Indexer_t>
 PORTABLE_INLINE_FUNCTION Real EOSPAC::TemperatureFromDensityInternalEnergy(
     const Real rho, const Real sie, Indexer_t &&lambda) const {
+#if defined(__CUDA_ARCH__) || __HIP_DEVICE_COMPILE__
+  EOS_ERROR("EOSPAC calls not supported on device\n");
+  return 0; // compiler happy
+#else
   using namespace EospacWrapper;
   EOS_REAL R[1] = {rho}, E[1] = {sieToSesame(sie)}, T[1], dTdr[1], dTde[1];
   EOS_INTEGER nxypairs = 1;
   EOS_INTEGER table = TofRE_table_;
   eosSafeInterpolate(&table, nxypairs, R, E, T, dTdr, dTde, "TofRE", Verbosity::Quiet);
   return Real(temperatureFromSesame(T[0]));
+#endif // ON_DEVICE
 }
 
-SG_PIF_NOWARN
 template <typename Indexer_t>
 PORTABLE_INLINE_FUNCTION Real EOSPAC::PressureFromDensityTemperature(
     const Real rho, const Real temp, Indexer_t &&lambda) const {
+#if defined(__CUDA_ARCH__) || __HIP_DEVICE_COMPILE__
+  EOS_ERROR("EOSPAC calls not supported on device\n");
+  return 0; // compiler happy
+#else
   using namespace EospacWrapper;
   EOS_REAL R[1] = {rho}, P[1], T[1] = {temperatureToSesame(temp)}, dPdr[1], dPdT[1];
   EOS_INTEGER nxypairs = 1;
   EOS_INTEGER table = PofRT_table_;
   eosSafeInterpolate(&table, nxypairs, R, T, P, dPdr, dPdT, "PofRT", Verbosity::Quiet);
   return Real(pressureFromSesame(P[0]));
+#endif // ON DEVICE
 }
 
-SG_PIF_NOWARN
 template <typename Indexer_t>
 PORTABLE_INLINE_FUNCTION Real EOSPAC::EntropyFromDensityTemperature(
     const Real rho, const Real temperature, Indexer_t &&lambda) const {
   EntropyIsNotEnabled("EOSPAC");
   return 1.0;
 }
 
-SG_PIF_NOWARN
 template <typename Indexer_t>
 PORTABLE_INLINE_FUNCTION void
 EOSPAC::FillEos(Real &rho, Real &temp, Real &sie, Real &press, Real &cv, Real &bmod,
                 const unsigned long output, Indexer_t &&lambda) const {
+#if defined(__CUDA_ARCH__) || __HIP_DEVICE_COMPILE__
+  EOS_ERROR("EOSPAC calls not supported on device\n");
+#else
   using namespace EospacWrapper;
   EOS_REAL R[1] = {rho}, T[1] = {temperatureToSesame(temp)};
   EOS_REAL E[1] = {sieToSesame(sie)}, P[1] = {pressureToSesame(press)};
@@ -1388,42 +1397,61 @@ EOSPAC::FillEos(Real &rho, Real &temp, Real &sie, Real &press, Real &cv, Real &b
     }
     bmod = bulkModulusFromSesame(std::max(BMOD, 0.0));
   }
+#endif // ON DEVICE
 }
 
-SG_PIF_NOWARN
 template <typename Indexer_t>
 PORTABLE_INLINE_FUNCTION Real EOSPAC::InternalEnergyFromDensityTemperature(
     const Real rho, const Real temp, Indexer_t &&lambda) const {
+#if defined(__CUDA_ARCH__) || __HIP_DEVICE_COMPILE__
+  EOS_ERROR("EOSPAC calls not supported on device\n");
+  return 0; // compiler happy
+#else
   using namespace EospacWrapper;
   Real RHO = rho, TEMP = temp, sie, press, cv, bmod;
   const unsigned long output = thermalqs::specific_internal_energy;
   FillEos(RHO, TEMP, sie, press, cv, bmod, output, lambda);
   return sie;
+#endif // ON DEVICE
 }
-SG_PIF_NOWARN
+
 template <typename Indexer_t>
 PORTABLE_INLINE_FUNCTION Real EOSPAC::BulkModulusFromDensityTemperature(
     const Real rho, const Real temp, Indexer_t &&lambda) const {
+#if defined(__CUDA_ARCH__) || __HIP_DEVICE_COMPILE__
+  EOS_ERROR("EOSPAC calls not supported on device\n");
+  return 0; // compiler happy
+#else
   using namespace EospacWrapper;
   Real RHO = rho, TEMP = temp, sie, press, cv, bmod;
   const unsigned long output = thermalqs::bulk_modulus;
   FillEos(RHO, TEMP, sie, press, cv, bmod, output, lambda);
   return bmod;
+#endif // ON DEVICE
 }
-SG_PIF_NOWARN
+
 template <typename Indexer_t>
 PORTABLE_INLINE_FUNCTION Real EOSPAC::SpecificHeatFromDensityTemperature(
     const Real rho, const Real temp, Indexer_t &&lambda) const {
+#if defined(__CUDA_ARCH__) || __HIP_DEVICE_COMPILE__
+  EOS_ERROR("EOSPAC calls not supported on device\n");
+  return 0; // compiler happy
+#else
   using namespace EospacWrapper;
   Real RHO = rho, TEMP = temp, sie, press, cv, bmod;
   const unsigned long output = thermalqs::specific_heat;
   FillEos(RHO, TEMP, sie, press, cv, bmod, output, lambda);
   return cv;
+#endif // ON DEVICE
 }
-SG_PIF_NOWARN
+
 template <typename Indexer_t>
 PORTABLE_INLINE_FUNCTION Real EOSPAC::PressureFromDensityInternalEnergy(
     const Real rho, const Real sie, Indexer_t &&lambda) const {
+#if defined(__CUDA_ARCH__) || __HIP_DEVICE_COMPILE__
+  EOS_ERROR("EOSPAC calls not supported on device\n");
+  return 0; // compiler happy
+#else
   using namespace EospacWrapper;
   EOS_INTEGER options[]{EOS_Y_CONVERT, EOS_F_CONVERT};
   EOS_REAL values[]{sieFromSesame(1.0), pressureFromSesame(1.0)};
@@ -1435,11 +1463,16 @@ PORTABLE_INLINE_FUNCTION Real EOSPAC::PressureFromDensityInternalEnergy(
                      options, values, nopts);
 
   return Real(P[0]);
+#endif // ON DEVICE
 }
-SG_PIF_NOWARN
+
 template <typename Indexer_t>
 PORTABLE_INLINE_FUNCTION Real
 EOSPAC::MinInternalEnergyFromDensity(const Real rho, Indexer_t &&lambda) const {
+#if defined(__CUDA_ARCH__) || __HIP_DEVICE_COMPILE__
+  EOS_ERROR("EOSPAC calls not supported on device\n");
+  return 0; // compiler happy
+#else
   using namespace EospacWrapper;
   EOS_INTEGER options[]{EOS_F_CONVERT};
   EOS_REAL values[]{sieFromSesame(1.0)};
@@ -1451,36 +1484,55 @@ EOSPAC::MinInternalEnergyFromDensity(const Real rho, Indexer_t &&lambda) const {
                      options, values, nopts);
 
   return Real(S[0]);
+#endif // ON DEVICE
 }
-SG_PIF_NOWARN
+
 template <typename Indexer_t>
 PORTABLE_INLINE_FUNCTION Real EOSPAC::EntropyFromDensityInternalEnergy(
     const Real rho, const Real sie, Indexer_t &&lambda) const {
+#if defined(__CUDA_ARCH__) || __HIP_DEVICE_COMPILE__
+  EOS_ERROR("EOSPAC calls not supported on device\n");
+  return 0; // compiler happy
+#else
   using namespace EospacWrapper;
   const Real temp = TemperatureFromDensityInternalEnergy(rho, sie, lambda);
   return EntropyFromDensityTemperature(rho, temp, lambda);
+#endif // ON DEVICE
 }
-SG_PIF_NOWARN
+
 template <typename Indexer_t>
 PORTABLE_INLINE_FUNCTION Real EOSPAC::SpecificHeatFromDensityInternalEnergy(
     const Real rho, const Real sie, Indexer_t &&lambda) const {
+#if defined(__CUDA_ARCH__) || __HIP_DEVICE_COMPILE__
+  EOS_ERROR("EOSPAC calls not supported on device\n");
+  return 0; // compiler happy
+#else
   using namespace EospacWrapper;
   Real temp = TemperatureFromDensityInternalEnergy(rho, sie, lambda);
   return SpecificHeatFromDensityTemperature(rho, temp, lambda);
+#endif // ON DEVICE
 }
-SG_PIF_NOWARN
+
 template <typename Indexer_t>
 PORTABLE_INLINE_FUNCTION Real EOSPAC::BulkModulusFromDensityInternalEnergy(
     const Real rho, const Real sie, Indexer_t &&lambda) const {
+#if defined(__CUDA_ARCH__) || __HIP_DEVICE_COMPILE__
+  EOS_ERROR("EOSPAC calls not supported on device\n");
+  return 0; // compiler happy
+#else
   using namespace EospacWrapper;
   Real temp = TemperatureFromDensityInternalEnergy(rho, sie, lambda);
   return BulkModulusFromDensityTemperature(rho, temp, lambda);
+#endif // ON DEVICE
 }
 
-SG_PIF_NOWARN
 template <typename Indexer_t>
 PORTABLE_INLINE_FUNCTION Real EOSPAC::GruneisenParamFromDensityTemperature(
     const Real rho, const Real temperature, Indexer_t &&lambda) const {
+#if defined(__CUDA_ARCH__) || __HIP_DEVICE_COMPILE__
+  EOS_ERROR("EOSPAC calls not supported on device\n");
+  return 0; // compiler happy
+#else
   using namespace EospacWrapper;
   EOS_REAL R[1] = {rho}, T[1] = {temperatureToSesame(temperature)};
   EOS_REAL E[1], P[1], dx[1], dy[1];
@@ -1494,19 +1546,28 @@ PORTABLE_INLINE_FUNCTION Real EOSPAC::GruneisenParamFromDensityTemperature(
   DPDT = dy[0];
   DPDE = DPDT / DEDT;
   return robust::ratio(pressureFromSesame(sieToSesame(DPDE)), rho);
+#endif // ON DEVICE
 }
-SG_PIF_NOWARN
+
 template <typename Indexer_t>
 PORTABLE_INLINE_FUNCTION Real EOSPAC::GruneisenParamFromDensityInternalEnergy(
     const Real rho, const Real sie, Indexer_t &&lambda) const {
+#if defined(__CUDA_ARCH__) || __HIP_DEVICE_COMPILE__
+  EOS_ERROR("EOSPAC calls not supported on device\n");
+  return 0; // compiler happy
+#else
   Real temperature = TemperatureFromDensityInternalEnergy(rho, sie, lambda);
   return GruneisenParamFromDensityTemperature(rho, temperature, lambda);
+#endif // ON DEVICE
 }
 
 SG_PIF_NOWARN
 template <typename Indexer_t>
 PORTABLE_INLINE_FUNCTION void EOSPAC::DensityEnergyFromPressureTemperature(
     const Real press, const Real temp, Indexer_t &&lambda, Real &rho, Real &sie) const {
+#if defined(__CUDA_ARCH__) || __HIP_DEVICE_COMPILE__
+  EOS_ERROR("EOSPAC calls not supported on device\n");
+#else
   using namespace EospacWrapper;
   EOS_REAL P[1] = {pressureToSesame(press)};
   EOS_REAL T[1] = {temperatureToSesame(temp)};
@@ -1521,14 +1582,17 @@ PORTABLE_INLINE_FUNCTION void EOSPAC::DensityEnergyFromPressureTemperature(
   table = EofRT_table_;
   eosSafeInterpolate(&table, nxypairs, R, T, E, dx, dy, "EofPRT", Verbosity::Quiet);
   sie = sieFromSesame(E[0]);
+#endif // ON DEVICE
 }
 
-SG_PIF_NOWARN
 template <typename Indexer_t>
 PORTABLE_INLINE_FUNCTION void
 EOSPAC::ValuesAtReferenceState(Real &rho, Real &temp, Real &sie, Real &press, Real &cv,
                                Real &bmod, Real &dpde, Real &dvdt,
                                Indexer_t &&lambda) const {
+#if defined(__CUDA_ARCH__) || __HIP_DEVICE_COMPILE__
+  EOS_ERROR("EOSPAC calls not supported on device\n");
+#else
   using namespace EospacWrapper;
   rho = rho_ref_;
   temp = temp_ref_;
@@ -1538,6 +1602,7 @@ EOSPAC::ValuesAtReferenceState(Real &rho, Real &temp, Real &sie, Real &press, Re
   bmod = bmod_ref_;
   dpde = dpde_ref_;
   dvdt = dvdt_ref_;
+#endif // ON DEVICE
 }
 
 } // namespace singularity