Merge pull request #101 from hpsim/feat/on_device_permute

Permute on device implementation, see #101

.github/workflows/build-foam.yml

@@ -88,7 +88,7 @@ jobs:
         mkdir -p build
         cd build
         [ -d "/github/home/$GINKGO_CHECKOUT_VERSION" ] && cp -rp /github/home/$GINKGO_CHECKOUT_VERSION third_party
-        cmake -G Ninja -DOGL_ALLOW_REFERENCE_ONLY=On -DOGL_BUILD_UNITTEST=OFF -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
+        cmake -G Ninja -DOGL_ALLOW_REFERENCE_ONLY=On -DOGL_BUILD_UNITTEST=ON -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
 
     - name: Build OGL
       working-directory: ${{github.workspace}}/build
@@ -114,14 +114,14 @@ jobs:
         name: ogl_build_${{ matrix.OF.path }}
         path: ${{github.workspace}}/build
 
-  # unit_tests:
-  #   needs: [build, setup_build_matrix]
-  #   uses: ./.github/workflows/unit-test.yml
-  #   with:
-  #     path: ${{ matrix.OF.path }}
-  #   strategy:
-  #     fail-fast: false
-  #     matrix: ${{ fromJson(needs.setup_build_matrix.outputs.matrix) }}
+  unit_tests:
+    needs: [build, setup_build_matrix]
+    uses: ./.github/workflows/unit-test.yml
+    with:
+      path: ${{ matrix.OF.path }}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.setup_build_matrix.outputs.matrix) }}
 
   integration_tests:
     needs: [build, setup_build_matrix]

.github/workflows/check_changelog.yaml

@@ -0,0 +1,17 @@
+name: "Pull Request Workflow"
+on:
+  pull_request:
+    # The specific activity types are listed here to include "labeled" and "unlabeled"
+    # (which are not included by default for the "pull_request" trigger).
+    # This is needed to allow skipping enforcement of the changelog in PRs with specific labels,
+    # as defined in the (optional) "skipLabels" property.
+    types: [opened, synchronize, reopened, ready_for_review, labeled, unlabeled]
+
+jobs:
+  # Enforces the update of a changelog file on every pull request
+  changelog:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: dangoslen/changelog-enforcer@v3
+      with:
+        changeLogPath: CHANGELOG.md

.github/workflows/format-check.yml

@@ -16,6 +16,6 @@ jobs:
         fallback-style: 'Mozilla' # optional
     - name: check for todo fixme note
       run: |
-        NTODOS="$(grep -r 'TODO DONT MERGE' | wc -l)"
+        NTODOS="$(grep -r 'TODO DONT MERGE' --exclude-dir=.github | wc -l)"
         echo "Found $NTODOS TODO DONT MERGE notes" 
         ! grep -q -r "TODO DONT MERGE" --exclude-dir=.github  

CHANGELOG.md

@@ -0,0 +1,2 @@
+# 0.5.0 (unreleased)
+- Add on device permutation functionality [PR #101](https://github.com/hpsim/OGL/pull/101) 

CMakeLists.txt

@@ -196,7 +196,14 @@ target_include_directories(
          $ENV{FOAM_SRC}/meshTools/lnInclude $ENV{FOAM_SRC}/OpenFOAM/lnInclude
          $ENV{FOAM_SRC}/OSspecific/POSIX/lnInclude ${CMAKE_CURRENT_SOURCE_DIR})
 
-target_link_libraries(OGL PUBLIC Ginkgo::ginkgo stdc++fs)
+target_link_libraries(OGL
+    PRIVATE
+         $ENV{FOAM_LIBBIN}/libOpenFOAM.so
+         $ENV{FOAM_LIBBIN}/libfiniteVolume.so
+         $ENV{FOAM_LIBBIN}/$ENV{FOAM_MPI}/libPstream.so
+    PUBLIC
+        Ginkgo::ginkgo
+        stdc++fs)
 
 if(${GINKGO_WITH_OGL_EXTENSIONS})
   target_compile_definitions(OGL PRIVATE GINKGO_WITH_OGL_EXTENSIONS=1)

DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H

@@ -147,12 +147,12 @@ struct MatrixInitFunctor {
         gko::array<scalar> device_non_local_values{device_exec,
                                                    *non_local_coeffs.get()};
 
-        auto value_view = val_array::view(
-            device_exec, device_values.get_num_elems(), value_ptr);
+        auto value_view =
+            val_array::view(device_exec, device_values.get_size(), value_ptr);
 
-        auto non_local_view = val_array::view(
-            device_exec, device_non_local_values.get_num_elems(),
-            non_local_value_ptr);
+        auto non_local_view =
+            val_array::view(device_exec, device_non_local_values.get_size(),
+                            non_local_value_ptr);
 
         value_view = device_values;
         non_local_view = device_non_local_values;
@@ -206,7 +206,7 @@ struct MatrixInitFunctor {
             *coeffs.get());
 
         gko::device_matrix_data<scalar, label> non_local_A_data(
-            exec, gko::dim<2>(num_rows, non_local_cols->get_num_elems()),
+            exec, gko::dim<2>(num_rows, non_local_cols->get_size()),
             *non_local_rows.get(), *non_local_cols.get(),
             *non_local_coeffs.get());
 
@@ -239,11 +239,16 @@ private:
     using dist_mtx =
         gko::experimental::distributed::Matrix<scalar, label, label>;
 
+    const objectRegistry &db_;
 
     const label verbose_;
 
     const bool export_;
 
+    const word field_name_;
+
+    const word matrix_format_;
+
     mutable PersistentBase<dist_mtx, MatrixInitFunctor> gkomatrix_;
 
     mutable label prev_solve_iters_ = 0;
@@ -260,14 +265,18 @@ public:
                   const PersistentPartition &partition,
                   const dictionary &controlDict, const word sys_matrix_name,
                   const label verbose)
-        : verbose_(verbose),
+        : db_(db),
+          verbose_(verbose),
           export_(controlDict.lookupOrDefault<Switch>("export", false)),
+          field_name_(sys_matrix_name),
+          matrix_format_(
+              controlDict.lookupOrDefault<word>("matrixFormat", "Coo")),
           gkomatrix_{
               sys_matrix_name + "_matrix", db,
               MatrixInitFunctor(
                   db, exec, partition, col_idxs, row_idxs, coeffs,
                   non_local_col_idxs, non_local_row_idxs, non_local_coeffs,
-                  controlDict.lookupOrDefault<word>("matrixFormat", "Coo"),
+                  matrix_format_,
                   controlDict.lookupOrDefault<Switch>("regenerate", false),
                   verbose_, sys_matrix_name),
               controlDict.lookupOrDefault<Switch>("updateSysMatrix", true),
@@ -281,6 +290,27 @@ public:
 
 
     bool get_export() const { return export_; }
+
+    /** Exports local and non-local matrix to processor?/<time>/.mtx files
+     */
+    void write() const
+    {
+        export_mtx(
+            field_name_,
+            gko::as<
+                gko::experimental::distributed::Matrix<scalar, label, label>>(
+                get().get())
+                ->get_local_matrix(),
+            "local", db_, matrix_format_);
+
+        export_mtx(
+            field_name_,
+            gko::as<
+                gko::experimental::distributed::Matrix<scalar, label, label>>(
+                get().get())
+                ->get_non_local_matrix(),
+            "non_local", db_, matrix_format_);
+    }
 };
 
 }  // namespace Foam

DevicePersistent/ExecutorHandler/ExecutorHandler.H

@@ -77,8 +77,7 @@ struct ExecutorInitFunctor {
             };
 
             return gko::share(gko::CudaExecutor::create(
-                device_id_ % gko::CudaExecutor::get_num_devices(), host_exec,
-                false, gko::allocation_mode::device));
+                device_id_ % gko::CudaExecutor::get_num_devices(), host_exec));
         }
         if (executor_name_ == "dpcpp") {
             if (version.dpcpp_version.tag == not_compiled_tag) {
@@ -98,10 +97,8 @@ struct ExecutorInitFunctor {
                        "with HIP backend enabled."
                     << abort(FatalError);
             };
-
             auto ret = gko::share(gko::HipExecutor::create(
-                device_id_ % gko::HipExecutor::get_num_devices(), host_exec,
-                true));
+                device_id_ % gko::HipExecutor::get_num_devices(), host_exec));
             return ret;
         }
         if (executor_name_ == "omp") {

DevicePersistent/Vector/Vector.H

@@ -114,6 +114,10 @@ class PersistentVector
     using dist_vec = gko::experimental::distributed::Vector<scalar>;
     using vec = gko::matrix::Dense<scalar>;
 
+    const objectRegistry &db_;
+
+    const word name_;
+
     const T *memory_;
 
     const PersistentPartition partition_;
@@ -150,35 +154,16 @@ public:
               VectorInitFunctor<T>(exec, name, partition, memory, verbose,
                                    init_on_device),
               update, verbose),
+          db_(db),
+          name_(name),
           memory_(memory),
           partition_(partition),
           exec_(exec),
           update_(update)
     {}
 
-    // label get_global_size() const { return partition_.size(); }
-
-    bool get_update() const { return update_; }
-
-    T *get_data() const { return this->get_persistent_object()->get_data(); }
-
-    void set_data(T *data)
-    {
-        this->get_persistent_object()->get_data() = data;
-    };
-
-    const T *get_const_data() const
-    {
-        return this->get_persistent_object()->get_const_data();
-    };
-
-
-    std::shared_ptr<gko::experimental::distributed::Vector<T>> get_vector()
-        const
-    {
-        return this->get_persistent_object();
-    }
-
+    /** Copies the content of the distributed vector back to the original source
+     **/
     void copy_back()
     {
         const auto local_size = partition_.get_local_size();
@@ -204,7 +189,38 @@ public:
         to_view = host_buffer_view;
     }
 
+    /** Writes the content of the distributed vector to disk
+     **
+     ** Data is stored as .mtx file under processor?/<time>/<name_>.mtx
+     **/
+    void write() const
+    {
+        export_vec(name_, get_vector()->get_local_vector(), db_);
+    }
+
+    // getter and setter
+
+    bool get_update() const { return update_; }
+
+    T *get_data() const { return this->get_persistent_object()->get_data(); }
+
+    void set_data(T *data)
+    {
+        this->get_persistent_object()->get_data() = data;
+    };
+
+    const T *get_const_data() const
+    {
+        return this->get_persistent_object()->get_const_data();
+    };
+
     const ExecutorHandler &get_exec_handler() const { return exec_; }
+
+    std::shared_ptr<gko::experimental::distributed::Vector<T>> get_vector()
+        const
+    {
+        return this->get_persistent_object();
+    }
 };
 
 }  // namespace Foam