Merge pull request #81 from hpsim/cyclic-fix-asym

remove comment

.github/workflows/build-foam.yml

@@ -1,5 +1,6 @@
 name: Build OpenFOAM (ESI|FOUNDATION)
 on:
+  push:
   pull_request:
     types: synchronize
   schedule:
@@ -45,6 +46,7 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
+
     - name: Install ninja-build tool
       # You may pin to the exact commit or the version.
       # uses: seanmiddleditch/gha-setup-ninja@16b940825621068d98711680b6c3ff92201f8fc0
@@ -62,6 +64,12 @@ jobs:
         echo "FOAM_SRC=$FOAM_INST_DIR/${{matrix.OF.path}}/src"     >> $GITHUB_ENV
         echo "Ginkgo_DIR=${{env.FOAM_INST_DIR}}/root-${{matrix.OF.version}}/platforms/linux64GccDPInt32Opt/lib/cmake/Ginkgo" >> $GITHUB_ENV
 
+    - name: Cache OpenFOAM
+      uses: actions/cache@v3
+      with:
+        key: ${{ matrix.OF.path }}
+        path: /home/runner/work/_temp/_github_home
+
     - name: Configure CMake
       # Use a bash shell so we can use the same syntax for environment variable
       # access regardless of the host operating system
@@ -71,7 +79,7 @@ jobs:
         mkdir -p $FOAM_USER_LIBBIN
         mkdir build
         cd build
-        cmake -G Ninja -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
+        cmake -G Ninja -DOGL_DATA_VALIDATION=On -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
 
     - name: Build OGL
       working-directory: ${{github.workspace}}/build
@@ -84,38 +92,22 @@ jobs:
       run: ctest -C $BUILD_TYPE
 
     - name: Install OGL
-      working-directory: ${{github.workspace}}/build
+      working-directory: ${{github.workspace}}
       run: |
         cmake --install ./build
 
+    - name: Copy OGL to cached folder
+      working-directory: ${{github.workspace}}
+      run: |
+        cp -r $FOAM_USER_LIBBIN /github/home
+
     - name: Archive production artifacts
       uses: actions/upload-artifact@v3
       if: failure()
       with:
         name: ogl_build_${{ matrix.OF.path }}
         path: ${{github.workspace}}/build
 
-  run-integration-tests:
-    needs: [build-ogl]
-    runs-on: ubuntu-latest
-    steps:
-    - name: Cache OpenFOAM
-      uses: actions/cache@v3
-      with:
-        key: ${{ matrix.OF.path }}
-        path: $HOME/OpenFOAM/
-
-    - name: Source OF enviroment
-      run: |
-        echo "$HOME/.esi_env=$FOAM_INST_DIR/env_file"            >> $GITHUB_ENV
-        echo "WM_PROJECT_DIR=$FOAM_INST_DIR/${{matrix.OF.path}}" >> $GITHUB_ENV
-        echo "PLATFORM=$FOAM_INST_DIR/${{matrix.OF.path}}/platforms/linux64GccDPInt32Opt" >> $GITHUB_ENV
-        echo "FOAM_USER_PATH=${{env.FOAM_INST_DIR}}/${{github.actor}}-${{matrix.OF.version}}" >> $GITHUB_ENV
-        cat $HOME/.esi_env >> $GITHUB_ENV
-
-    - name: Install OBR
-      run: |
-          git clone  github.com/hpsim/OBR.git
-          cd OBR
-          pip install .
-
+  tests:
+    needs: build-ogl
+    uses: ./.github/workflows/integration-tests.yml

.github/workflows/integration-tests.yml

@@ -0,0 +1,111 @@
+name: Integration Tests
+on: ["workflow_call"]
+
+env:
+  BUILD_TYPE: Release
+  FOAM_INST_DIR: /root/OpenFOAM
+  WM_PROJECT: OpenFOAM
+  WM_OPTIONS: linux64GccDPInt32Opt
+  WM_COMPILER_TYPE: system
+  WM_COMPILER: Gcc
+  WM_PRECISION_OPTION: DP
+  WM_LABEL_SIZE: 32
+  WM_COMPILE_OPTION: Opt
+  WM_OSTYPE: POSIX
+  WM_ARCH: linux64
+  WM_ARCH_OPTION: 64
+  WM_LINK_LANGUAGE: c++
+  WM_LABEL_OPTION:  Int32
+  OMPI_ALLOW_RUN_AS_ROOT: 1
+  OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1
+
+jobs:
+  integtest:
+    runs-on: ubuntu-latest
+    container: greole/ofbase
+    strategy:
+      fail-fast: false
+      matrix:
+        OF: [
+          {
+            version: "v2212",
+            path: "openfoam",
+            cyclic_case: "LES/periodicPlaneChannel" 
+            },
+          {
+            version: "10",
+            path: "OpenFOAM-10",
+            cyclic_case: "LES/channel395" 
+            }]
+        Case: ["cavity", "channel"]
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Source OF enviroment
+      run: |
+        mkdir -p $FOAM_INST_DIR
+        echo "WM_PROJECT_DIR=$FOAM_INST_DIR/${{matrix.OF.path}}" >> $GITHUB_ENV
+        echo "FOAM_USER_PATH=$FOAM_INST_DIR/root-${{matrix.OF.version}}"     >> $GITHUB_ENV
+        echo "FOAM_ETC=$FOAM_INST_DIR/${{matrix.OF.path}}/etc" >> $GITHUB_ENV
+        echo "FOAM_TUTORIALS=$FOAM_INST_DIR/${{matrix.OF.path}}/tutorials" >> $GITHUB_ENV
+        echo "FOAM_USER_LIBBIN=$FOAM_INST_DIR/root-${{matrix.OF.version}}/platforms/linux64GccDPInt32Opt/lib"     >> $GITHUB_ENV
+        echo "FOAM_SRC=$FOAM_INST_DIR/${{matrix.OF.path}}/src"     >> $GITHUB_ENV
+        echo "Ginkgo_DIR=${{env.FOAM_INST_DIR}}/root-${{matrix.OF.version}}/platforms/linux64GccDPInt32Opt/lib/cmake/Ginkgo" >> $GITHUB_ENV
+        echo "GINKGO_EXECUTOR=reference" >> $GITHUB_ENV
+        echo "CYCLIC_CASE=${{matrix.OF.cyclic_case}}" >> $GITHUB_ENV
+        echo "OMPI_MCA_btl_vader_single_copy_mechanism=none" >> $GITHUB_ENV
+
+    - name: Cache OpenFOAM
+      uses: actions/cache@v3
+      with:
+        key: ${{ matrix.OF.path }}
+        path: /home/runner/work/_temp/_github_home
+
+    - name: Copy OGL to cached folder
+      working-directory: ${{github.workspace}}
+      run: |
+        mkdir -p $FOAM_USER_LIBBIN
+        cp -r /github/home/lib/* $FOAM_USER_LIBBIN 
+
+    - name: Install OBR
+      working-directory: /tmp
+      run: |
+          apt-get update
+          apt install -y pip python3.10-venv rename
+          git clone https://github.com/hpsim/OBR.git -b sandbox
+          cd OBR
+          python3 -m venv obr-env
+          . obr-env/bin/activate
+          pip install .
+
+    - name: Run test case
+      working-directory: /github/home/
+      run: |
+        export PATH=$WM_PROJECT_DIR/platforms/linux64GccDPInt32Opt/bin:$PATH
+        export LD_LIBRARY_PATH=$FOAM_USER_LIBBIN:$WM_PROJECT_DIR/platforms/linux64GccDPInt32Opt/lib:$WM_PROJECT_DIR/platforms/linux64GccDPInt32Opt/lib/sys-openmpi/:$WM_PROJECT_DIR/platforms/linux64GccDPInt32Opt/lib/dummy/:$LD_LIBRARY_PATH
+        export OBR_RUN_CMD="mpirun --bind-to core --map-by core -np {np} {solver} -parallel -case {path}/case >  {path}/case/{solver}_{timestamp}.log 2>&1"
+
+        .  /tmp/OBR/obr-env/bin/activate
+        mkdir ${{matrix.Case}} && cd ${{matrix.Case}}
+        obr init --config /__w/OGL/OGL/test/${{matrix.Case}}.yaml
+        obr run -o fetchCase
+        obr run -o generate
+        obr run -o runParallelSolver -t 1
+        obr status
+        obr query -q "{key:'state'}"
+        res=$(obr query -q "{key:'state'}"|grep completed|wc -l)
+        # if [ $res -eq 2 ]; then exit 0; else exit 1; fi
+
+    - name: Rename log files
+      if: always()
+      working-directory: /github/home/
+      run: |
+        find . -name "*:*" -exec rename 's|:|-|g' {} \;
+
+    - name: Archive integration artifacts
+      uses: actions/upload-artifact@v3
+      if: always()
+      with:
+        name: ogl_integration_${{ matrix.OF.path }}
+        path: /github/home

CMakeLists.txt

@@ -17,6 +17,7 @@ list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake/Modules/")
 
 include(CheckIncludeFileCXX)
 option(OGL_USE_EXTERNAL_GINKGO "Use external ginkgo" FALSE)
+option(OGL_DATA_VALIDATION "Check if resulting matrix data is valid" FALSE)
 option(OGL_USE_FOAM_FOUNDATION_VERSION
        "Build OGL for the OpenFOAM Foundation version" FALSE)
 option(GINKGO_BUILD_CUDA "Build Ginkgo with cuda backend" FALSE)
@@ -192,6 +193,10 @@ if(${GINKGO_BUILD_CUDA})
   target_link_libraries(OGL PUBLIC nvToolsExt)
 endif()
 
+if(OGL_DATA_VALIDATION)
+  target_compile_definitions(OGL PRIVATE DATA_VALIDATION=1)
+endif()
+
 if(EXISTS $ENV{WM_PROJECT_DIR}/CONTRIBUTORS.md)
   target_compile_definitions(OGL PRIVATE WITH_ESI_VERSION=1)
 endif()

DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H

@@ -296,8 +296,13 @@ struct MatrixInitFunctor {
         auto non_local_cols = non_local_col_idxs_.get_array();
         auto non_local_rows = non_local_row_idxs_.get_array();
 
+#ifdef DATA_VALIDATION
+        bool debug = true;
+#else
+        bool debug = false;
+#endif
         // check if sorted
-        if (false) {
+        if (debug) {
             bool is_sorted_rows = true;
             bool is_sorted_cols = true;
             auto rows_data = rows->get_const_data();