Merge pull request #68 from gridap/optional_mumps_support

Optional mumps support
gridap · Jan 24, 2022 · 7a3ecc7 · 7a3ecc7
2 parents 5a7a1c4 + 26fd2f8
commit 7a3ecc7
Show file tree

Hide file tree

Showing 7 changed files with 134 additions and 61 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -1,74 +1,40 @@
 name: CI
 on: [push, pull_request]
 jobs:
-  Tests:
+  test:
     name: Tests ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }}
     runs-on: ${{ matrix.os }}
-    env:
-      JULIA_MPI_BINARY: "system"
-      JULIA_PETSC_LIBRARY: "/opt/petsc/3.15.4/lib/libpetsc"
     strategy:
       fail-fast: false
       matrix:
         version:
           - '1.6'
         os:
-          - ubuntu-latest
+          - ubuntu-18.04
         arch:
           - x64
     steps:
       - uses: actions/checkout@v2
-      - name: Cache petsc
-        id: cache-petsc
-        uses: actions/cache@v2
-        with:
-         path: ${{env.JULIA_PETSC_LIBRARY}}
-         key: ${{ runner.os }}-build-${{ env.JULIA_PETSC_LIBRARY }}-
-         restore-keys: |
-          ${{ runner.os }}-build-${{ env.JULIA_PETSC_LIBRARY }}-
-          ${{ runner.os }}-build-
-          ${{ runner.os }}-       
       - uses: julia-actions/setup-julia@v1
         with:
           version: ${{ matrix.version }}
           arch: ${{ matrix.arch }}
-      - name: Install petsc dependencies
-        run: |
-           sudo apt-get update 
-           sudo apt-get install -y wget gfortran g++ openmpi-bin libopenmpi-dev  
-      - name: Install petsc
-        if: steps.cache-petsc.outputs.cache-hit != 'true'
-        run: |
-          # Install p4est 2.2 from sources
-          CURR_DIR=$(pwd)
-          PACKAGE=petsc
-          VERSION=3.15.4
-          INSTALL_ROOT=/opt
-          PETSC_INSTALL=$INSTALL_ROOT/$PACKAGE/$VERSION
-          TAR_FILE=$PACKAGE-$VERSION.tar.gz
-          URL="https://ftp.mcs.anl.gov/pub/petsc/release-snapshots/"
-          ROOT_DIR=/tmp
-          SOURCES_DIR=$ROOT_DIR/$PACKAGE-$VERSION
-          BUILD_DIR=$SOURCES_DIR/build
-          wget -q $URL/$TAR_FILE -O $ROOT_DIR/$TAR_FILE
-          mkdir -p $SOURCES_DIR
-          tar xzf $ROOT_DIR/$TAR_FILE -C $SOURCES_DIR --strip-components=1
-          cd $SOURCES_DIR
-          ./configure --prefix=$PETSC_INSTALL --with-cc=mpicc --with-cxx=mpicxx --with-fc=mpif90 \
-             --download-mumps --download-scalapack --download-parmetis --download-metis \
-             --download-ptscotch --with-debugging --with-x=0 --with-shared=1 \
-             --with-mpi=1 --with-64-bit-indices
-          make
-          make install  
-      - run: julia --project=. -e 'using Pkg; Pkg.instantiate(); Pkg.build(); Pkg.precompile()'
-      - run: julia --project=. --color=yes --check-bounds=yes test/sequential/runtests.jl
-      - run: julia --project=. --color=yes --check-bounds=yes test/mpi/runtests.jl
+      - uses: actions/cache@v1
+        env:
+          cache-name: cache-artifacts
+        with:
+          path: ~/.julia/artifacts
+          key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-test-${{ env.cache-name }}-
+            ${{ runner.os }}-test-
+            ${{ runner.os }}-
+      - uses: julia-actions/julia-buildpkg@v1
+      - uses: julia-actions/julia-runtest@v1
       - uses: julia-actions/julia-processcoverage@v1
       - uses: codecov/codecov-action@v1
-        env:
-          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
         with:
-          file: lcov.info        
+          file: lcov.info
   docs:
     name: Documentation
     runs-on: ubuntu-latest

diff --git a/.github/workflows/ci_extra.yml b/.github/workflows/ci_extra.yml
@@ -0,0 +1,64 @@
+name: CI_EXTRA
+on: [push, pull_request]
+jobs:
+  test:
+    name: Tests ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }}
+    runs-on: ${{ matrix.os }}
+    env:
+      JULIA_MPI_BINARY: "system"
+      JULIA_PETSC_LIBRARY: "/opt/petsc/3.15.4/lib/libpetsc"
+    strategy:
+      fail-fast: false
+      matrix:
+        version:
+          - '1.6'
+        os:
+          - ubuntu-latest
+        arch:
+          - x64
+    steps:
+      - uses: actions/checkout@v2
+      - name: Cache petsc
+        id: cache-petsc
+        uses: actions/cache@v2
+        with:
+         path: ${{env.JULIA_PETSC_LIBRARY}}
+         key: ${{ runner.os }}-build-${{ env.JULIA_PETSC_LIBRARY }}-
+         restore-keys: |
+          ${{ runner.os }}-build-${{ env.JULIA_PETSC_LIBRARY }}-
+          ${{ runner.os }}-build-
+          ${{ runner.os }}-  
+      - uses: julia-actions/setup-julia@v1
+        with:
+          version: ${{ matrix.version }}
+          arch: ${{ matrix.arch }}
+      - name: Install petsc dependencies
+        run: |
+           sudo apt-get update 
+           sudo apt-get install -y wget gfortran g++ openmpi-bin libopenmpi-dev  
+      - name: Install petsc
+        ##if: steps.cache-petsc.outputs.cache-hit != 'true'
+        run: |
+          CURR_DIR=$(pwd)
+          PACKAGE=petsc
+          VERSION=3.15.4
+          INSTALL_ROOT=/opt
+          PETSC_INSTALL=$INSTALL_ROOT/$PACKAGE/$VERSION
+          TAR_FILE=$PACKAGE-$VERSION.tar.gz
+          URL="https://ftp.mcs.anl.gov/pub/petsc/release-snapshots/"
+          ROOT_DIR=/tmp
+          SOURCES_DIR=$ROOT_DIR/$PACKAGE-$VERSION
+          BUILD_DIR=$SOURCES_DIR/build
+          wget -q $URL/$TAR_FILE -O $ROOT_DIR/$TAR_FILE
+          mkdir -p $SOURCES_DIR
+          tar xzf $ROOT_DIR/$TAR_FILE -C $SOURCES_DIR --strip-components=1
+          cd $SOURCES_DIR
+          ./configure --prefix=$PETSC_INSTALL --with-cc=mpicc --with-cxx=mpicxx --with-fc=mpif90 \
+             --download-mumps --download-scalapack --download-parmetis --download-metis \
+             --download-ptscotch --with-debugging --with-x=0 --with-shared=1 \
+             --with-mpi=1 --with-64-bit-indices
+          make
+          make install
+      - run: julia --project=. -e 'using Pkg; Pkg.instantiate(); Pkg.build(); Pkg.precompile()'
+      - run: julia --project=. --color=yes --check-bounds=yes test/sequential/runtests.jl
+      - run: julia --project=. --color=yes --check-bounds=yes test/mpi/runtests.jl
diff --git a/src/GridapPETSc.jl b/src/GridapPETSc.jl
@@ -59,7 +59,10 @@ function __init__()
     libpetsc_handle[] = PETSc_jll.libpetsc_handle
   end
   for (handle,sym) in _PRELOADS
-    handle[] = Libdl.dlsym(libpetsc_handle[],sym)
+    _handle = Libdl.dlsym(libpetsc_handle[],sym;throw_error=false)
+    if _handle !== nothing
+      handle[] = _handle
+    end
   end
 end
 

diff --git a/src/PETSC.jl b/src/PETSC.jl
@@ -18,6 +18,7 @@ module PETSC
 using Libdl
 using GridapPETSc: libpetsc_handle
 using GridapPETSc: _PRELOADS
+using Gridap.Helpers: @check
 using MPI
 
 include("Config.jl")
@@ -59,6 +60,7 @@ macro wrapper(fn,rt,argts,args,url)
     push!(_PRELOADS,($hn,$fn))
     @doc $str
     @inline function $(fn.value)($(args.args...))
+      @check $(hn)[] != C_NULL "Missing symbol. Re-configure and compile PETSc."
       ccall($(hn)[],$rt,$argts,$(args.args...))
     end
   end

diff --git a/test/PLaplacianTests.jl b/test/PLaplacianTests.jl
@@ -3,6 +3,7 @@ using Gridap.Algebra
 using GridapDistributed
 using PartitionedArrays
 using GridapPETSc
+using GridapPETSc: PETSC
 using Test
 
 
@@ -26,17 +27,28 @@ function mysnessetup(snes)
   @check_error_code GridapPETSc.PETSC.MatMumpsSetCntl(mumpsmat[], 3, 1.0e-6)
 end
 
-
 function main(parts)
-  options = "-snes_type newtonls -snes_linesearch_type basic  -snes_linesearch_damping 1.0 -snes_rtol 1.0e-14 -snes_atol 0.0 -snes_monitor -snes_converged_reason"
+  main(parts,:gmres)
+  if PETSC.MatMumpsSetIcntl_handle[] != C_NULL
+    main(parts,:mumps)
+  end
+end
 
+function main(parts,solver)
+  if solver == :mumps
+    options = "-snes_type newtonls -snes_linesearch_type basic  -snes_linesearch_damping 1.0 -snes_rtol 1.0e-14 -snes_atol 0.0 -snes_monitor -snes_converged_reason"
+  elseif solver == :gmres
+    options = "-snes_type newtonls -snes_linesearch_type basic  -snes_linesearch_damping 1.0 -snes_rtol 1.0e-14 -snes_atol 0.0 -snes_monitor -pc_type jacobi -ksp_type gmres -ksp_monitor -snes_converged_reason"
+  else
+    error()
+  end
   GridapPETSc.with(args=split(options)) do
-     main(parts,FullyAssembledRows())
-     main(parts,SubAssembledRows())
+     main(parts,solver,FullyAssembledRows())
+     main(parts,solver,SubAssembledRows())
   end
 end
 
-function main(parts,strategy)
+function main(parts,solver,strategy)
 
   domain = (0,4,0,4)
   cells = (100,100)
@@ -69,9 +81,13 @@ function main(parts,strategy)
   # fill!(x,1)
   # @test (norm(A*x-_A*x)+1) ≈ 1
 
-  nls = PETScNonlinearSolver(mysnessetup)
-  solver = FESolver(nls)
-  uh = solve(solver,op)
+  if solver == :mumps
+    nls = PETScNonlinearSolver(mysnessetup)
+  else
+    nls = PETScNonlinearSolver()
+  end
+  fesolver = FESolver(nls)
+  uh = solve(fesolver,op)
 
   Ωo = Triangulation(model)
   dΩo = Measure(Ωo,2*k)

diff --git a/test/PoissonTests.jl b/test/PoissonTests.jl
@@ -4,6 +4,7 @@ using Gridap.Algebra
 using Gridap.FESpaces
 using GridapDistributed
 using GridapPETSc
+using GridapPETSc: PETSC
 using PartitionedArrays
 using Test
 
@@ -25,7 +26,20 @@ function mykspsetup(ksp)
 end
 
 function main(parts)
-  options = "-info  -ksp_error_if_not_converged true"
+  main(parts,:cg)
+  if PETSC.MatMumpsSetIcntl_handle[] != C_NULL
+    main(parts,:mumps)
+  end
+end
+
+function main(parts,solver)
+  if solver == :mumps
+    options = "-info  -ksp_error_if_not_converged true"
+  elseif solver == :cg
+    options = "-info -pc_type jacobi -ksp_type cg -ksp_monitor -ksp_rtol 1.0e-12"
+  else
+    error()
+  end
   GridapPETSc.with(args=split(options)) do
       domain = (0,4,0,4)
       cells = (4,4)
@@ -56,7 +70,11 @@ function main(parts)
       assem=SparseMatrixAssembler(SparseMatrixCSR{0,PetscScalar,PetscInt},Vector{Float64},U,V)
       op = AffineFEOperator(a,l,U,V,assem)
 
-      ls = PETScLinearSolver(mykspsetup)
+      if solver == :mumps
+        ls = PETScLinearSolver(mykspsetup)
+      else
+        ls = PETScLinearSolver()
+      end
       fels = LinearFESolver(ls)
       uh = solve(fels,op)
       eh = u - uh

diff --git a/test/mpi/mpiexec.jl b/test/mpi/mpiexec.jl
@@ -5,7 +5,11 @@ function run_mpi_driver(;procs,file)
   testdir = joinpath(mpidir,"..")
   repodir = joinpath(testdir,"..")
   mpiexec() do cmd
-    run(`$cmd -n $procs --allow-run-as-root --oversubscribe $(Base.julia_cmd()) --project=$repodir $(joinpath(mpidir,file))`)
+    if MPI.MPI_LIBRARY == MPI.OpenMPI
+      run(`$cmd -n $procs --allow-run-as-root --oversubscribe $(Base.julia_cmd()) --project=$repodir $(joinpath(mpidir,file))`)
+    else
+      run(`$cmd -n $procs $(Base.julia_cmd()) --project=$repodir $(joinpath(mpidir,file))`)
+    end
     @test true
   end
 end