diff --git a/.clang-format b/.clang-format index 3cbc1d236..b0d09486c 100644 --- a/.clang-format +++ b/.clang-format @@ -109,4 +109,3 @@ Standard: Cpp11 TabWidth: 8 UseTab: Never ... - diff --git a/.cmake-format.py b/.cmake-format.py index 0c386a443..5baf6552e 100644 --- a/.cmake-format.py +++ b/.cmake-format.py @@ -1,6 +1,5 @@ +# SPDX-License-Identifier: Unlicense # SPDX-FileCopyrightText: 2024 OGL authors -# -# SPDX-License-Identifier: GPL-3.0-or-later # ---------------------------------- # Options affecting listfile parsing @@ -29,7 +28,7 @@ disable = False # How wide to allow formatted cmake files - line_width = 80 + line_width = 100 # How many spaces to tab for indent tab_size = 2 @@ -168,7 +167,7 @@ with section("lint"): # a list of lint codes to disable - disabled_codes = [] + disabled_codes = ["C0103"] # regular expression pattern describing valid function names function_pattern = '[0-9a-z_]+' @@ -242,4 +241,3 @@ # A dictionary containing any per-command configuration overrides. Currently # only `command_case` is supported. per_command = {} - diff --git a/.github/workflows/build-foam.yml b/.github/workflows/build-foam.yml index 30585b861..b7ad85d26 100644 --- a/.github/workflows/build-foam.yml +++ b/.github/workflows/build-foam.yml @@ -11,7 +11,7 @@ defaults: on: push: - branches: + branches: - dev - main pull_request: @@ -41,7 +41,7 @@ jobs: - name: Set matrix data id: set-matrix run: | - pwd + pwd echo "matrix=$(jq -c . < .github/workflows/test_matrix.json)" >> $GITHUB_OUTPUT build: @@ -58,23 +58,22 @@ jobs: - name: Get Ginkgo checkout version shell: bash run: | - grep -A1 "set(GINKGO_CHECKOUT_VERSION" CMakeLists.txt|tail -n1|grep -o "[0-9a-z\-]*" > GINKGO_CHECKOUT_VERSION + grep -A1 "set(GINKGO_CHECKOUT_VERSION" CMakeLists.txt|tail -n1|grep -o "[0-9a-z\-_.]*" > GINKGO_CHECKOUT_VERSION export GINKGO_CHECKOUT_VERSION=$(cat GINKGO_CHECKOUT_VERSION) echo "GINKGO_CHECKOUT_VERSION=$GINKGO_CHECKOUT_VERSION" >> $GITHUB_ENV - name: Cache build folder uses: actions/cache@v4 + if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip-cache')}} with: - key: build-${{ matrix.OF.path }}-${{env.GINKGO_CHECKOUT_VERSION}} - path: | - ${{github.workspace}}/build + key: build_PR_${{ github.event.pull_request.number }}_${{matrix.OF.path}} + path: ${{github.workspace}}/build - name: Cache FOAM_USER_LIBBIN uses: actions/cache@v4 with: - key: FOAM_USER_LIBBIN-${{ matrix.OF.version }}-${{ github.sha }} - path: | - ${{matrix.OF.foam_user_libbin}} + key: FOAM_USER_LIBBIN-${{ matrix.OF.path }}-${{ github.sha }} + path: ${{matrix.OF.foam_user_libbin}} - name: Configure CMake # Use a bash shell so we can use the same syntax for environment variable @@ -83,25 +82,19 @@ jobs: run: | source /root/OpenFOAM/${{matrix.OF.path}}/etc/bashrc mkdir -p ${{matrix.OF.foam_user_libbin}} - mkdir -p build - cd build - [ -d "/github/home/$GINKGO_CHECKOUT_VERSION" ] && cp -rp /github/home/$GINKGO_CHECKOUT_VERSION third_party - cmake -GNinja -DOGL_ALLOW_REFERENCE_ONLY=On -DOGL_BUILD_UNITTEST=ON -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DCMAKE_INSTALL_PREFIX=${{matrix.OF.foam_user_libbin}} .. + cmake --preset develop - name: Build OGL - working-directory: ${{github.workspace}}/build + working-directory: ${{github.workspace}} run: | source /root/OpenFOAM/${{matrix.OF.path}}/etc/bashrc - cmake --build . --config $BUILD_TYPE + cmake --build --preset develop - name: Install OGL working-directory: ${{github.workspace}} run: | source /root/OpenFOAM/${{matrix.OF.path}}/etc/bashrc - cmake --install ./build - ls $FOAM_USER_LIBBIN - ls ${{matrix.OF.foam_user_libbin}} - + cmake --build --preset develop --target install - name: Check folders working-directory: ${{github.workspace}} diff --git a/.github/workflows/compile_on_aws.yml b/.github/workflows/compile_on_aws.yml index d018ad71f..bdde16e75 100644 --- a/.github/workflows/compile_on_aws.yml +++ b/.github/workflows/compile_on_aws.yml @@ -44,9 +44,9 @@ jobs: ] pre-runner-script: | #!/bin/bash - sudo yum update -y && \ + sudo yum update -y && \ sudo yum install docker git libicu ninja-build libasan10 -y - sudo amazon-linux-extras install epel -y + sudo amazon-linux-extras install epel -y sudo yum install Lmod -y sudo systemctl enable docker sudo curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.5/install.sh | bash @@ -84,7 +84,7 @@ jobs: module load conda conda activate python3 -m pip install --upgrade pip - pip install setuptools --upgrade + pip install setuptools --upgrade git clone https://github.com/exasim-project/OBR cd OBR pip install . @@ -123,8 +123,8 @@ jobs: module load openmpi module load OpenFOAM export GINKGO_EXECUTOR=cuda - obr init -g --config test/integration.yaml - obr run -o runParallelSolver + obr init -g --config test/integration.yaml + obr run -o runParallelSolver obr status stop-runner: diff --git a/.github/workflows/format-check.yml b/.github/workflows/format-check.yml index ab5ef499a..cf45f0328 100644 --- a/.github/workflows/format-check.yml +++ b/.github/workflows/format-check.yml @@ -21,5 +21,5 @@ jobs: - name: check for todo fixme note run: | NTODOS="$(grep -r 'TODO DONT MERGE' --exclude-dir=.github | wc -l)" - echo "Found $NTODOS TODO DONT MERGE notes" - ! grep -q -r "TODO DONT MERGE" --exclude-dir=.github + echo "Found $NTODOS TODO DONT MERGE notes" + ! grep -q -r "TODO DONT MERGE" --exclude-dir=.github diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 214859156..77aaa3389 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -19,7 +19,7 @@ env: WM_LINK_LANGUAGE: c++ WM_LABEL_OPTION: Int32 WM_MPLIB: SYSTEMOPENMPI - MPI_BUFFER_SIZE: 20000000 + MPI_BUFFER_SIZE: 20000000 BUILD_TYPE: Release OMPI_ALLOW_RUN_AS_ROOT: 1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1 @@ -27,7 +27,7 @@ env: OBR_RUN_CMD: "mpirun --oversubscribe -np {np} {solver} -parallel -case {path}/case > {path}/case/{solver}_{timestamp}.log 2>&1" GINKGO_EXECUTOR: reference -on: +on: workflow_call: inputs: path: @@ -74,10 +74,10 @@ jobs: run: | ls python3 -m pip install --upgrade pip - pip install setuptools --upgrade - pip uninstall obr -y - pip uninstall owls -y - cd /root/OBR + pip install setuptools --upgrade + pip uninstall obr -y + pip uninstall owls -y + cd /root/OBR git pull origin dev pip install -e . obr --version @@ -89,7 +89,7 @@ jobs: export LD_LIBRARY_PATH=$FOAM_USER_LIBBIN:$WM_PROJECT_DIR/platforms/linux64GccDPInt32Opt/lib:$WM_PROJECT_DIR/platforms/linux64GccDPInt32Opt/lib/sys-openmpi/:$WM_PROJECT_DIR/platforms/linux64GccDPInt32Opt/lib/openmpi-system:$WM_PROJECT_DIR/platforms/linux64GccDPInt32Opt/lib/dummy/:$LD_LIBRARY_PATH mkdir -p integration && cd integration export CYCLIC_CASE=${{inputs.cyclic_case}} - obr init -gy --config /__w/OGL/OGL/test/integration.yaml + obr init -g --config /__w/OGL/OGL/test/integration/lidDrivenCavity.yaml cat workspace/*/case/*log setup_integration_matrix: @@ -108,7 +108,7 @@ jobs: name: run-${{matrix.Case}}-${{inputs.version}} strategy: fail-fast: false - matrix: + matrix: Case: ${{fromJson(needs.setup_integration_matrix.outputs.matrix)}} steps: @@ -128,7 +128,7 @@ jobs: - name: Get Ginkgo checkout version shell: bash run: | - grep -A1 "set(GINKGO_CHECKOUT_VERSION" CMakeLists.txt|tail -n1|grep -o "[0-9a-z\-]*" > GINKGO_CHECKOUT_VERSION + grep -A1 "set(GINKGO_CHECKOUT_VERSION" CMakeLists.txt|tail -n1|grep -o "[0-9a-z\-_.]*" > GINKGO_CHECKOUT_VERSION export GINKGO_CHECKOUT_VERSION=$(cat GINKGO_CHECKOUT_VERSION) echo "GINKGO_CHECKOUT_VERSION=$GINKGO_CHECKOUT_VERSION" >> $GITHUB_ENV echo "Ginkgo_DIR=${{inputs.foam_user_libbin}}/cmake/Ginkgo" >> $GITHUB_ENV @@ -137,10 +137,10 @@ jobs: working-directory: /github/home/ run: | python3 -m pip install --upgrade pip - pip install setuptools --upgrade - pip uninstall obr -y - pip uninstall owls -y - cd /root/OBR + pip install setuptools --upgrade + pip uninstall obr -y + pip uninstall owls -y + cd /root/OBR git pull origin dev pip install -e . @@ -193,7 +193,7 @@ jobs: name: validate-${{matrix.Case}}-${{inputs.version}} strategy: fail-fast: false - matrix: + matrix: Case: ${{fromJson(needs.setup_integration_matrix.outputs.matrix)}} steps: @@ -204,10 +204,10 @@ jobs: run: | ls python3 -m pip install --upgrade pip - pip install setuptools --upgrade - pip uninstall obr -y - pip uninstall owls -y - cd /root/OBR + pip install setuptools --upgrade + pip uninstall obr -y + pip uninstall owls -y + cd /root/OBR git pull origin dev pip install -e . @@ -220,7 +220,7 @@ jobs: - name: Validation of exported mtx files working-directory: /github/home/${{matrix.Case}} run: | - python3 /__w/OGL/OGL/test/data_validation.py workspace + python3 /__w/OGL/OGL/test/integration/data_validation.py workspace - name: Run obr status working-directory: /github/home/${{matrix.Case}} @@ -236,14 +236,14 @@ jobs: --filter preconditioner==none \ --filter matrixFormat==Coo \ --filter global==completed \ - --validate_against=/__w/OGL/OGL/test/validation.json + --validate_against=/__w/OGL/OGL/test/integration/validation.json - name: Validate preconditioned state working-directory: /github/home/${{matrix.Case}} run: | obr query \ -q global -q continuityErrors -q CourantNumber \ - --validate_against=/__w/OGL/OGL/test/validation.json \ + --validate_against=/__w/OGL/OGL/test/integration/validation.json \ --filter preconditioner!=none \ --filter global==completed \ --filter case==${{matrix.Case}} @@ -253,7 +253,7 @@ jobs: run: | obr query \ -q global -q continuityErrors -q CourantNumber \ - --validate_against=/__w/OGL/OGL/test/validation.json \ + --validate_against=/__w/OGL/OGL/test/integration/validation.json \ --filter matrixFormat!=Coo \ --filter global==completed \ --filter case==${{matrix.Case}} diff --git a/.github/workflows/static_checks.yaml b/.github/workflows/static_checks.yaml index 1649dd778..f5fb4eb28 100644 --- a/.github/workflows/static_checks.yaml +++ b/.github/workflows/static_checks.yaml @@ -5,37 +5,42 @@ name: Static checks run-name: Static checks +defaults: + run: + shell: bash -o pipefail -i {0} + on: pull_request: types: [opened, synchronize] + +env: + USER: user + BUILD_TYPE: Release + OMPI_ALLOW_RUN_AS_ROOT: 1 + OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1 + + jobs: build-compilation-db: name: Build with IWYU runs-on: ubuntu-latest + container: greole/ofbase steps: - uses: actions/checkout@v4 - - - name: Add clang repo - run: | - sudo add-apt-repository 'deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-16 main' - wget https://apt.llvm.org/llvm-snapshot.gpg.key - sudo apt-key add llvm-snapshot.gpg.key - - name: Install dependencies run: | - sudo apt update - sudo apt install \ + apt update + apt install -y \ ninja-build \ - iwyu \ - clang-16 \ - libomp-16-dev - + libomp-dev \ + iwyu - name: Create Compilation Database run: | - cmake --preset ninja-cpuonly-release \ - -DCMAKE_CXX_COMPILER=clang++-16 \ - -DCMAKE_EXPORT_COMPILE_COMMANDS=ON - cmake --build --preset ninja-cpuonly-release + source /root/OpenFOAM/openfoam/etc/bashrc + cmake --preset develop \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DCMAKE_EXPORT_COMPILE_COMMANDS=ON + cmake --build --preset develop - uses: actions/upload-artifact@v4 with: name: build-artifact @@ -43,40 +48,33 @@ jobs: clang-tidy-check: name: Clang-tidy Check runs-on: ubuntu-latest + container: greole/ofbase needs: [build-compilation-db] steps: - uses: actions/checkout@v4 - - - name: Add clang repo - run: | - sudo add-apt-repository 'deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-16 main' - wget https://apt.llvm.org/llvm-snapshot.gpg.key - sudo apt-key add llvm-snapshot.gpg.key - - name: Install dependencies run: | - sudo apt update - sudo apt install \ - clang-tidy-16 \ - libomp-16-dev - + apt update + apt install -y \ + clang-tidy \ + jq - uses: actions/download-artifact@v4 with: name: build-artifact path: build - - name: Run clang-tidy run: | - # Create list of .cpp files belonging to this repository - git ls-files | grep -E "\.(cpp)" > pattern + git config --global --add safe.directory /__w/OGL/OGL + # Create list of all source files belonging to this repository + git ls-files | grep -E "\.(C)" > pattern # Create list of .cpp files that are in this repository and part of the # compilation database # also filters out " at the begin and end of the filename - jq ".[] | .file" build/ReleaseAll/compile_commands.json \ + jq ".[] | .file" build/ReleaseCPUOnly/compile_commands.json \ | sed 's/^"\(.*\)"$/\1/' \ | grep -F -f pattern - > files # run clang-tidy on all specified files - clang-tidy-16 --fix --extra-arg=-w -p build/ReleaseAll $(cat files) + clang-tidy --fix --extra-arg=-w -p build/ReleaseCPUOnly $(cat files) - name: Check for fixes run: | diff --git a/.github/workflows/test_matrix.json b/.github/workflows/test_matrix.json index 838863ca9..d305459cd 100644 --- a/.github/workflows/test_matrix.json +++ b/.github/workflows/test_matrix.json @@ -1,20 +1,20 @@ { -"OF": [ - { - "version": "v2212", - "fork": "ESI", - "path": "openfoam", - "repo": "https://develop.openfoam.com/Development/openfoam.git", - "cyclic_case": "periodicPlaneChannel" , - "foam_user_libbin": "/github/home/OpenFOAM/user-v2312/platforms/linux64GccDPInt32Opt/lib/" + "OF": [ + { + "version": "v2212", + "fork": "ESI", + "path": "openfoam", + "repo": "https://develop.openfoam.com/Development/openfoam.git", + "cyclic_case": "periodicPlaneChannel", + "foam_user_libbin": "/github/home/OpenFOAM/user-v2312/platforms/linux64GccDPInt32Opt/lib/" }, - { - "version": "10", - "fork": "Foundation", - "path": "OpenFOAM-10", - "repo": "https://github.com/OpenFOAM/OpenFOAM-10.git", - "cyclic_case": "channel395" , - "foam_user_libbin": "/github/home/OpenFOAM/user-10/platforms/linux64GccDPInt32Opt/lib" + { + "version": "10", + "fork": "Foundation", + "path": "OpenFOAM-10", + "repo": "https://github.com/OpenFOAM/OpenFOAM-10.git", + "cyclic_case": "channel395", + "foam_user_libbin": "/github/home/OpenFOAM/user-10/platforms/linux64GccDPInt32Opt/lib" } -] + ] } diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 39b735da7..f8425b869 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -7,7 +7,7 @@ defaults: run: shell: bash -o pipefail -i {0} -on: +on: workflow_call: inputs: path: @@ -22,6 +22,11 @@ on: env: USER: user + CTEST_OUTPUT_ON_FAILURE: 1 + OMPI_ALLOW_RUN_AS_ROOT: 1 + OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1 + OMPI_MCA_rmaps_base_oversubscribe: 1 + OMPI_MCA_btl_vader_single_copy_mechanism: none run-name: Unit tests on ${{github.ref_name}} @@ -33,8 +38,9 @@ jobs: - uses: actions/checkout@v4 - name: Get Ginkgo checkout version + shell: bash run: | - grep -A1 "set(GINKGO_CHECKOUT_VERSION" CMakeLists.txt|tail -n1|grep -o "[0-9a-z\-]*" > GINKGO_CHECKOUT_VERSION + grep -A1 "set(GINKGO_CHECKOUT_VERSION" CMakeLists.txt|tail -n1|grep -o "[0-9a-z\-_.]*" > GINKGO_CHECKOUT_VERSION export GINKGO_CHECKOUT_VERSION=$(cat GINKGO_CHECKOUT_VERSION) echo "GINKGO_CHECKOUT_VERSION=$GINKGO_CHECKOUT_VERSION" >> $GITHUB_ENV @@ -48,13 +54,13 @@ jobs: - name: Cache build folder uses: actions/cache@v4 with: - key: build-${{ inputs.path }}-${{env.GINKGO_CHECKOUT_VERSION}} + # key: build-${{ inputs.path }}-${{env.GINKGO_CHECKOUT_VERSION}} + key: build_PR_${{ github.event.pull_request.number }}_${{inputs.path}} path: | ${{github.workspace}}/build - name: Test OGL - working-directory: ${{github.workspace}}/build + working-directory: ${{github.workspace}} run: | source /root/OpenFOAM/${{inputs.path}}/etc/bashrc - ls $FOAM_USER_LIBBIN - ./unitTests/matrixConversion + ctest --preset develop diff --git a/.gitignore b/.gitignore index f80c5214c..f9f0d8a01 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,4 @@ build builds -.vscode \ No newline at end of file +.vscode diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..5f84942e8 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,23 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + - id: pretty-format-json + args: [--autofix, --no-sort-keys] + - repo: https://github.com/pre-commit/mirrors-clang-format + rev: 'v14.0.0' # The default in Ubuntu 22.04, which is used in our CI + hooks: + - id: clang-format + types_or: [c++] + - repo: https://github.com/cheshirekow/cmake-format-precommit + rev: v0.6.10 + hooks: + - id: cmake-format + - id: cmake-lint + - repo: https://github.com/fsfe/reuse-tool + rev: v3.0.1 + hooks: + - id: reuse diff --git a/.reuse/dep5 b/.reuse/dep5 index 78eea379a..bf55f75d6 100644 --- a/.reuse/dep5 +++ b/.reuse/dep5 @@ -11,11 +11,7 @@ Files: common/.common.H.swo Copyright: 2024 OGL authors License: GPL-3.0-or-later -Files: test/validation.json -Copyright: 2024 OGL authors -License: GPL-3.0-or-later - -Files: test/channel_validation.json +Files: test/** Copyright: 2024 OGL authors License: GPL-3.0-or-later @@ -34,6 +30,11 @@ License: GPL-3.0-or-later Files: CITATION.cff Copyright: 2024 OGL authors License: GPL-3.0-or-later + +Files: .pre-commit-config.yaml +Copyright: 2024 OGL authors +License: GPL-3.0-or-later + # Sample paragraph, commented out: # # Files: src/* diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index da7ce8c27..000000000 --- a/.travis.yml +++ /dev/null @@ -1,208 +0,0 @@ -# SPDX-FileCopyrightText: 2024 OGL authors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -language: cpp -sudo: true -compiler: - - gcc - -dist: focal -os: - - linux - - -cache: - directories: - - $HOME/cache - -stages: - - build - - run - -addons: - apt: - update: true - packages: - - flexc++ - - libfl-dev - - lcov - - libopenmpi-dev - - ipython3 - -script: skip - -_build: &_build - stage: build - if: - branch = dev OR env(EXTERNAL_GINKGO) = OFF - before_install: - - sudo sh -c "wget --no-check-certificate -O - https://dl.openfoam.org/gpg.key | apt-key add -" - - sudo add-apt-repository http://dl.openfoam.org/ubuntu - - sudo apt update - - sudo apt-get -y install openfoam$VERSION - - source /opt/openfoam$VERSION/etc/bashrc - - mkdir -p $HOME/data - - install: - - ./scripts/travis_install_external_ginkgo.sh - - before_script: - - cmake --version - - mkdir build - - cd build - - cmake -DOGL_USE_EXTERNAL_GINKGO=$EXTERNAL_GINKGO .. - - script: - - make -j4 && make install - - -_run: &_run - stage: run - before_install: - - git clone -b develop https://github.com/greole/owls.git $HOME/owls - - cd $HOME/owls - - python setup.py install --user - - source /opt/openfoam$VERSION/etc/bashrc - - install: - - git clone https://github.com/greole/OBR.git $HOME/OBR - - pip install docopt --user - - pip install packaging --user - - cd $HOME/OBR/ - - if [[ ! -d $HOME/cache/airFoil2D ]]; then python obr_create_tree.py --folder $HOME/cache/airFoil2D --parameters examples/airFoil2DCI.json; fi - - - - script: - - ls /opt - - ls $FOAM_SRC - - ls $FOAM_USER_LIBBIN - - cd $HOME/OBR - - cat $HOME/OBR/examples/airFoil2DCI.json - - python obr_benchmark_cases.py --folder $HOME/cache/airFoil2D --results_folder=Test --report=report.csv - - mkdir -p $HOME/data - - cp report.csv $HOME/data/report.csv - - cat $HOME/data/report.csv - -_validate: &_validate - stage: validate - script: - - cat $HOME/data/report.csv - - echo $TRAVIS_BUILD_DIR - - python $TRAVIS_BUILD_DIR/scripts/travis_check_results.py $HOME/data/report.csv - -jobs: - Include: - # Version 8 builds - - <<: *_build - ENV: - - VERSION=8 - - EXTERNAL_GINKGO=OFF - - GINKGO_VERSION=ogl - workspaces: - create: - name: - foam_env_8 - paths: - - /home/travis/OpenFOAM/travis-8 - - /opt/openfoam8 - - <<: *_build - ENV: - - VERSION=8 - - EXTERNAL_GINKGO=ON - - GINKGO_VERSION=ogl - workspaces: - create: - name: - foam_user_libbin_8 - paths: - /home/travis/OpenFOAM/travis-8 - # - <<: *_build - # ENV: - # - VERSION=8 - # - EXTERNAL_GINKGO=ON - # - GINKGO_VERSION=ogl - # - <<: *_build - # ENV: - # - VERSION=8 - # - EXTERNAL_GINKGO=ON - # - GINKGO_VERSION=ogl - # Version 9 builds - - <<: *_build - ENV: - - VERSION=9 - - EXTERNAL_GINKGO=ON - - GINKGO_VERSION=ogl - - <<: *_build - ENV: - - VERSION=9 - - EXTERNAL_GINKGO=OFF - - GINKGO_VERSION=ogl - # Runs - - <<: *_run - ENV: - - VERSION=8 - - SOLVER=CG - - PRECOND=NoPrecond - workspaces: - use: - - foam_env_8 - create: - name: - CGData - paths: - /home/travis/data - - <<: *_run - ENV: - - VERSION=8 - - SOLVER=BiCGStab - - PRECOND=NoPrecond - workspaces: - use: - - foam_env_8 - create: - name: - BiCGStabData - paths: - /home/travis/data - - <<: *_run - ENV: - - VERSION=8 - - SOLVER=CG - - PRECOND=BJ - workspaces: - use: - - foam_env_9 - - <<: *_run - ENV: - - VERSION=8 - - SOLVER=BiCGStab - - PRECOND=BJ - workspaces: - use: - - foam_env_8 - - <<: *_run - ENV: - - VERSION=8 - - SOLVER=IR - - PRECOND=NoPrecond - workspaces: - use: - - foam_env_8 - - <<: *_validate - ENV: - - SOLVER=CG - - PRECOND=NoPrecond - workspaces: - use: - - foam_env_8 - - CGData - - <<: *_validate - ENV: - - SOLVER=BiCGStab - - PRECOND=NoPrecond - workspaces: - use: - - foam_env_8 - - BiCGStabData diff --git a/AUTHORS.md b/AUTHORS.md index 70d61adb8..f628ff0d0 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -1,5 +1,5 @@ -Ziad Boutanios, Karlsruhe Institute of Technology, Ziad.boutanios@kit.edu +Ziad Boutanios, Karlsruhe Institute of Technology, Ziad.boutanios@kit.edu Terry Cojean, Karlsruhe Institute of Technology, terry.cojean@kit.edu Hendrik Hetmann, Upstream CFD, hendrik.hetmann@upstream-cfd.com Gregor Olenik, Karlsruhe Institute of Technology, gregor.olenik@kit.edu -Chih-Ta Wang, Technical University of Munich, chihta.wang@tum.de \ No newline at end of file +Chih-Ta Wang, Technical University of Munich, chihta.wang@tum.de diff --git a/BaseWrapper/lduBase/GKOlduBase.C b/BaseWrapper/lduBase/GKOlduBase.C deleted file mode 100644 index 60ff51d1c..000000000 --- a/BaseWrapper/lduBase/GKOlduBase.C +++ /dev/null @@ -1,14 +0,0 @@ -// SPDX-FileCopyrightText: 2024 OGL authors -// -// SPDX-License-Identifier: GPL-3.0-or-later - -#include -#include -#include - -#include "GKOlduBase.H" - - -// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * // - -// ************************************************************************* // diff --git a/CHANGELOG.md b/CHANGELOG.md index 70ebd49d4..75650f3af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,11 +3,12 @@ SPDX-FileCopyrightText: 2024 OGL authors SPDX-License-Identifier: GPL-3.0-or-later --> - +# 0.6.0 (unreleased) +- Build Ginkgo as subdirectory build [PR #135](https://github.com/hpsim/OGL/pull/135) # 0.5.4 (2024/06/21) - Add a warning message to users who use dpcpp as the executor name [PR #129](https://github.com/hpsim/OGL/pull/129) -- Add support for SP scalars and determine label and scalar size from env variable [PR #120](https://github.com/hpsim/OGL/pull/120) -- Notify user of a unsupported executor argument [PR #118](https://github.com/hpsim/OGL/pull/118) +- Add support for SP scalars and determine label and scalar size from env variable [PR #120](https://github.com/hpsim/OGL/pull/120) +- Notify user of a unsupported executor argument [PR #118](https://github.com/hpsim/OGL/pull/118) - Add a list of all OGL authors [PR #126](https://github.com/hpsim/OGL/pull/126) - Replace #ifdef include guard with #pragma once [PR #126](https://github.com/hpsim/OGL/pull/126) - Switch to SPDX headers, fix formatting issue[PR #122](https://github.com/hpsim/OGL/pull/112) @@ -16,10 +17,10 @@ SPDX-License-Identifier: GPL-3.0-or-later # 0.5.2 (2024/03/08) - Fix issue with cyclic boundary conditions [PR #108](https://github.com/hpsim/OGL/pull/108) # 0.5.1 (2024/01/18) -- Fix issue with failing cmake build +- Fix issue with failing cmake build # 0.5.0 (2024/01/11) ## Features -- Add on device permutation functionality [PR #101](https://github.com/hpsim/OGL/pull/101) +- Add on device permutation functionality [PR #101](https://github.com/hpsim/OGL/pull/101) ## Stability and Fixes - Fix issue with hanging diverging cases [PR #104](https://github.com/hpsim/OGL/pull/96) - Implemented unit tests for ldu conversion [PR #97](https://github.com/hpsim/OGL/pull/97) diff --git a/CITATION.cff b/CITATION.cff index 70033f7d5..5b196b976 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -16,9 +16,9 @@ preferred-citation: - family-names: "Boutanios" given-names: "Ziad" - family-names: "Anzt" - given-names: "Hartwig" + given-names: "Hartwig" doi: "10.1007/s11012-024-01806-1" journal: "Meccanica" month: 6 title: "Towards a platform‑portable linear algebra backend for OpenFOAM" - year: 2024 \ No newline at end of file + year: 2024 diff --git a/CMakeLists.txt b/CMakeLists.txt index 895020167..fe6045177 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,35 +7,56 @@ cmake_minimum_required(VERSION 3.13.0) project( OGL LANGUAGES C CXX - DESCRIPTION - "A wrapper for Ginkgo solver to provide GPGPU capabilities to OpenFOAM") + DESCRIPTION "A wrapper for Ginkgo solver to provide GPGPU capabilities to OpenFOAM") if(NOT DEFINED ENV{FOAM_SRC}) message(FATAL_ERROR "You must source OpenFOAM before building OGL") endif() -# Configuration options +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(BUILD_SHARED_LIBS ON) +# Configuration options include(cmake/build_type_helpers.cmake) list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake/Modules/") include(CheckIncludeFileCXX) option(OGL_USE_EXTERNAL_GINKGO "Use external ginkgo" FALSE) option(OGL_DATA_VALIDATION "Check if resulting matrix data is valid" FALSE) -option(OGL_USE_FOAM_FOUNDATION_VERSION - "Build OGL for the OpenFOAM Foundation version" FALSE) +option(OGL_USE_FOAM_FOUNDATION_VERSION "Build OGL for the OpenFOAM Foundation version" FALSE) option(OGL_BUILD_UNITTEST "Build OGL unittests" FALSE) option(OGL_CUDA_ARCHITECTURES "Which cuda architecture to build for" "native") -option(GINKGO_BUILD_CUDA "Build Ginkgo with cuda backend" FALSE) -option(GINKGO_BUILD_HIP "Build Ginkgo with hip backend" FALSE) -option(GINKGO_BUILD_SYCL "Build Ginkgo with sycl backend" FALSE) -option(GINKGO_BUILD_OMP "Build Ginkgo with omp backend" FALSE) option(GINKGO_BUILD_REFERENCE "Build Ginkgo with reference backend" TRUE) -option(OGL_ALLOW_REFERENCE_ONLY - "Enable builds of Ginkgo with reference backend only" FALSE) +option(OGL_ALLOW_REFERENCE_ONLY "Enable builds of Ginkgo with reference backend only" FALSE) +option(OGL_ENABLE_SANITIZE_ADDRESS "Enable address sanitizer" FALSE) +option(OGL_ENABLE_SANITIZE_LEAK "Enable leak sanitizer" FALSE) +option(OGL_ENABLE_SANITIZE_UB "Enable undefined behaviour sanitizer" FALSE) +option(OGL_ENABLE_SANITIZE_THREAD "Enable thread sanitizer" FALSE) +option(OGL_ENABLE_SANITIZE_MEMORY "Enable memory sanitizer" FALSE) +option(OGL_DEVEL_TOOLS "Add development tools to the build system" FALSE) + +option(GINKGO_WITH_OGL_EXTENSIONS "Whether ginkgo was build with OGL extension" FALSE) +option(GINKGO_BUILD_REFERENCE "Build Ginkgo with reference backend" TRUE) + option(GINKGO_FORCE_GPU_AWARE_MPI "Build Ginkgo using device aware MPI" TRUE) -option(GINKGO_WITH_OGL_EXTENSIONS "Whether ginkgo was build with OGL extension" - FALSE) +option(GINKGO_BUILD_SYCL "Build Ginkgo with sycl backend" FALSE) +option(GINKGO_BUILD_OMP "Build Ginkgo with omp backend" FALSE) + +include(cmake/AutoEnableDevice.cmake) + +set(GINKGO_BUILD_MPI + ON + CACHE INTERNAL "") +set(GINKGO_BUILD_TESTS + OFF + CACHE INTERNAL "") +set(GINKGO_BUILD_EXAMPLES + OFF + CACHE INTERNAL "") +set(GINKGO_BUILD_BENCHMARKS + OFF + CACHE INTERNAL "") if((NOT OGL_ALLOW_REFERENCE_ONLY) AND (NOT OGL_USE_EXTERNAL_GINKGO)) if((NOT GINKGO_BUILD_CUDA) @@ -43,59 +64,83 @@ if((NOT OGL_ALLOW_REFERENCE_ONLY) AND (NOT OGL_USE_EXTERNAL_GINKGO)) AND (NOT GINKGO_BUILD_SYCL)) message( FATAL_ERROR - "No GPU Backend was selected, set either:\n -DGINKGO_BUILD_CUDA=True\n -DGINKGO_BUILD_HIP=True\n -DGINKGO_BUILD_SYCL=True\n or set:\n -DOGL_ALLOW_REFERENCE_ONLY=True\n to turn off this check" - ) + "No GPU Backend was selected, set either:\n\ +-DGINKGO_BUILD_CUDA=True\n\ +-DGINKGO_BUILD_HIP=True\n\ +-DGINKGO_BUILD_SYCL=True\n\ +or set:\n\ +-DOGL_ALLOW_REFERENCE_ONLY=True\n\ +to turn off this check") endif() endif() set(GINKGO_CHECKOUT_VERSION - "fc86d48b78cebd2b2c5833a2dcf0fe40f615cf19" + "ogl_0600_gko190" CACHE STRING "Use specific version of ginkgo") +if(OGL_DEVEL_TOOLS) + find_program(PRE_COMMIT pre-commit) + if(NOT PRE_COMMIT) + message( + FATAL_ERROR + "The pre-commit command was not found. " + "It is necessary if you want to commit changes to OGL. " + "If that is not the case, set OGL_DEVEL_TOOLS=OFF. " + "Otherwise install pre-commit via pipx (or pip) using:\n" + " pipx install pre-commit") + endif() + + execute_process( + COMMAND "${PRE_COMMIT}" "install" + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + RESULT_VARIABLE pre-commit-result + OUTPUT_VARIABLE pre-commit-output + ERROR_VARIABLE pre-commit-error) + if(pre-commit-result) + message( + FATAL_ERROR "Failed to install the git hooks via pre-commit." + "Please check the error message:\n" "${pre-commit-output}\n${pre-commit-error}") + endif() + if(pre-commit-output MATCHES "^Running in migration mode with existing hooks") + message(WARNING "An existing git hook was encountered during `pre-commit install`." + "The old git hook will also be executed." + "Consider removing it with `pre-commit install -f`") + elseif(NOT pre-commit-output MATCHES "^pre-commit installed at") + message(WARNING "`pre-commit install` did not exit normally." + "Please check the output message:\n" "${pre-commit-output}") + endif() + + add_custom_target( + pre-commit + COMMENT "execute precommit" + COMMAND bash -c "${PRE_COMMIT} run" + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + VERBATIM) +endif() + include(CheckIncludeFileCXX) check_include_file_cxx(cxxabi.h GKO_HAVE_CXXABI_H) include(cmake/package_helpers.cmake) include(cmake/build_helpers.cmake) include(cmake/install_helpers.cmake) +include(cmake/Sanitizer.cmake) +include(cmake/OpenFOAM.cmake) -ginkgo_find_package(Ginkgo "Ginkgo::ginkgo" FALSE 1.7.0) - -add_subdirectory(third_party) - -# If ginkgo is not found at this point, look for the third_party installation -if(NOT ${OGL_USE_EXTERNAL_GINKGO}) - find_package( - Ginkgo - CONFIG - 1.7.0 - REQUIRED - PATHS - $ENV{FOAM_USER_LIBBIN}/cmake/Ginkgo - ${CMAKE_CURRENT_BINARY_DIR}/third_party/ginkgo/build/install/lib/cmake) -endif() - -# C++ 14 standard or later is required to interface with Ginkgo OGL itself uses -# some c++ 17 features - -set(CMAKE_CXX_STANDARD 17) -set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(CMAKE_POSITION_INDEPENDENT_CODE ON) +ginkgo_find_package(Ginkgo "Ginkgo::ginkgo" FALSE 1.8.0) +include(cmake/ginkgo.cmake) set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wall -O0 -ggdb") -set(CMAKE_CXX_FLAGS_RELEASE - "${CMAKE_CXX_FLAGS_RELEASE} \ - -Wall -Wpedantic -Wextra -march=native -fopenmp -fPIC -Wno-undefined-var-template" -) + +set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} \ + -Wall -Wpedantic -Wextra -march=native -fopenmp -Wno-undefined-var-template") if("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang") message("Compiling with clang") set(CMAKE_EXPORT_COMPILE_COMMANDS ON) - set(CMAKE_CXX_FLAGS_DEBUG - "${CMAKE_CXX_FLAGS_DEBUG} \ - -fno-omit-frame-pointer \ - -fsanitize=address") + set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} \ + -fno-omit-frame-pointer") set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} \ @@ -112,9 +157,10 @@ elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU") set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} \ --coverage\ + --fopenmp \ -fprofile-arcs \ - -ggdb3") - # * fsanitize = address \ - ftest - coverage \ + -ggdb3 \ + ") set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} \ @@ -125,7 +171,6 @@ elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU") ") endif() -option(BUILD_SHARED_LIBS "Build shared library" ON) add_library(OGL "") target_compile_definitions(OGL PUBLIC NoRepository) @@ -138,73 +183,53 @@ if(APPLE) target_compile_definitions(OGL PRIVATE APPLE arm64) else() target_compile_definitions(OGL PRIVATE linux64) + target_link_libraries(OGL PRIVATE stdc++fs) endif() -# Add a custom command that produces version.cpp, plus a dummy output that's not -# actually produced, in order to always force re-run version.cmake before the -# build +# Add a custom command that produces version.cpp, plus a dummy output that's not actually produced, +# in order to always force re-run version.cmake before the build add_custom_command( - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/version.C - ${CMAKE_CURRENT_BINARY_DIR}/_version.C + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/version.cpp ${CMAKE_CURRENT_BINARY_DIR}/_version.cpp COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/version.cmake - ${GINKGO_CHECKOUT_VERSION}) + ${GINKGO_CHECKOUT_VERSION} + COMMENT "Create a version file") + +add_subdirectory(include) target_sources( OGL - PRIVATE common/common.C - ${CMAKE_CURRENT_BINARY_DIR}/version.C - lduLduBase/lduLduBase.C - StoppingCriterion/StoppingCriterion.C - DevicePersistent/Base/Base.C - DevicePersistent/Partition/Partition.C - DevicePersistent/Array/Array.C - DevicePersistent/Vector/Vector.C - DevicePersistent/DeviceIdGuard/DeviceIdGuard.C - DevicePersistent/ExecutorHandler/ExecutorHandler.C - DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.C - Preconditioner/Preconditioner.C - BaseWrapper/lduBase/GKOlduBase.C - BaseWrapper/CoupledLduBase/GKOCoupledLduBase.C - HostMatrix/HostMatrix.C - HostMatrix/HostMatrixFreeFunctions.C - Solver/CG/GKOCG.C - Solver/BiCGStab/GKOBiCGStab.C - # Solver / IR / GKOIR.C Solver / Multigrid / GKOMultigrid.C - Solver/GMRES/GKOGMRES.C - # LduMatrix / GKOACG / GKOACG.C - PUBLIC common/common.H - StoppingCriterion/StoppingCriterion.H - lduLduBase/lduLduBase.H - HostMatrix/HostMatrix.H - DevicePersistent/Base/Base.H - DevicePersistent/Partition/Partition.H - DevicePersistent/Array/Array.H - DevicePersistent/Vector/Vector.H - DevicePersistent/ExecutorHandler/ExecutorHandler.H - DevicePersistent/DeviceIdGuard/DeviceIdGuard.H - DevicePersistent/CsrMatrixWrapper/CsrMatrixWrapper.H - Preconditioner/Preconditioner.H - BaseWrapper/lduBase/GKOlduBase.H - BaseWrapper/CoupledLduBase/GKOCoupledLduBase.H - Solver/CG/GKOCG.H - Solver/IR/GKOIR.H - # Solver / Multigrid / GKOMultigrid.H - Solver/BiCGStab/GKOBiCGStab.H - Solver/GMRES/GKOGMRES.H - # LduMatrix / GKOACG / GKOACG.H + PRIVATE src/common.cpp + src/CommunicationPattern.cpp + src/Preconditioner.cpp + src/lduLduBase.cpp + src/StoppingCriterion.cpp + src/BaseWrapper/lduBase.cpp + src/BaseWrapper/CoupledLduBase.cpp + ${CMAKE_CURRENT_BINARY_DIR}/version.cpp + src/DevicePersistent/Base.cpp + src/DevicePersistent/Vector.cpp + src/DevicePersistent/DeviceIdGuard.cpp + src/DevicePersistent/DistributedMatrixAdapter.cpp + src/DevicePersistent/ExecutorHandler.cpp + src/MatrixWrapper/HostMatrix.cpp + src/MatrixWrapper/SparsityPattern.cpp + src/MatrixWrapper/Distributed.cpp + src/MatrixWrapper/Combination.cpp + src/Repartitioner.cpp + src/Solver/CG.cpp + # src/Solver/BiCGStab.cpp + # src/Solver/GMRES.cpp + # src/Solver/Multigrid.cpp src/Solver/IR.cpp ) -target_include_directories( - OGL SYSTEM - PUBLIC $ENV{FOAM_SRC}/finiteVolume/lnInclude - $ENV{FOAM_SRC}/meshTools/lnInclude $ENV{FOAM_SRC}/OpenFOAM/lnInclude - $ENV{FOAM_SRC}/OSspecific/POSIX/lnInclude ${CMAKE_CURRENT_SOURCE_DIR}) +enable_sanitizers( + OGL ${OGL_ENABLE_SANITIZE_ADDRESS} ${OGL_ENABLE_SANITIZE_LEAK} ${OGL_ENABLE_SANITIZE_UB} + ${OGL_ENABLE_SANITIZE_THREAD} ${OGL_ENABLE_SANITIZE_MEMORY}) target_link_libraries( OGL - PRIVATE $ENV{FOAM_LIBBIN}/libOpenFOAM.so $ENV{FOAM_LIBBIN}/libfiniteVolume.so - $ENV{FOAM_LIBBIN}/$ENV{FOAM_MPI}/libPstream.so - PUBLIC Ginkgo::ginkgo stdc++fs) + PUBLIC OGL_public_api + PRIVATE OpenFOAM Ginkgo::ginkgo) if(${GINKGO_WITH_OGL_EXTENSIONS}) target_compile_definitions(OGL PRIVATE GINKGO_WITH_OGL_EXTENSIONS=1) @@ -228,12 +253,22 @@ endif() if(${OGL_BUILD_UNITTEST}) enable_testing() - add_subdirectory(unitTests) + add_subdirectory(test/unit) endif() add_custom_target( cmake-format COMMAND find . -name CMakeLists.txt -exec cmake-format -i {} + - WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}) - -install(TARGETS OGL DESTINATION $ENV{FOAM_USER_LIBBIN}) + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} + COMMENT "a target to format cmake files") + +install( + TARGETS OGL + ginkgo + ginkgo_device + ginkgo_hip + ginkgo_cuda + ginkgo_omp + ginkgo_dpcpp + ginkgo_reference + DESTINATION $ENV{FOAM_USER_LIBBIN}) diff --git a/CMakePresets.json b/CMakePresets.json index 890dd7613..2659bf43c 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -1,72 +1,66 @@ { - "version": 2, + "version": 6, "cmakeMinimumRequired": { "major": 3, - "minor": 20, + "minor": 25, "patch": 0 }, "configurePresets": [ { "name": "default", - "displayName": "Default Config", + "hidden": true, "description": "Default build using Ninja generator", "generator": "Ninja", - "binaryDir": "${sourceDir}/build/Release", "cacheVariables": { - "CMAKE_BUILD_TYPE": "Release" + "CMAKE_INSTALL_PREFIX": "$env{FOAM_USER_LIBBIN}/.." } }, { - "name": "ninja-cpuonly-release", - "inherits": "default", - "displayName": "No GPU, Ninja", - "description": "Release build without GPU support using Ninja Multi-Config generator", - "generator": "Ninja Multi-Config", - "binaryDir": "${sourceDir}/build/ReleaseCPUOnly", + "name": "release", + "inherits": [ + "default" + ], + "description": "Default build using Ninja generator", + "binaryDir": "${sourceDir}/build/release", "cacheVariables": { - "OGL_ALLOW_REFERENCE_ONLY": { - "type": "BOOL", - "value": "ON" - } + "CMAKE_BUILD_TYPE": "Release", + "CMAKE_INSTALL_PREFIX": "$env{FOAM_USER_LIBBIN}/.." } }, { - "name": "ninja-cuda-release", - "inherits": "default", - "displayName": "CUDA, Ninja", - "description": "Release build for cuda using Ninja Multi-Config generator", - "generator": "Ninja Multi-Config", - "binaryDir": "${sourceDir}/build/ReleaseCuda", + "name": "debug", + "inherits": [ + "default" + ], + "description": "Default build using Ninja generator", + "binaryDir": "${sourceDir}/build/debug", "cacheVariables": { - "GINKGO_BUILD_CUDA": { + "CMAKE_BUILD_TYPE": "Debug", + "OGL_ALLOW_REFERENCE_ONLY": { "type": "BOOL", "value": "ON" - } - } - }, - { - "name": "ninja-hip-release", - "inherits": "default", - "displayName": "HIP, Ninja", - "description": "Release build for hip using Ninja Multi-Config generator", - "generator": "Ninja Multi-Config", - "binaryDir": "${sourceDir}/build/ReleaseHip", - "cacheVariables": { - "GINKGO_BUILD_HIP": { + }, + "OGL_BUILD_UNITTEST": { "type": "BOOL", "value": "ON" } } }, { - "name": "ninja-sycl-release", - "inherits": "default", - "displayName": "SYCL, Ninja", - "description": "Release build for sycl using Ninja Multi-Config generator", - "generator": "Ninja Multi-Config", - "binaryDir": "${sourceDir}/build/ReleaseSycl", + "name": "develop", + "description": "Default build using Ninja generator", + "binaryDir": "${sourceDir}/build/develop", + "inherits": [ + "default" + ], "cacheVariables": { - "GINKGO_BUILD_SYCL": { + "CMAKE_BUILD_TYPE": "RelWithDebInfo", + "CMAKE_INSTALL_PREFIX": "$env{FOAM_USER_LIBBIN}/..", + "OGL_ALLOW_REFERENCE_ONLY": { + "type": "BOOL", + "value": "ON" + }, + "OGL_BUILD_UNITTEST": { "type": "BOOL", "value": "ON" } @@ -75,29 +69,46 @@ ], "buildPresets": [ { - "name": "default", - "configurePreset": "default" - }, - { - "name": "ninja-cpuonly-release", - "configurePreset": "ninja-cpuonly-release", + "name": "release", + "configurePreset": "release", "configuration": "Release" }, { - "name": "ninja-cuda-release", - "configurePreset": "ninja-cuda-release", + "name": "debug", + "configurePreset": "debug", "configuration": "Release" }, { - "name": "ninja-hip-release", - "configurePreset": "ninja-hip-release", + "name": "develop", + "configurePreset": "develop", + "configuration": "Develop" + } + ], + "testPresets": [ + { + "name": "debug", + "configurePreset": "debug", "configuration": "Release" }, { - "name": "ninja-sycl-release", - "configurePreset": "ninja-sycl-release", - "configuration": "Release" + "name": "develop", + "configurePreset": "develop", + "configuration": "Develop" + } + ], + "workflowPresets": [ + { + "name": "release", + "steps": [ + { + "type": "configure", + "name": "release" + }, + { + "type": "build", + "name": "release" + } + ] } ] - } diff --git a/DevicePersistent/Array/Array.H b/DevicePersistent/Array/Array.H deleted file mode 100644 index 2981ad885..000000000 --- a/DevicePersistent/Array/Array.H +++ /dev/null @@ -1,231 +0,0 @@ -// SPDX-FileCopyrightText: 2024 OGL authors -// -// SPDX-License-Identifier: GPL-3.0-or-later - - -#pragma once - -#include - -#include - -#include "DevicePersistent/Base/Base.H" -#include "DevicePersistent/ExecutorHandler/ExecutorHandler.H" -#include "common/common.H" - -namespace Foam { - - -template -struct ArrayInitFunctor { - const word name_; - - const ExecutorHandler &exec_; - - const label size_; - - const label verbose_; - - const bool on_device_; - - // Memory from which array will be initialised - const T *other_; - - - ArrayInitFunctor(const ExecutorHandler &exec, const word name, - const label size, const label verbose, - const bool on_device) - : exec_(exec), - name_(name), - size_(size), - on_device_(on_device), - verbose_(verbose), - other_(NULL) - {} - - ArrayInitFunctor(const ExecutorHandler &exec, const T *other, - const word name, const label size, const label verbose, - const bool on_device) - : exec_(exec), - name_(name), - size_(size), - on_device_(on_device), - verbose_(verbose), - other_(other) - {} - - - // update persistent array from host memory - void update(std::shared_ptr> persistent_array) const - { - if (other_ != NULL) { - word msg{"updating array " + name_ + " of size " + - std::to_string(size_)}; - LOG_1(verbose_, msg) - - auto host_view = gko::array::view(exec_.get_ref_exec(), size_, - const_cast(other_)); - persistent_array->operator=(host_view); - } - } - - std::shared_ptr> init() const - { - auto exec = - (on_device_) ? exec_.get_device_exec() : exec_.get_ref_exec(); - word msg{"initialising array " + name_ + " of size " + - std::to_string(size_)}; - msg += (on_device_) ? " on device" : " on host"; - LOG_1(verbose_, msg) - - if (other_ != NULL) { - auto host_view = gko::array::view(exec_.get_ref_exec(), size_, - const_cast(other_)); - return std::make_shared>(exec, host_view); - } else { - return std::make_shared>(exec, size_); - } - } -}; - -template -class PersistentArray - : public PersistentBase, ArrayInitFunctor> { - using vec = gko::matrix::Dense; - - T *memory_; - - mutable label size_; - - const ExecutorHandler &exec_; - - // indicating if the underlying array needs to - // updated even if was found in the object registry - const bool update_; - - // The same as dpb but for the global data - // NOTE if on the host: - // - This should exist with the full length only on the master - // on the device: - // - global data is needed on the device for example for the - // col and row idxs - mutable std::shared_ptr> persistent_object_global_{}; - - -public: - /* PersistentArray constructor - * - * @param name name of the underlying field or data - * @param objectRegistry reference to registry for storage - * @param exec executor handler - * @param verbose whether to print infos out - * @param update whether to update the underlying array if found in registry - * TODO fix documentation - * @param init_on_device whether to update the underlying array if found in - * registry - */ - PersistentArray(const word name, const objectRegistry &db, - const ExecutorHandler &exec, const label size, - const label verbose, const bool update, - const bool on_device) - : PersistentBase, ArrayInitFunctor>( - name, db, - ArrayInitFunctor(exec, name, size, verbose, on_device), update, - verbose), - size_(size), - exec_(exec), - update_(update) - {} - - /* PersistentArray constructor using existing memory - * - * @param memory ptr to memory on host from which the gko array is - * initialized - * @param name name of the underlying field or data - * @param objectRegistry reference to registry for storage - * @param exec executor handler - * @param verbose whether to print infos out - * @param update whether to update the underlying array if found in registry - * @param init_on_device whether the array is to be initialized on the - * device or host - */ - PersistentArray(T *memory, const word name, const objectRegistry &db, - const ExecutorHandler &exec, const label size, - const label verbose, const bool update, - const bool init_on_device) - : PersistentBase, ArrayInitFunctor>( - name, db, - ArrayInitFunctor(exec, memory, name, size, verbose, - init_on_device), - update, verbose), - memory_(memory), - size_(size), - exec_(exec), - update_(update) - {} - - bool get_update() const { return update_; } - - T *get_data() const { return this->get_persistent_object()->get_data(); } - - void set_data(T *data) { this->get_persistent_object()->get_data() = data; } - - const T *get_const_data() const - { - return this->get_persistent_object()->get_const_data(); - } - - - std::shared_ptr> get_array() const - { - return this->get_persistent_object(); - } - - /* Create a Dense vector from underlying data - * */ - std::shared_ptr> get_dense_vec() const - { - auto array = get_array(); - auto result = gko::share(vec::create( - exec_.get_device_exec(), gko::dim<2>(array->get_num_elems(), 1), - gko::array::view(exec_.get_device_exec(), array->get_num_elems(), - get_data()), - 1)); - return result; - } - - /* Copies from a distributed::Vector back into raw host memory - * - * TODO store and use *memory - * */ - void copy_back( - const std::shared_ptr> dist_x) - { - auto host_view = gko::array::view(exec_.get_ref_exec(), size_, - const_cast(memory_)); - - auto dist_device_exec = dist_x->get_executor(); - auto x_view = gko::array::view(dist_device_exec, size_, - dist_x->get_local()->get_values()); - - host_view = x_view; - } - - void copy_back(const std::shared_ptr> device_x) - { - auto host_view = gko::array::view(exec_.get_ref_exec(), size_, - const_cast(memory_)); - - auto x_view = gko::array::view(exec_.get_device_exec(), size_, - device_x->get_values()); - - host_view = x_view; - } - - void set_size(const label size) const { size_ = size; } - - - const ExecutorHandler &get_exec_handler() const { return exec_; } -}; - -} // namespace Foam diff --git a/DevicePersistent/Base/Base.C b/DevicePersistent/Base/Base.C deleted file mode 100644 index 66660a4d4..000000000 --- a/DevicePersistent/Base/Base.C +++ /dev/null @@ -1,36 +0,0 @@ -// SPDX-FileCopyrightText: 2024 OGL authors -// -// SPDX-License-Identifier: GPL-3.0-or-later - -#include "Base.H" - -namespace Foam { - -// to store the std::shared_ptr in the IO registry the type needs to be -// declared -defineTemplateTypeNameWithName(DevicePersistentBase>, - "PersistentCSRMatrix"); -defineTemplateTypeNameWithName(DevicePersistentBase>, - "PersistentScalarArray"); -defineTemplateTypeNameWithName( - DevicePersistentBase>, - "PersistentScalarVector"); -defineTemplateTypeNameWithName(DevicePersistentBase>, - "PersistentLabelArray"); -defineTemplateTypeNameWithName(DevicePersistentBase>, - "PersistentLabelVec"); -defineTemplateTypeNameWithName(DevicePersistentBase>, - "PersistentScalarVec"); -defineTemplateTypeNameWithName(DevicePersistentBase, - "PersistentLinOp"); - -// typedef needed to avoid confusion with the comma separated template -// arguments as macro arguments -typedef gko::experimental::distributed::localized_partition