From 0b5311caf8a1f1da76685766b19d57d9e38f425d Mon Sep 17 00:00:00 2001 From: jan-janssen Date: Fri, 16 Feb 2024 22:26:22 +0100 Subject: [PATCH 1/7] Use mpiexec only for parallel execution --- pympipool/shared/interface.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/pympipool/shared/interface.py b/pympipool/shared/interface.py index 0fee6905..325b8d73 100644 --- a/pympipool/shared/interface.py +++ b/pympipool/shared/interface.py @@ -62,9 +62,11 @@ def generate_command(self, command_lst): cores=self._cores, oversubscribe=self._oversubscribe, ) - return super().generate_command( + command_final_lst = super().generate_command( command_lst=command_prepend_lst + command_lst, ) + print(command_final_lst) + return command_final_lst class SrunInterface(SubprocessInterface): @@ -98,10 +100,13 @@ def generate_command(self, command_lst): def generate_mpiexec_command(cores, oversubscribe=False): - command_prepend_lst = [MPI_COMMAND, "-n", str(cores)] - if oversubscribe: - command_prepend_lst += ["--oversubscribe"] - return command_prepend_lst + if cores == 1: + return [] + else: + command_prepend_lst = [MPI_COMMAND, "-n", str(cores)] + if oversubscribe: + command_prepend_lst += ["--oversubscribe"] + return command_prepend_lst def generate_slurm_command( From 308b7b4cb12807a08f1cf5b00bc2d7531bad82ec Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Fri, 16 Feb 2024 22:54:32 +0100 Subject: [PATCH 2/7] Update interface.py --- pympipool/shared/interface.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pympipool/shared/interface.py b/pympipool/shared/interface.py index 325b8d73..29c481e4 100644 --- a/pympipool/shared/interface.py +++ b/pympipool/shared/interface.py @@ -62,11 +62,9 @@ def generate_command(self, command_lst): cores=self._cores, oversubscribe=self._oversubscribe, ) - command_final_lst = super().generate_command( + return super().generate_command( command_lst=command_prepend_lst + command_lst, ) - print(command_final_lst) - return command_final_lst class SrunInterface(SubprocessInterface): From 596acb115b4259223bf0975d6fbff6c86d971548 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sun, 18 Feb 2024 07:42:55 +0100 Subject: [PATCH 3/7] Add CI for benchmark --- .github/workflows/benchmarl.yml | 66 +++++++++++++++++++++++++++++++++ tests/benchmark/llh.py | 47 +++++++++++++++++++++++ 2 files changed, 113 insertions(+) create mode 100644 .github/workflows/benchmarl.yml create mode 100644 tests/benchmark/llh.py diff --git a/.github/workflows/benchmarl.yml b/.github/workflows/benchmarl.yml new file mode 100644 index 00000000..3c10b02c --- /dev/null +++ b/.github/workflows/benchmarl.yml @@ -0,0 +1,66 @@ +# This workflow is used to run the unittest of pyiron + +name: Benchmark + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + build: + + runs-on: ${{ matrix.operating-system }} + strategy: + matrix: + include: + - operating-system: ubuntu-latest + python-version: '3.12' + label: linux-64-py-3-12-openmpi + prefix: /Users/runner/miniconda3/envs/my-env + environment-file: .ci_support/environment-openmpi.yml + + - operating-system: ubuntu-latest + python-version: '3.12' + label: linux-64-py-3-12-mpich + prefix: /usr/share/miniconda3/envs/my-env + environment-file: .ci_support/environment-mpich.yml + + steps: + - uses: actions/checkout@v2 + - uses: conda-incubator/setup-miniconda@v2.2.0 + with: + python-version: ${{ matrix.python-version }} + mamba-version: "*" + channels: conda-forge + miniforge-variant: Mambaforge + channel-priority: strict + auto-update-conda: true + environment-file: ${{ matrix.environment-file }} + - name: Test + shell: bash -l {0} + timeout-minutes: 5 + run: | + mamba install -y flux-core=0.58.0 coverage + pip install versioneer[toml]==0.29 + pip install . --no-deps --no-build-isolation + python tests/benchmark/llh.py static + python tests/benchmark/llh.py process + python tests/benchmark/llh.py thread + mpiexec -n 4 python -m mpi4py.futures tests/benchmark/llh.py mpi4py + python tests/benchmark/llh.py pympipool + env: + OMPI_MCA_plm: 'isolated' + OMPI_MCA_rmaps_base_oversubscribe: 'yes' + OMPI_MCA_btl_vader_single_copy_mechanism: 'none' + - name: Test Flux + shell: bash -l {0} + timeout-minutes: 5 + run: > + flux start + python tests/benchmark/llh.py flux + env: + OMPI_MCA_plm: 'isolated' + OMPI_MCA_rmaps_base_oversubscribe: 'yes' + OMPI_MCA_btl_vader_single_copy_mechanism: 'none' diff --git a/tests/benchmark/llh.py b/tests/benchmark/llh.py new file mode 100644 index 00000000..13958f3f --- /dev/null +++ b/tests/benchmark/llh.py @@ -0,0 +1,47 @@ +import sys +from time import time + + +def llh_numpy(mean, sigma): + import numpy + data = numpy.random.normal(size=100000000).astype('float64') + s = (data - mean) ** 2 / (2 * (sigma ** 2)) + pdfs = numpy.exp(- s) + pdfs /= numpy.sqrt(2 * numpy.pi) * sigma + return numpy.log(pdfs).sum() + + +def run_with_executor(executor=None, mean=0.1, sigma=1.1, runs=32, **kwargs): + with executor(**kwargs) as exe: + future_lst = [exe.submit(llh_numpy, mean=mean, sigma=sigma) for i in range(runs)] + return [f.result() for f in future_lst] + + +def run_static(mean=0.1, sigma=1.1, runs=32): + return [llh_numpy(mean=mean, sigma=sigma) for i in range(runs)] + + +if __name__ == "__main__": + run_mode = sys.argv[1] + start_time = time() + if run_mode == "static": + run_static(mean=0.1, sigma=1.1, runs=32) + elif run_mode == "process": + from concurrent.futures import ProcessPoolExecutor + run_with_executor(executor=ProcessPoolExecutor, mean=0.1, sigma=1.1, runs=32, max_workers=4) + elif run_mode == "thread": + from concurrent.futures import ThreadPoolExecutor + run_with_executor(executor=ThreadPoolExecutor, mean=0.1, sigma=1.1, runs=32, max_workers=4) + elif run_mode == "pympipool": + from pympipool.mpi.executor import PyMPIExecutor + run_with_executor(executor=PyMPIExecutor, mean=0.1, sigma=1.1, runs=32, max_workers=4) + elif run_mode == "flux": + from pympipool.flux.executor import PyFluxExecutor + run_with_executor(executor=PyFluxExecutor, mean=0.1, sigma=1.1, runs=32, max_workers=4) + elif run_mode == "mpi4py": + from mpi4py.futures import MPIPoolExecutor + run_with_executor(executor=MPIPoolExecutor, mean=0.1, sigma=1.1, runs=32, max_workers=4) + else: + raise ValueError(run_mode) + stop_time = time() + print("Result:", stop_time-start_time) From b9d553cf4ba71230118486d8dffefec47b0e0abd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sun, 18 Feb 2024 07:50:13 +0100 Subject: [PATCH 4/7] save runtime in extra file --- .github/workflows/benchmarl.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/benchmarl.yml b/.github/workflows/benchmarl.yml index 3c10b02c..5b059631 100644 --- a/.github/workflows/benchmarl.yml +++ b/.github/workflows/benchmarl.yml @@ -40,16 +40,16 @@ jobs: environment-file: ${{ matrix.environment-file }} - name: Test shell: bash -l {0} - timeout-minutes: 5 + timeout-minutes: 10 run: | mamba install -y flux-core=0.58.0 coverage pip install versioneer[toml]==0.29 pip install . --no-deps --no-build-isolation - python tests/benchmark/llh.py static - python tests/benchmark/llh.py process - python tests/benchmark/llh.py thread - mpiexec -n 4 python -m mpi4py.futures tests/benchmark/llh.py mpi4py - python tests/benchmark/llh.py pympipool + python tests/benchmark/llh.py static >> timing.log + python tests/benchmark/llh.py process >> timing.log + python tests/benchmark/llh.py thread >> timing.log + mpiexec -n 4 python -m mpi4py.futures tests/benchmark/llh.py mpi4py >> timing.log + python tests/benchmark/llh.py pympipool >> timing.log env: OMPI_MCA_plm: 'isolated' OMPI_MCA_rmaps_base_oversubscribe: 'yes' @@ -59,7 +59,8 @@ jobs: timeout-minutes: 5 run: > flux start - python tests/benchmark/llh.py flux + python tests/benchmark/llh.py flux >> timing.log + cat timing.log env: OMPI_MCA_plm: 'isolated' OMPI_MCA_rmaps_base_oversubscribe: 'yes' From 705d47a302b5010167820aed87daa18dbef3fb6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sun, 18 Feb 2024 08:00:41 +0100 Subject: [PATCH 5/7] split running and analysis --- .github/workflows/benchmarl.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/benchmarl.yml b/.github/workflows/benchmarl.yml index 5b059631..1ab1187d 100644 --- a/.github/workflows/benchmarl.yml +++ b/.github/workflows/benchmarl.yml @@ -60,8 +60,12 @@ jobs: run: > flux start python tests/benchmark/llh.py flux >> timing.log - cat timing.log env: OMPI_MCA_plm: 'isolated' OMPI_MCA_rmaps_base_oversubscribe: 'yes' OMPI_MCA_btl_vader_single_copy_mechanism: 'none' + - name: Analyse log + shell: bash -l {0} + timeout-minutes: 5 + run: | + cat timing.log \ No newline at end of file From e05ad846a87204552da5d101d89a06e2cf950131 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sun, 18 Feb 2024 08:12:41 +0100 Subject: [PATCH 6/7] benchmark without flux --- .github/workflows/benchmarl.yml | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/.github/workflows/benchmarl.yml b/.github/workflows/benchmarl.yml index 1ab1187d..6cef2983 100644 --- a/.github/workflows/benchmarl.yml +++ b/.github/workflows/benchmarl.yml @@ -42,7 +42,6 @@ jobs: shell: bash -l {0} timeout-minutes: 10 run: | - mamba install -y flux-core=0.58.0 coverage pip install versioneer[toml]==0.29 pip install . --no-deps --no-build-isolation python tests/benchmark/llh.py static >> timing.log @@ -50,22 +49,8 @@ jobs: python tests/benchmark/llh.py thread >> timing.log mpiexec -n 4 python -m mpi4py.futures tests/benchmark/llh.py mpi4py >> timing.log python tests/benchmark/llh.py pympipool >> timing.log + cat timing.log env: OMPI_MCA_plm: 'isolated' OMPI_MCA_rmaps_base_oversubscribe: 'yes' OMPI_MCA_btl_vader_single_copy_mechanism: 'none' - - name: Test Flux - shell: bash -l {0} - timeout-minutes: 5 - run: > - flux start - python tests/benchmark/llh.py flux >> timing.log - env: - OMPI_MCA_plm: 'isolated' - OMPI_MCA_rmaps_base_oversubscribe: 'yes' - OMPI_MCA_btl_vader_single_copy_mechanism: 'none' - - name: Analyse log - shell: bash -l {0} - timeout-minutes: 5 - run: | - cat timing.log \ No newline at end of file From f76f7b6bb94e4c76bfe3a94af747a544d8ae4cf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sun, 18 Feb 2024 08:35:13 +0100 Subject: [PATCH 7/7] validate benchmark --- .../workflows/{benchmarl.yml => benchmark.yml} | 1 + tests/benchmark/llh.py | 2 +- tests/benchmark/test_results.py | 17 +++++++++++++++++ 3 files changed, 19 insertions(+), 1 deletion(-) rename .github/workflows/{benchmarl.yml => benchmark.yml} (96%) create mode 100644 tests/benchmark/test_results.py diff --git a/.github/workflows/benchmarl.yml b/.github/workflows/benchmark.yml similarity index 96% rename from .github/workflows/benchmarl.yml rename to .github/workflows/benchmark.yml index 6cef2983..3d47fec7 100644 --- a/.github/workflows/benchmarl.yml +++ b/.github/workflows/benchmark.yml @@ -50,6 +50,7 @@ jobs: mpiexec -n 4 python -m mpi4py.futures tests/benchmark/llh.py mpi4py >> timing.log python tests/benchmark/llh.py pympipool >> timing.log cat timing.log + python -m unittest tests/benchmark/test_results.py env: OMPI_MCA_plm: 'isolated' OMPI_MCA_rmaps_base_oversubscribe: 'yes' diff --git a/tests/benchmark/llh.py b/tests/benchmark/llh.py index 13958f3f..6973e2ac 100644 --- a/tests/benchmark/llh.py +++ b/tests/benchmark/llh.py @@ -44,4 +44,4 @@ def run_static(mean=0.1, sigma=1.1, runs=32): else: raise ValueError(run_mode) stop_time = time() - print("Result:", stop_time-start_time) + print(run_mode, stop_time-start_time) diff --git a/tests/benchmark/test_results.py b/tests/benchmark/test_results.py new file mode 100644 index 00000000..3886cee9 --- /dev/null +++ b/tests/benchmark/test_results.py @@ -0,0 +1,17 @@ +import unittest + + +class TestResults(unittest.TestCase): + def test_result(self): + with open("timing.log") as f: + content = f.readlines() + timing_dict = {l.split()[0]: float(l.split()[1]) for l in content} + self.assertEqual(min(timing_dict, key=timing_dict.get), "process") + self.assertEqual(max(timing_dict, key=timing_dict.get), "static") + self.assertTrue(timing_dict["process"] < timing_dict["pympipool"]) + self.assertTrue(timing_dict["pympipool"] < timing_dict["process"] * 1.1) + self.assertTrue(timing_dict["process"] < timing_dict["mpi4py"]) + self.assertTrue(timing_dict["pympipool"] < timing_dict["mpi4py"]) + self.assertTrue(timing_dict["mpi4py"] < timing_dict["process"] * 1.15) + self.assertTrue(timing_dict["thread"] < timing_dict["static"]) + self.assertTrue(timing_dict["mpi4py"] < timing_dict["thread"])