initial commit

.ci_support/environment.yml

@@ -0,0 +1,9 @@
+channels:
+- conda-forge
+dependencies:
+- python
+- coverage
+- coveralls
+- dill =0.3.5.1
+- mpi4py =3.1.3
+- tqdm =4.64.0

.github/delete-merged-branch-config.yml

@@ -0,0 +1,3 @@
+exclude:
+  - main
+delete_closed_pr: false

.github/dependabot.yml

@@ -0,0 +1,7 @@
+version: 2
+updates:
+- package-ecosystem: pip
+  directory: "/"
+  schedule:
+    interval: daily
+  open-pull-requests-limit: 10

.github/workflows/black.yml

@@ -0,0 +1,28 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: Black
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: conda-incubator/setup-miniconda@v2
+      with:
+        auto-update-conda: true
+        python-version: 3.9
+        channel-priority: strict
+    - name: Setup
+      shell: bash -l {0}
+      run: |
+        conda install -c conda-forge black
+    - name: Test
+      shell: bash -l {0}
+      run: black --check pympipool

.github/workflows/deploy.yml

@@ -0,0 +1,32 @@
+name: PyPi Release
+
+on:
+    push:
+    pull_request:
+
+# based on https://github.com/pypa/gh-action-pypi-publish
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+
+    - name: Install dependencies
+      run: >-
+        python -m pip install --user --upgrade setuptools wheel
+    - name: Convert dependencies
+      run: >-
+        sed -i 's/==/>=/g' setup.py; cat setup.py
+    - name: Build
+      run: >-
+        python setup.py sdist bdist_wheel
+    - name: Publish distribution 📦 to PyPI
+      if: startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release'
+      uses: pypa/gh-action-pypi-publish@master
+      with:
+        user: __token__
+        password: ${{ secrets.pypi_password }}

.github/workflows/pypicheck.yml

@@ -0,0 +1,30 @@
+name: Pip check
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.operating-system }}
+    strategy:
+      matrix:
+        operating-system: [ubuntu-latest]
+        python-version: [3.9]
+
+    steps:
+    - uses: actions/checkout@v2
+    - uses: conda-incubator/setup-miniconda@v2
+      with:
+        auto-update-conda: true
+        python-version: ${{ matrix.python-version }}
+        channel-priority: strict
+        environment-file: .ci_support/environment.yml
+    - name: Setup
+      shell: bash -l {0}
+      run: |
+        pip install --no-deps .
+        pip check

.github/workflows/unittest.yml

@@ -0,0 +1,38 @@
+# This workflow is used to run the unittest of pyiron
+
+name: Unittests
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.operating-system }}
+    strategy:
+      matrix:
+        operating-system: [ubuntu-latest, windows-latest, macos-latest]
+        python-version: ['3.10']
+        include:
+        - operating-system: ubuntu-latest
+          python-version: 3.8
+        - operating-system: ubuntu-latest
+          python-version: 3.9
+
+    steps:
+    - uses: actions/checkout@v2
+    - uses: conda-incubator/setup-miniconda@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+        mamba-version: "*"
+        channels: conda-forge
+        channel-priority: strict
+        auto-update-conda: true
+        environment-file: .ci_support/environment.yml
+    - name: Test
+      shell: bash -l {0}
+      timeout-minutes: 30
+      run: coverage run --omit pympipool/_version.py -m unittest discover tests

LICENSE

@@ -0,0 +1,29 @@
+BSD 3-Clause License
+
+Copyright (c) 2022, Jan Janssen
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+* Neither the name of the copyright holder nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

MANIFEST.in

@@ -0,0 +1,3 @@
+include versioneer.py
+include pympipool/_version.py
+include LICENSE

README.md

@@ -0,0 +1,15 @@
+# pympipool
+Scale functions over multiple compute nodes using mpi4py
+
+```python
+from pympipool import Pool
+
+def calc(i):
+    import numpy as np
+    return np.array(i ** 2)
+
+with Pool(cores=2) as p:
+    print(p.map(function=calc, lst=[1, 2, 3, 4]))
+
+>>> [array(1), array(4), array(9), array(16)]
+```

pympipool/__init__.py

@@ -0,0 +1,66 @@
+import subprocess
+import os
+import dill
+import inspect
+
+
+class Pool(object):
+    def __init__(self, cores=1):
+        self._cores = cores
+        self._process = None
+
+    def __enter__(self):
+        path = os.path.abspath(
+            os.path.join(__file__, "..", "__main__.py")
+        )
+        self._process = subprocess.Popen(
+            [
+                "mpiexec",
+                "--oversubscribe",
+                "-n",
+                str(self._cores),
+                "python",
+                "-m",
+                "mpi4py.futures",
+                path
+            ],
+            stdout=subprocess.PIPE,
+            stderr=None,
+            stdin=subprocess.PIPE,
+            # cwd=self.working_directory,
+        )
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self._send_raw(
+            input_dict={"c": "close"}
+        )
+        self._process.stdout.close()
+        self._process.stdin.close()
+
+    def map(self, function, lst):
+        self._send(
+            function=function,
+            lst=lst
+        )
+        output = self._receive()
+        return output
+
+    def _send(self, function, lst):
+        self._send_raw(
+            input_dict={
+                "f": inspect.getsource(function),
+                "l": lst
+            }
+        )
+
+    def _send_raw(self, input_dict):
+        dill.dump(
+            input_dict,
+            self._process.stdin
+        )
+        self._process.stdin.flush()
+
+    def _receive(self):
+        output = dill.load(self._process.stdout)
+        return output

pympipool/__main__.py

@@ -0,0 +1,47 @@
+import dill
+import textwrap
+from mpi4py import MPI
+MPI.pickle.__init__(
+    dill.dumps,
+    dill.loads,
+    dill.HIGHEST_PROTOCOL,
+)
+from mpi4py.futures import MPIPoolExecutor
+from tqdm import tqdm
+import sys
+
+
+def get_function_from_string(function_str):
+    function_dedent_str = textwrap.dedent(function_str)
+    exec(function_dedent_str)
+    return eval(function_dedent_str.split("(")[0][4:])
+
+
+def exec_funct(executor, funct, lst):
+    results = executor.map(funct, lst)
+    return list(tqdm(results, desc="Configs", total=len(lst)))
+
+
+def main():
+    with MPIPoolExecutor() as executor:
+        while True:
+            output = None
+            if executor is not None:
+                input_dict = dill.load(sys.stdin.buffer)
+                if "c" in input_dict.keys() and input_dict["c"] == "close":
+                    break
+                elif "f" in input_dict.keys() and "l" in input_dict.keys():
+                    output = exec_funct(
+                        executor=executor,
+                        funct=get_function_from_string(
+                            function_str=input_dict["f"]
+                        ),
+                        lst=input_dict["l"]
+                    )
+                if output is not None:
+                    dill.dump(output, sys.stdout.buffer)
+                    sys.stdout.flush()
+
+
+if __name__ == '__main__':
+    main()