Merge remote-tracking branch 'origin/main' into no_more_wait

.ci_support/environment-docs.yml

@@ -1,13 +1,13 @@
 channels:
 - conda-forge
 dependencies:
-  - nbsphinx
-  - sphinx
-  - myst-parser
-  - numpy
-  - mpich
-  - cloudpickle
-  - mpi4py
-  - tqdm
-  - pyzmq
-  - flux-core
+- nbsphinx
+- sphinx
+- myst-parser
+- numpy
+- openmpi
+- cloudpickle =3.0.0
+- mpi4py =3.1.5
+- tqdm =4.66.1
+- pyzmq =25.1.2
+- flux-core

.ci_support/environment-mpich.yml

@@ -7,4 +7,4 @@ dependencies:
 - cloudpickle =3.0.0
 - mpi4py =3.1.5
 - tqdm =4.66.1
-- pyzmq =25.1.1
+- pyzmq =25.1.2

.ci_support/environment-openmpi.yml

@@ -7,4 +7,4 @@ dependencies:
 - cloudpickle =3.0.0
 - mpi4py =3.1.5
 - tqdm =4.66.1
-- pyzmq =25.1.1
+- pyzmq =25.1.2

.ci_support/environment-win.yml

@@ -7,4 +7,4 @@ dependencies:
 - cloudpickle =3.0.0
 - mpi4py =3.1.5
 - tqdm =4.66.1
-- pyzmq =25.1.1
+- pyzmq =25.1.2

.github/workflows/check-macos-latest.yml

@@ -0,0 +1,35 @@
+# We are waiting on the macos-latest image to play nicely with MPI
+
+name: Is-macos-latest-working-yet
+
+on:
+  schedule:
+    - cron: '0 23 * * 2'
+  workflow_dispatch:
+
+jobs:
+  openmpi-on-macos-latest:
+    runs-on: macos-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: conda-incubator/[email protected]
+      with:
+        python-version: "3.11"
+        mamba-version: "*"
+        channels: conda-forge
+        miniforge-variant: Mambaforge
+        channel-priority: strict
+        auto-update-conda: true
+        environment-file: .ci_support/environment-openmpi.yml
+    - name: Test
+      shell: bash -l {0}
+      timeout-minutes: 10
+      run: |
+        pip install versioneer[toml]==0.29
+        pip install . --no-deps --no-build-isolation
+        cd tests
+        python -m unittest discover .
+      env:
+        OMPI_MCA_plm: 'isolated'
+        OMPI_MCA_rmaps_base_oversubscribe: 'yes'
+        OMPI_MCA_btl_vader_single_copy_mechanism: 'none'

.github/workflows/dependabot.yml

@@ -25,6 +25,7 @@ jobs:
           sed -i "/${package}/s/${from}/${to}/g" .ci_support/environment-mpich.yml
           sed -i "/${package}/s/${from}/${to}/g" .ci_support/environment-openmpi.yml
           sed -i "/${package}/s/${from}/${to}/g" .ci_support/environment-win.yml
+          sed -i "/${package}/s/${from}/${to}/g" .ci_support/environment-docs.yml
       - name: UpdateDependabotPR commit
         run: |
           git config --local user.email "[email protected]"

.github/workflows/unittest-flux.yml

@@ -42,7 +42,7 @@ jobs:
       shell: bash -l {0}
       timeout-minutes: 5
       run: |
-        mamba install -y flux-core coverage
+        mamba install -y flux-core=0.58.0 coverage
         pip install versioneer[toml]==0.29
         pip install . --no-deps --no-build-isolation
         cd tests

.github/workflows/unittest-mpich.yml

@@ -15,7 +15,7 @@ jobs:
     strategy:
       matrix:
         include:
-        - operating-system: macos-latest
+        - operating-system: macos-11
           python-version: '3.11'
           label: osx-64-py-3-11-mpich
           prefix: /Users/runner/miniconda3/envs/my-env

.github/workflows/unittest-openmpi.yml

@@ -15,7 +15,7 @@ jobs:
     strategy:
       matrix:
         include:
-        - operating-system: macos-latest
+        - operating-system: macos-11
           python-version: '3.11'
           label: osx-64-py-3-11-openmpi
           prefix: /Users/runner/miniconda3/envs/my-env

.readthedocs.yml

@@ -6,9 +6,13 @@
 version: 2
 
 build:
-  os: "ubuntu-20.04"
+  os: "ubuntu-22.04"
   tools:
-    python: "mambaforge-4.10"
+    python: "mambaforge-22.9"
+  jobs:
+    pre_build:
+      - pip install versioneer[toml]==0.29
+      - pip install . --no-deps --no-build-isolation
 
 # Build documentation in the docs/ directory with Sphinx
 sphinx:
@@ -19,10 +23,4 @@ formats: []
 
 # Install pyiron from conda
 conda:
-  environment: .ci_support/environment-docs.yml
-
-# Optionally set the version of Python and requirements required to build your docs
-python:
-  install:
-    - method: pip
-      path: .
+  environment: .ci_support/environment-docs.yml

docs/source/examples.md

@@ -14,7 +14,7 @@ with ThreadPoolExecutor(
     future = exe.submit(sum, [1, 1])
     print(future.result())
 ```
-In this case `max_workers=1` limits the number of threads uses by the `ThreadPoolExecutor` to one. Then the `sum()` 
+In this case `max_workers=1` limits the number of threads used by the `ThreadPoolExecutor` to one. Then the `sum()`
 function is submitted to the executor with a list with two ones `[1, 1]` as input. A [`concurrent.futures.Future`](https://docs.python.org/3/library/concurrent.futures.html#module-concurrent.futures)
 object is returned. The `Future` object allows to check the status of the execution with the `done()` method which 
 returns `True` or `False` depending on the state of the execution. Or the main process can wait until the execution is 

docs/source/index.rst

@@ -8,29 +8,27 @@ pympipool - up-scale python functions for high performance computing
 Up-scaling python functions for high performance computing (HPC) can be challenging. While the python standard library
 provides interfaces for multiprocessing and asynchronous task execution, namely
 `multiprocessing <https://docs.python.org/3/library/multiprocessing.html>`_ and
-`concurrent.futures <https://docs.python.org/3/library/concurrent.futures.html#module-concurrent.futures>`_ both are
+`concurrent.futures <https://docs.python.org/3/library/concurrent.futures.html#module-concurrent.futures>`_, both are
 limited to the execution on a single compute node. So a series of python libraries have been developed to address the
 up-scaling of python functions for HPC. Starting in the datascience and machine learning community with solutions
-like `dask <https://www.dask.org>`_ over more HPC focused solutions like
-`fireworks <https://materialsproject.github.io/fireworks/>`_ and `parsl <http://parsl-project.org>`_ up to Python
+like `dask <https://www.dask.org>`_, over to more HPC-focused solutions like
+`fireworks <https://materialsproject.github.io/fireworks/>`_ and `parsl <http://parsl-project.org>`_, up to Python
 bindings for the message passing interface (MPI) named `mpi4py <https://mpi4py.readthedocs.io>`_. Each of these
-solutions has their advantages and disadvantages, in particular scaling beyond serial python functions, including thread
-based parallelism, MPI parallel python application or assignment of GPUs to individual python function remains
-challenging.
+solutions has its advantages and disadvantages. However, one disadvantage common to all these libraries is the relative difficulty of scaling from serial functions to functions that make use of thread-based, MPI-based, or GPU-based parallelism.
 
 To address these challenges :code:`pympipool` is developed with three goals in mind:
 
 * Extend the standard python library `concurrent.futures.Executor <https://docs.python.org/3/library/concurrent.futures.html#module-concurrent.futures>`_ interface, to minimize the barrier of up-scaling an existing workflow to be used on HPC resources.
-* Integrate thread based parallelism, MPI parallel python functions based on `mpi4py <https://mpi4py.readthedocs.io>`_ and GPU assignment. This allows the users to accelerate their workflows one function at a time.
+* Integrate thread-based parallelism, MPI-parallel python functions based on `mpi4py <https://mpi4py.readthedocs.io>`_, and GPU assignment. This allows users to accelerate their workflows one function at a time.
 * Embrace `Jupyter <https://jupyter.org>`_ notebooks for the interactive development of HPC workflows, as they allow the users to document their though process right next to the python code and their results all within one document.
 
 HPC Context
 -----------
-In contrast to frameworks like `dask <https://www.dask.org>`_, `fireworks <https://materialsproject.github.io/fireworks/>`_
-and `parsl <http://parsl-project.org>`_ which can be used to submit a number of worker processes directly the the HPC
+Frameworks like `dask <https://www.dask.org>`_, `fireworks <https://materialsproject.github.io/fireworks/>`_
+and `parsl <http://parsl-project.org>`_ can be used to submit a number of worker processes directly to the HPC
 queuing system and then transfer tasks from either the login node or an interactive allocation to these worker processes
-to accelerate the execution, `mpi4py <https://mpi4py.readthedocs.io>`_ and :code:`pympipool` follow a different
-approach. Here the user creates their HPC allocation first and then `mpi4py <https://mpi4py.readthedocs.io>`_ or
+to accelerate the execution. By contrast, `mpi4py <https://mpi4py.readthedocs.io>`_ and :code:`pympipool` follow a different
+approach, in which the user creates their HPC allocation first and then `mpi4py <https://mpi4py.readthedocs.io>`_ or
 :code:`pympipool` can be used to distribute the tasks within this allocation. The advantage of this approach is that
 no central data storage is required as the workers and the scheduling task can communicate directly.
 
@@ -69,6 +67,7 @@ The same code can also be executed inside a jupyter notebook directly which enab
 The standard `concurrent.futures.Executor <https://docs.python.org/3/library/concurrent.futures.html#module-concurrent.futures>`_
 interface is extended by adding the option :code:`cores_per_worker=2` to assign multiple MPI ranks to each function call.
 To create two workers :code:`max_workers=2` each with two cores each requires a total of four CPU cores to be available.
+
 After submitting the function :code:`calc()` with the corresponding parameter to the executor :code:`exe.submit(calc, 0)`
 a python `concurrent.futures.Future <https://docs.python.org/3/library/concurrent.futures.html#future-objects>`_ is
 returned. Consequently, the :code:`pympipool.Executor` can be used as a drop-in replacement for the

docs/source/installation.md

@@ -38,10 +38,10 @@ pip install pympipool
 ## High Performance Computing
 `pympipool` currently provides interfaces to the [SLURM workload manager](https://www.schedmd.com) and the 
 [flux framework](https://flux-framework.org). With the [flux framework](https://flux-framework.org) being the 
-recommended solution as it can be installed without root user rights and it can be integrated in existing resource
+recommended solution as it can be installed without root permissions and it can be integrated in existing resource
 managers like the [SLURM workload manager](https://www.schedmd.com). The advantages of using `pympipool` in combination
-with these resource schedulers is the fine-grained resource allocation. In addition, to scaling beyond a single compute
-node they add the ability to assign GPUs and thread based parallelism. The two resource manager are internally linked to
+with these resource schedulers is the fine-grained resource allocation. In addition to scaling beyond a single compute
+node, they add the ability to assign GPUs and thread based parallelism. The two resource manager are internally linked to
 two interfaces: 
 
 * `pympipool.slurm.PySlurmExecutor`: The interface for the [SLURM workload manager](https://www.schedmd.com).

pyproject.toml

@@ -26,7 +26,7 @@ classifiers = [
 dependencies = [
     "cloudpickle==3.0.0",
     "mpi4py==3.1.5",
-    "pyzmq==25.1.1",
+    "pyzmq==25.1.2",
     "tqdm==4.66.1",
 ]
 dynamic = ["version"]