Merge branch 'main' into python312

.ci_support/environment-docs.yml

@@ -1,12 +1,13 @@
 channels:
 - conda-forge
 dependencies:
-  - nbsphinx
-  - sphinx
-  - myst-parser
-  - numpy
-  - mpich
-  - cloudpickle
-  - mpi4py
-  - tqdm
-  - pyzmq
+- nbsphinx
+- sphinx
+- myst-parser
+- numpy
+- openmpi
+- cloudpickle =3.0.0
+- mpi4py =3.1.5
+- tqdm =4.66.1
+- pyzmq =25.1.2
+- flux-core

.ci_support/environment-mpich.yml

@@ -4,7 +4,7 @@ dependencies:
 - python
 - numpy
 - mpich
-- cloudpickle =2.2.1
-- mpi4py =3.1.4
+- cloudpickle =3.0.0
+- mpi4py =3.1.5
 - tqdm =4.66.1
-- pyzmq =25.1.1
+- pyzmq =25.1.2

.ci_support/environment-old.yml

@@ -0,0 +1,10 @@
+channels:
+- conda-forge
+dependencies:
+- python
+- numpy
+- openmpi =4.1.4
+- cloudpickle =2.0.0
+- mpi4py =3.1.4
+- tqdm =4.44.0
+- pyzmq =25.0.0

.ci_support/environment-openmpi.yml

@@ -4,7 +4,7 @@ dependencies:
 - python
 - numpy
 - openmpi
-- cloudpickle =2.2.1
-- mpi4py =3.1.4
+- cloudpickle =3.0.0
+- mpi4py =3.1.5
 - tqdm =4.66.1
-- pyzmq =25.1.1
+- pyzmq =25.1.2

.ci_support/environment-win.yml

@@ -4,7 +4,7 @@ dependencies:
 - python
 - numpy
 - msmpi
-- cloudpickle =2.2.1
-- mpi4py =3.1.4
+- cloudpickle =3.0.0
+- mpi4py =3.1.5
 - tqdm =4.66.1
-- pyzmq =25.1.1
+- pyzmq =25.1.2

.ci_support/release.py

@@ -0,0 +1,61 @@
+def get_setup_version_and_pattern(setup_content):
+    depend_lst, version_lst = [], []
+    for l in setup_content:
+        if '==' in l:
+            lst = l.split('[')[-1].split(']')[0].replace(' ', '').replace('"', '').replace("'", '').split(',')
+            for dep in lst:
+                if dep != '\n':
+                    version_lst.append(dep.split('==')[1])
+                    depend_lst.append(dep.split('==')[0])
+
+    version_high_dict = {d: v for d, v in zip(depend_lst, version_lst)}
+    return version_high_dict
+
+
+def get_env_version(env_content):
+    read_flag = False
+    depend_lst, version_lst = [], []
+    for l in env_content:
+        if 'dependencies:' in l:
+            read_flag = True
+        elif read_flag:
+            lst = l.replace('-', '').replace(' ', '').replace('\n', '').split("=")
+            if len(lst) == 2:
+                depend_lst.append(lst[0])
+                version_lst.append(lst[1])
+    return {d:v for d, v in zip(depend_lst, version_lst)}
+
+
+def update_dependencies(setup_content, version_low_dict, version_high_dict):
+    version_combo_dict = {}
+    for dep, ver in version_high_dict.items():
+        if dep in version_low_dict.keys() and version_low_dict[dep] != ver:
+            version_combo_dict[dep] = dep + ">=" + version_low_dict[dep] + ",<=" + ver
+        else:
+            version_combo_dict[dep] = dep + "==" + ver
+
+    setup_content_new = ""
+    pattern_dict = {d:d + "==" + v for d, v in version_high_dict.items()}
+    for l in setup_content:
+        for k, v in pattern_dict.items():
+            if v in l:
+                l = l.replace(v, version_combo_dict[k])
+        setup_content_new +=l
+    return setup_content_new
+
+
+if __name__ == "__main__":
+    with open('pyproject.toml', "r") as f:
+        setup_content = f.readlines()
+
+    with open('environment.yml', "r") as f:
+        env_content = f.readlines()
+
+    setup_content_new = update_dependencies(
+        setup_content=setup_content[2:],
+        version_low_dict=get_env_version(env_content=env_content),
+        version_high_dict=get_setup_version_and_pattern(setup_content=setup_content[2:]),
+    )
+
+    with open('pyproject.toml', "w") as f:
+        f.writelines("".join(setup_content[:2]) + setup_content_new)

.github/workflows/black.yml

@@ -1,5 +1,4 @@
-# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
-# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+# This workflow is used to check the differences and find conflict markers or whitespace errors
 
 name: Black
 
@@ -14,19 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
-    - uses: conda-incubator/[email protected]
+    - uses: psf/black@stable
       with:
-        python-version: '3.11'
-        mamba-version: "*"
-        channels: conda-forge
-        miniforge-variant: Mambaforge
-        channel-priority: strict
-        auto-update-conda: true
-        environment-file: .ci_support/environment-openmpi.yml
-    - name: Setup
-      shell: bash -l {0}
-      run: |
-        conda install -c conda-forge black
-    - name: Test
-      shell: bash -l {0}
-      run: black --check pympipool
+        options: "--check --diff"
+        src: ./${{ github.event.repository.name }}

.github/workflows/check-macos-latest.yml

@@ -0,0 +1,35 @@
+# We are waiting on the macos-latest image to play nicely with MPI
+
+name: Is-macos-latest-working-yet
+
+on:
+  schedule:
+    - cron: '0 23 * * 2'
+  workflow_dispatch:
+
+jobs:
+  openmpi-on-macos-latest:
+    runs-on: macos-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: conda-incubator/[email protected]
+      with:
+        python-version: "3.11"
+        mamba-version: "*"
+        channels: conda-forge
+        miniforge-variant: Mambaforge
+        channel-priority: strict
+        auto-update-conda: true
+        environment-file: .ci_support/environment-openmpi.yml
+    - name: Test
+      shell: bash -l {0}
+      timeout-minutes: 10
+      run: |
+        pip install versioneer[toml]==0.29
+        pip install . --no-deps --no-build-isolation
+        cd tests
+        python -m unittest discover .
+      env:
+        OMPI_MCA_plm: 'isolated'
+        OMPI_MCA_rmaps_base_oversubscribe: 'yes'
+        OMPI_MCA_btl_vader_single_copy_mechanism: 'none'

.github/workflows/dependabot.yml

@@ -0,0 +1,38 @@
+name: Update Dependabot
+
+on:
+  pull_request_target:
+    branches: [ main ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    if: (github.actor == 'dependabot[bot]')
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ github.event.pull_request.head.ref }} # Check out the head of the actual branch, not the PR
+          fetch-depth: 0 # otherwise, you will fail to push refs to dest repo
+          token: ${{ secrets.DEPENDABOT_WORKFLOW_TOKEN }}
+      - name: UpdateEnvironmentFile
+        env:
+          PR_TITLE: ${{ github.event.pull_request.title }}
+        shell: bash -l {0}
+        run: |
+          package=$(echo "$PR_TITLE" | awk '{print $2}')
+          from=$(echo "$PR_TITLE" | awk '{print $4}')
+          to=$(echo "$PR_TITLE" | awk '{print $6}')
+          sed -i "/${package}/s/${from}/${to}/g" .ci_support/environment-mpich.yml
+          sed -i "/${package}/s/${from}/${to}/g" .ci_support/environment-openmpi.yml
+          sed -i "/${package}/s/${from}/${to}/g" .ci_support/environment-win.yml
+          sed -i "/${package}/s/${from}/${to}/g" .ci_support/environment-docs.yml
+      - name: UpdateDependabotPR commit
+        run: |
+          git config --local user.email "[email protected]"
+          git config --local user.name "pyironrunner"
+          git commit -m "[dependabot skip] Update environment" -a
+      - name: UpdateDependabotPR push
+        uses: ad-m/github-push-action@master
+        with:
+          github_token: ${{ secrets.DEPENDABOT_WORKFLOW_TOKEN }}
+          branch: ${{ github.event.pull_request.head.ref }}

.github/workflows/deploy.yml

@@ -1,32 +1,41 @@
+# This workflow is used to upload and deploy a new release to PyPi
+# Based on https://github.com/pypa/gh-action-pypi-publish
+
 name: PyPi Release
 
 on:
     push:
     pull_request:
+    workflow_dispatch:
 
-# based on https://github.com/pypa/gh-action-pypi-publish
 jobs:
   build:
+    if: startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release'
     runs-on: ubuntu-latest
-
+    environment:
+      name: pypi
+      url: https://pypi.org/p/${{ github.event.repository.name }}
+    permissions:
+      id-token: write
     steps:
     - uses: actions/checkout@v2
-    - uses: actions/setup-python@v2
+    - uses: conda-incubator/setup-miniconda@v2.2.0
       with:
-        python-version: '3.11'
-
-    - name: Install dependencies
-      run: >-
-        python -m pip install --user --upgrade setuptools wheel
+        python-version: 3.11
+        mamba-version: "*"
+        channels: conda-forge
+        miniforge-variant: Mambaforge
+        channel-priority: strict
+        auto-update-conda: true
+        environment-file: .ci_support/environment-openmpi.yml
     - name: Convert dependencies
-      run: >-
-        sed -i 's/==/>=/g' setup.py; cat setup.py
+      run: |
+        cp .ci_support/environment-old.yml environment.yml
+        python .ci_support/release.py; cat pyproject.toml
     - name: Build
-      run: >-
+      shell: bash -l {0}
+      run: |
+        pip install versioneer[toml]==0.29
         python setup.py sdist bdist_wheel
     - name: Publish distribution 📦 to PyPI
-      if: startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release'
-      uses: pypa/gh-action-pypi-publish@master
-      with:
-        user: __token__
-        password: ${{ secrets.pypi_password }}
+      uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/format_black.yml

@@ -0,0 +1,33 @@
+# This workflow will checkout the branch of the PR, apply black formatting and commit the result to the PR. Does not work for forks.
+
+name: Format black
+
+on:
+  pull_request:
+    types: [labeled]
+
+jobs:
+  build:
+    if: contains(github.event.pull_request.labels.*.name, 'format_black' )
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        token: ${{ secrets.DEPENDABOT_WORKFLOW_TOKEN }}
+        ref: ${{ github.event.pull_request.head.ref }} # Check out the head of the actual branch, not the PR
+        fetch-depth: 0 # otherwise, you will fail to push refs to dest repo
+    - name: format black
+      uses: psf/black@stable
+      with:
+        options: ""
+        src: "./${{ github.event.repository.name }}"
+    - name: commit
+      run: |
+        git config --local user.email "[email protected]"
+        git config --local user.name "pyiron-runner"
+        git commit -m "Format black" -a
+    - name: push
+      uses: ad-m/github-push-action@master
+      with:
+        github_token: ${{ secrets.DEPENDABOT_WORKFLOW_TOKEN }}
+        branch: ${{ github.event.pull_request.head.ref }}

.github/workflows/pypicheck.yml

@@ -23,5 +23,6 @@ jobs:
     - name: Setup
       shell: bash -l {0}
       run: |
-        pip install --no-deps .
+        pip install versioneer[toml]==0.29
+        pip install . --no-deps --no-build-isolation
         pip check

.github/workflows/unittest-flux.yml

@@ -38,22 +38,15 @@ jobs:
         channel-priority: strict
         auto-update-conda: true
         environment-file: ${{ matrix.environment-file }}
-    - name: Install flux
-      shell: bash -l {0}
-      run: mamba install -y flux-core coverage
-    - name: Setup
-      shell: bash -l {0}
-      run: pip install --no-deps .
     - name: Test
       shell: bash -l {0}
       timeout-minutes: 5
-      run: >
-        for f in $(ls tests/test_*.py); do
-          echo $f;
-          if [ $f != "tests/test_flux.py" ]; then
-            coverage run --omit pympipool/_version.py -m unittest $f;
-          fi
-        done
+      run: |
+        mamba install -y flux-core=0.58.0 coverage
+        pip install versioneer[toml]==0.29
+        pip install . --no-deps --no-build-isolation
+        cd tests
+        coverage run --omit pympipool/_version.py -m unittest discover .
       env:
         OMPI_MCA_plm: 'isolated'
         OMPI_MCA_rmaps_base_oversubscribe: 'yes'

.github/workflows/unittest-mpich.yml

@@ -15,7 +15,7 @@ jobs:
     strategy:
       matrix:
         include:
-        - operating-system: macos-latest
+        - operating-system: macos-11
           python-version: '3.12'
           label: osx-64-py-3-12-mpich
           prefix: /Users/runner/miniconda3/envs/my-env
@@ -56,10 +56,11 @@ jobs:
         channel-priority: strict
         auto-update-conda: true
         environment-file: .ci_support/environment-mpich.yml
-    - name: Setup
-      shell: bash -l {0}
-      run: pip install --no-deps .
     - name: Test
       shell: bash -l {0}
       timeout-minutes: 5
-      run: for f in $(ls tests/test_*.py); do echo $f; python -m unittest $f; done
+      run: |
+        pip install versioneer[toml]==0.29
+        pip install . --no-deps --no-build-isolation
+        cd tests
+        python -m unittest discover .