Various changes (#549)

* Change defaults python version

* Update docs for conda environment creation script

* Recommend python 3.10 instead of 3.9

* Add note on python version requirement for jax

* Update jax/gpu instructions

* Add note on availgpu.py script

* Move gpu util scripts

* Formatting fix

* Add readme files

* Minor phrasing improvement

* Improve formatting

* Avoid issues with ray and conda version of hyperopt

* Python 3.7 no longer supported

* Trivial edit

* Address PR review comments

* Switch for Python 3.12 for CI

* Attempt to resolve CI test failures

* Fix yaml syntax error

* Clean up

* Add comment

* Address PR comments

* Bump actions/cache version

* Avoid use of mambaforge in response to deprecation warning

.github/workflows/lint.yml

@@ -15,7 +15,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: "3.12"
       - name: Black code formatter
         uses: psf/black@stable
         with:

.github/workflows/mypy.yml

@@ -20,7 +20,7 @@ jobs:
       - name: Install Python 3
         uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: "3.12"
       - name: Install dependencies
         run: |
           pip install mypy

.github/workflows/pypi_upload.yml

@@ -21,7 +21,7 @@ jobs:
       - name: Install Python 3
         uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: "3.12"
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

.github/workflows/pytest_latest.yml

@@ -21,7 +21,7 @@ jobs:
       - name: Install Python 3
         uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: "3.12"
       - name: Install lastversion
         run: |
           python -m pip install --upgrade pip

.github/workflows/pytest_macos.yml

@@ -27,18 +27,16 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: recursive
-      # Set up conda/mamba environment
-      - name: Set up mambaforge
+      # Set up conda environment
+      - name: Set up miniconda
         uses: conda-incubator/setup-miniconda@v3
         with:
-            miniforge-variant: Mambaforge
             miniforge-version: latest
             activate-environment: test-env
-            use-mamba: true
-            python-version: "3.10"
+            python-version: "3.12"
       # Configure conda environment cache
       - name: Set up conda environment cache
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ${{ env.CONDA }}/envs
           key: conda-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('dev_requirements.txt') }}-${{ env.CACHE_NUMBER }}
@@ -54,14 +52,15 @@ jobs:
       - name: Install dependencies
         if: steps.cache.outputs.cache-hit != 'true'
         run: |
-          mamba install -c conda-forge pytest pytest-cov
+          conda install -c conda-forge pytest pytest-cov
           python -m pip install --upgrade pip
           pip install pytest-split
           pip install -r requirements.txt
           pip install -r dev_requirements.txt
-          mamba install -c conda-forge svmbir>=0.3.3
-          mamba install -c astra-toolbox astra-toolbox
-          mamba install -c conda-forge pyyaml
+          # Temporarily disabled due to svmbir failure with numpy 2.x under Python 3.12
+          # conda install -c conda-forge svmbir>=0.3.3
+          conda install -c astra-toolbox astra-toolbox
+          conda install -c conda-forge pyyaml
           pip install --upgrade --force-reinstall scipy>=1.6.0  # Temporary fix for GLIBCXX_3.4.30 not found in conda forge version
           pip install bm3d>=4.0.0
           pip install bm4d>=4.0.0

.github/workflows/pytest_ubuntu.yml

@@ -27,18 +27,16 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: recursive
-      # Set up conda/mamba environment
-      - name: Set up mambaforge
+      # Set up conda environment
+      - name: Set up miniconda
         uses: conda-incubator/setup-miniconda@v3
         with:
-            miniforge-variant: Mambaforge
             miniforge-version: latest
             activate-environment: test-env
-            use-mamba: true
-            python-version: "3.10"
+            python-version: "3.12"
       # Configure conda environment cache
       - name: Set up conda environment cache
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ${{ env.CONDA }}/envs
           key: conda-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('dev_requirements.txt') }}-${{ env.CACHE_NUMBER }}
@@ -57,14 +55,14 @@ jobs:
       - name: Install dependencies
         if: steps.cache.outputs.cache-hit != 'true'
         run: |
-          mamba install -c conda-forge pytest pytest-cov
+          conda install -c conda-forge pytest pytest-cov
           python -m pip install --upgrade pip
           pip install pytest-split
           pip install -r requirements.txt
           pip install -r dev_requirements.txt
-          mamba install -c conda-forge svmbir>=0.3.3
-          mamba install -c conda-forge astra-toolbox
-          mamba install -c conda-forge pyyaml
+          conda install -c conda-forge svmbir>=0.3.3
+          conda install -c conda-forge astra-toolbox
+          conda install -c conda-forge pyyaml
           pip install --upgrade --force-reinstall scipy>=1.6.0  # Temporary fix for GLIBCXX_3.4.30 not found in conda forge version
           pip install bm3d>=4.0.0
           pip install bm4d>=4.2.2
@@ -81,10 +79,11 @@ jobs:
           pytest -x --cov --level=2 --durations-path=$DURATIONS_FILE --splits=5 --group=${{ matrix.group }} --pyargs scico
       # Upload coverage data
       - name: Upload coverage
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
+          include-hidden-files: true
           name: coverage${{ matrix.group }}
-          path: .coverage
+          path: ${{ github.workspace }}/.coverage
       # Run doc tests
       - name: Run doc tests
         if: matrix.group == 1
@@ -97,20 +96,20 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python 3.10
+      - name: Set up Python 3.12
         uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: "3.12"
       - name: Install deps
         run: |
           python -m pip install --upgrade pip
           pip install coverage
       - name: Download all artifacts
         # Downloads coverage1, coverage2, etc.
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
       - name: Run coverage
         run: |
-          coverage combine coverage*/.coverage*
+          coverage combine coverage?/.coverage
           coverage report
           coverage xml
       - uses: codecov/codecov-action@v3

.github/workflows/test_examples.yml

@@ -25,18 +25,16 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: recursive
-      # Set up conda/mamba environment
-      - name: Set up mambaforge
+      # Set up conda environment
+      - name: Set up miniconda
         uses: conda-incubator/setup-miniconda@v3
         with:
-            miniforge-variant: Mambaforge
             miniforge-version: latest
             activate-environment: test-env
-            use-mamba: true
-            python-version: "3.10"
+            python-version: "3.12"
       # Configure conda environment cache
       - name: Set up conda environment cache
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ${{ env.CONDA }}/envs
           key: conda-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('dev_requirements.txt') }}-${{ hashFiles('examples/examples_requirements.txt') }}-${{ env.CACHE_NUMBER }}
@@ -55,12 +53,12 @@ jobs:
       - name: Install dependencies
         if: steps.cache.outputs.cache-hit != 'true'
         run: |
-          mamba install -c conda-forge pytest pytest-cov
+          conda install -c conda-forge pytest pytest-cov
           python -m pip install --upgrade pip
           pip install -r requirements.txt
           pip install -r dev_requirements.txt
-          mamba install -c conda-forge astra-toolbox
-          mamba install -c conda-forge pyyaml
+          conda install -c conda-forge astra-toolbox
+          conda install -c conda-forge pyyaml
           pip install --upgrade --force-reinstall scipy>=1.6.0  # Temporary fix for GLIBCXX_3.4.30 not found in conda forge version
           pip install -r examples/examples_requirements.txt
       # Install package to be tested

docs/source/contributing.rst

@@ -17,7 +17,7 @@ Installing a Development Version
 1. Fork both the ``scico`` and ``scico-data`` repositories, creating
    copies of these repositories in your own git account.
 
-2. Make sure that you have Python 3.8 or later installed in order to
+2. Make sure that you have Python 3.10 or later installed in order to
    create a conda virtual environment.
 
 3. Clone your fork from the source repo.
@@ -26,11 +26,11 @@ Installing a Development Version
 
       git clone --recurse-submodules [email protected]:<username>/scico.git
 
-4. Create a conda environment using Python 3.8 or later, e.g.:
+4. Create a conda environment using Python 3.10 or later, e.g.:
 
    ::
 
-      conda create -n scico python=3.9
+      conda create -n scico python=3.12
 
 5. Activate the created conda virtual environment:
 

docs/source/install.rst

@@ -3,22 +3,28 @@
 Installing SCICO
 ================
 
-SCICO requires Python version 3.8 or later. (Version 3.10 is
-recommended as it is the version under which SCICO has been most
-thoroughly tested.) It is supported on both Linux and MacOS, but is
-not currently supported on Windows due to the limited support for
-``jaxlib`` on Windows. However, Windows users can use SCICO via the
-`Windows Subsystem for Linux
+SCICO requires Python version 3.8 or later. (Version 3.12 is
+recommended as it is the version under which SCICO is tested in GitHub
+continuous integration, and since the most recent versions of JAX require
+version 3.10 or later.) SCICO is supported on both Linux and
+MacOS, but is not currently supported on Windows due to the limited
+support for ``jaxlib`` on Windows. However, Windows users can use
+SCICO via the `Windows Subsystem for Linux
 <https://docs.microsoft.com/en-us/windows/wsl/about>`_ (WSL). Guides
-exist for using WSL with `CPU only
-<https://docs.microsoft.com/en-us/windows/wsl/install-win10>`_ and
-with `GPU support
-<https://docs.microsoft.com/en-us/windows/win32/direct3d12/gpu-cuda-in-wsl>`_.
-
-While not required, installation of SCICO and its dependencies within a `Conda <https://conda.io/projects/conda/en/latest/user-guide/index.html>`_ environment
-is recommended. `Scripts <https://github.com/lanl/scico/tree/main/misc/conda>`_
-are provided for creating a `miniconda <https://docs.conda.io/en/latest/miniconda.html>`_ installation and an environment including all primary SCICO dependencies as well as dependencies
-for usage example, testing, and building the documentation.
+exist for using WSL with
+`CPU only <https://docs.microsoft.com/en-us/windows/wsl/install-win10>`_
+and with
+`GPU support <https://docs.microsoft.com/en-us/windows/win32/direct3d12/gpu-cuda-in-wsl>`_.
+
+While not required, installation of SCICO and its dependencies within a
+`Conda <https://conda.io/projects/conda/en/latest/user-guide/index.html>`_
+environment is recommended.
+`Scripts <https://github.com/lanl/scico/tree/main/misc/conda>`_
+are provided for creating a
+`miniconda <https://docs.conda.io/en/latest/miniconda.html>`_
+installation and an environment including all primary SCICO dependencies
+as well as dependencies for usage example, testing, and building the
+documentation.
 
 
 From PyPI
@@ -108,31 +114,22 @@ a version with GPU support:
 2. Install the version of jaxlib with GPU support, as described in the `JAX installation
    instructions  <https://jax.readthedocs.io/en/latest/installation.html>`_.
    In the simplest case, the appropriate command is
-   ::
-
-      pip install --upgrade "jax[cuda11]"
-
-   for CUDA 11, or
    ::
 
       pip install --upgrade "jax[cuda12]"
 
    for CUDA 12, but it may be necessary to explicitly specify the
    ``jaxlib`` version if the most recent release is not yet supported
-   by SCICO (as specified in the ``requirements.txt`` file), or if
-   using a version of CUDA older than 11.4, or CuDNN older than 8.2,
-   in which case the command would be of the form
-   ::
-
-      pip install --upgrade "jaxlib==0.4.2+cuda11.cudnn82" -f https://storage.googleapis.com/jax-releases/jax_cuda_releases.html
-
-   with appropriate substitution of ``jaxlib``, CUDA, and CuDNN version
-   numbers.
+   by SCICO (as specified in the ``requirements.txt`` file).
 
 
-The script `misc/envinfo.py <https://github.com/lanl/scico/blob/main/misc/envinfo.py>`_
+The script
+`misc/gpu/envinfo.py <https://github.com/lanl/scico/blob/main/misc/gpu/envinfo.py>`_
 in the source distribution is provided as an aid to debugging GPU support
-issues.
+issues. The script
+`misc/gpu/availgpu.py <https://github.com/lanl/scico/blob/main/misc/gpu/availgpu.py>`_
+can be used to automatically recommend a setting of the CUDA_VISIBLE_DEVICES
+environment variable that excludes GPUs that are already in use.
 
 
 

misc/README.rst

@@ -1,5 +1,9 @@
 Miscellaneous
 =============
 
-This directory is a temporary location for content for which there is no immediately obvious correct location.
+This directory is a temporary location for content for which there is no
+obviously more appropriate location:
 
+- ``conda``: Scripts intended to faciliate the installation of miniconda and an environment with all SCICO requirements.
+- ``gpu``: Scripts for debugging and managing JAX use of GPUs.
+- ``pytest``: Scripts for specialized use of ``pytest``.

misc/conda/README.rst

@@ -19,11 +19,11 @@ To install miniconda in ``/opt/conda`` do
    ./install_conda.sh -y /opt/conda
 
 
-To create a conda environment called ``py39`` with default Python version (3.9) and without GPU support
+To create a conda environment called ``scico`` with Python version 3.12 and without GPU support
 
 ::
 
-   ./make_conda_env.sh -y -e py39
+   ./make_conda_env.sh -y -p 3.12 -e scico
 
 
 To include GPU support, follow the `jax installation instructions <https://github.com/google/jax#pip-installation-gpu-cuda>`__ after

misc/conda/make_conda_env.sh

@@ -28,15 +28,15 @@ Usage: $SCRIPT [-h] [-y] [-g] [-p python_version] [-e env_name]
           [-v] Verbose operation
           [-t] Display actions that would be taken but do nothing
           [-y] Do not ask for confirmation
-          [-p python_version] Specify Python version (e.g. 3.9)
+          [-p python_version] Specify Python version (e.g. 3.12)
           [-e env_name] Specify conda environment name
 EOF
 )
 
 AGREE=no
 VERBOSE=no
 TEST=no
-PYVER="3.9"
+PYVER="3.12"
 ENVNM=py$(echo $PYVER | sed -e 's/\.//g')
 
 # Project requirements files
@@ -50,7 +50,7 @@ EOF
 )
 # Requirements that cannot be installed via conda (i.e. have to use pip)
 NOCONDA=$(cat <<-EOF
-flax bm3d bm4d py2jn colour_demosaicing ray[tune,train]
+flax bm3d bm4d py2jn colour_demosaicing hyperopt ray[tune,train]
 EOF
 )
 
@@ -257,6 +257,9 @@ echo
 echo "JAX installed without GPU support. To enable GPU support, install a"
 echo "version of jaxlib with CUDA support following the instructions at"
 echo "   https://jax.readthedocs.io/en/latest/installation.html#nvidia-gpu"
+echo "In most cases this just requires the command"
+echo "   pip install -U \"jax[cuda12]\""
+echo
 echo "ASTRA Toolbox installed without GPU support if this script was"
 echo "run on a host without CUDA drivers installed. To enable GPU support,"
 echo "uninstall and then reinstall the astra-toolbox conda package on a"

misc/gpu/README.rst

@@ -0,0 +1,7 @@
+GPU Utility Scripts
+===================
+
+These scripts are intended for debugging and managing JAX use of GPUs:
+
+- ``availgpu.py``: Automatically recommend a setting of the ``CUDA_VISIBLE_DEVICES`` environment variable that excludes GPUs that are already in use.
+- ``envinfo.py``: An aid to debugging JAX GPU access.

misc/pytest/README.rst

@@ -0,0 +1,10 @@
+Specialized Pytest Usage
+========================
+
+These scripts support specialized ``pytest`` usage:
+
+- ``pytest_cov.sh``: This script runs ``scico`` unit tests using the ``pytest-cov`` plugin for test coverage analysis.
+- ``pytest_fast.sh``: This script runs ``pytest`` tests in parallel using the ``pytest-xdist`` plugin. Some tests (those that do not function correctly when run in parallel) are run separately.
+- ``pytest_time.sh``: This script runs each ``scico`` unit test module and lists them all in order of decreasing run time.
+
+All of these scripts must be run from the repository root directory.