From 220802c3940ffdbbabce1af365b87208ad7e8df0 Mon Sep 17 00:00:00 2001 From: Brendt Wohlberg Date: Thu, 5 Sep 2024 15:20:05 -0600 Subject: [PATCH] Various changes (#549) * Change defaults python version * Update docs for conda environment creation script * Recommend python 3.10 instead of 3.9 * Add note on python version requirement for jax * Update jax/gpu instructions * Add note on availgpu.py script * Move gpu util scripts * Formatting fix * Add readme files * Minor phrasing improvement * Improve formatting * Avoid issues with ray and conda version of hyperopt * Python 3.7 no longer supported * Trivial edit * Address PR review comments * Switch for Python 3.12 for CI * Attempt to resolve CI test failures * Fix yaml syntax error * Clean up * Add comment * Address PR comments * Bump actions/cache version * Avoid use of mambaforge in response to deprecation warning --- .github/workflows/lint.yml | 2 +- .github/workflows/mypy.yml | 2 +- .github/workflows/pypi_upload.yml | 2 +- .github/workflows/pytest_latest.yml | 2 +- .github/workflows/pytest_macos.yml | 19 +++++----- .github/workflows/pytest_ubuntu.yml | 31 ++++++++------- .github/workflows/test_examples.yml | 16 ++++---- docs/source/contributing.rst | 6 +-- docs/source/install.rst | 59 ++++++++++++++--------------- misc/README.rst | 6 ++- misc/conda/README.rst | 4 +- misc/conda/make_conda_env.sh | 9 +++-- misc/gpu/README.rst | 7 ++++ misc/{ => gpu}/availgpu.py | 0 misc/{ => gpu}/envinfo.py | 0 misc/pytest/README.rst | 10 +++++ scico/_version.py | 15 ++------ 17 files changed, 99 insertions(+), 91 deletions(-) create mode 100644 misc/gpu/README.rst rename misc/{ => gpu}/availgpu.py (100%) rename misc/{ => gpu}/envinfo.py (100%) create mode 100644 misc/pytest/README.rst diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index a4eeb8cee..2e6a0eeaa 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -15,7 +15,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: "3.10" + python-version: "3.12" - name: Black code formatter uses: psf/black@stable with: diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 5e1d474de..02b680a64 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -20,7 +20,7 @@ jobs: - name: Install Python 3 uses: actions/setup-python@v5 with: - python-version: "3.10" + python-version: "3.12" - name: Install dependencies run: | pip install mypy diff --git a/.github/workflows/pypi_upload.yml b/.github/workflows/pypi_upload.yml index f60c78f47..a6e79f3a2 100644 --- a/.github/workflows/pypi_upload.yml +++ b/.github/workflows/pypi_upload.yml @@ -21,7 +21,7 @@ jobs: - name: Install Python 3 uses: actions/setup-python@v5 with: - python-version: "3.10" + python-version: "3.12" - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/.github/workflows/pytest_latest.yml b/.github/workflows/pytest_latest.yml index 60754eda6..fabbe22b5 100644 --- a/.github/workflows/pytest_latest.yml +++ b/.github/workflows/pytest_latest.yml @@ -21,7 +21,7 @@ jobs: - name: Install Python 3 uses: actions/setup-python@v5 with: - python-version: "3.10" + python-version: "3.12" - name: Install lastversion run: | python -m pip install --upgrade pip diff --git a/.github/workflows/pytest_macos.yml b/.github/workflows/pytest_macos.yml index ec97f4c02..1ab2e88fe 100644 --- a/.github/workflows/pytest_macos.yml +++ b/.github/workflows/pytest_macos.yml @@ -27,18 +27,16 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - # Set up conda/mamba environment - - name: Set up mambaforge + # Set up conda environment + - name: Set up miniconda uses: conda-incubator/setup-miniconda@v3 with: - miniforge-variant: Mambaforge miniforge-version: latest activate-environment: test-env - use-mamba: true - python-version: "3.10" + python-version: "3.12" # Configure conda environment cache - name: Set up conda environment cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ${{ env.CONDA }}/envs key: conda-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('dev_requirements.txt') }}-${{ env.CACHE_NUMBER }} @@ -54,14 +52,15 @@ jobs: - name: Install dependencies if: steps.cache.outputs.cache-hit != 'true' run: | - mamba install -c conda-forge pytest pytest-cov + conda install -c conda-forge pytest pytest-cov python -m pip install --upgrade pip pip install pytest-split pip install -r requirements.txt pip install -r dev_requirements.txt - mamba install -c conda-forge svmbir>=0.3.3 - mamba install -c astra-toolbox astra-toolbox - mamba install -c conda-forge pyyaml + # Temporarily disabled due to svmbir failure with numpy 2.x under Python 3.12 + # conda install -c conda-forge svmbir>=0.3.3 + conda install -c astra-toolbox astra-toolbox + conda install -c conda-forge pyyaml pip install --upgrade --force-reinstall scipy>=1.6.0 # Temporary fix for GLIBCXX_3.4.30 not found in conda forge version pip install bm3d>=4.0.0 pip install bm4d>=4.0.0 diff --git a/.github/workflows/pytest_ubuntu.yml b/.github/workflows/pytest_ubuntu.yml index babfb1d9f..e8c0a1705 100644 --- a/.github/workflows/pytest_ubuntu.yml +++ b/.github/workflows/pytest_ubuntu.yml @@ -27,18 +27,16 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - # Set up conda/mamba environment - - name: Set up mambaforge + # Set up conda environment + - name: Set up miniconda uses: conda-incubator/setup-miniconda@v3 with: - miniforge-variant: Mambaforge miniforge-version: latest activate-environment: test-env - use-mamba: true - python-version: "3.10" + python-version: "3.12" # Configure conda environment cache - name: Set up conda environment cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ${{ env.CONDA }}/envs key: conda-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('dev_requirements.txt') }}-${{ env.CACHE_NUMBER }} @@ -57,14 +55,14 @@ jobs: - name: Install dependencies if: steps.cache.outputs.cache-hit != 'true' run: | - mamba install -c conda-forge pytest pytest-cov + conda install -c conda-forge pytest pytest-cov python -m pip install --upgrade pip pip install pytest-split pip install -r requirements.txt pip install -r dev_requirements.txt - mamba install -c conda-forge svmbir>=0.3.3 - mamba install -c conda-forge astra-toolbox - mamba install -c conda-forge pyyaml + conda install -c conda-forge svmbir>=0.3.3 + conda install -c conda-forge astra-toolbox + conda install -c conda-forge pyyaml pip install --upgrade --force-reinstall scipy>=1.6.0 # Temporary fix for GLIBCXX_3.4.30 not found in conda forge version pip install bm3d>=4.0.0 pip install bm4d>=4.2.2 @@ -81,10 +79,11 @@ jobs: pytest -x --cov --level=2 --durations-path=$DURATIONS_FILE --splits=5 --group=${{ matrix.group }} --pyargs scico # Upload coverage data - name: Upload coverage - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: + include-hidden-files: true name: coverage${{ matrix.group }} - path: .coverage + path: ${{ github.workspace }}/.coverage # Run doc tests - name: Run doc tests if: matrix.group == 1 @@ -97,20 +96,20 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Set up Python 3.10 + - name: Set up Python 3.12 uses: actions/setup-python@v5 with: - python-version: "3.10" + python-version: "3.12" - name: Install deps run: | python -m pip install --upgrade pip pip install coverage - name: Download all artifacts # Downloads coverage1, coverage2, etc. - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 - name: Run coverage run: | - coverage combine coverage*/.coverage* + coverage combine coverage?/.coverage coverage report coverage xml - uses: codecov/codecov-action@v3 diff --git a/.github/workflows/test_examples.yml b/.github/workflows/test_examples.yml index 74645a29a..c2b5e4d23 100644 --- a/.github/workflows/test_examples.yml +++ b/.github/workflows/test_examples.yml @@ -25,18 +25,16 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - # Set up conda/mamba environment - - name: Set up mambaforge + # Set up conda environment + - name: Set up miniconda uses: conda-incubator/setup-miniconda@v3 with: - miniforge-variant: Mambaforge miniforge-version: latest activate-environment: test-env - use-mamba: true - python-version: "3.10" + python-version: "3.12" # Configure conda environment cache - name: Set up conda environment cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ${{ env.CONDA }}/envs key: conda-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('dev_requirements.txt') }}-${{ hashFiles('examples/examples_requirements.txt') }}-${{ env.CACHE_NUMBER }} @@ -55,12 +53,12 @@ jobs: - name: Install dependencies if: steps.cache.outputs.cache-hit != 'true' run: | - mamba install -c conda-forge pytest pytest-cov + conda install -c conda-forge pytest pytest-cov python -m pip install --upgrade pip pip install -r requirements.txt pip install -r dev_requirements.txt - mamba install -c conda-forge astra-toolbox - mamba install -c conda-forge pyyaml + conda install -c conda-forge astra-toolbox + conda install -c conda-forge pyyaml pip install --upgrade --force-reinstall scipy>=1.6.0 # Temporary fix for GLIBCXX_3.4.30 not found in conda forge version pip install -r examples/examples_requirements.txt # Install package to be tested diff --git a/docs/source/contributing.rst b/docs/source/contributing.rst index 89bae8854..6b080e30b 100644 --- a/docs/source/contributing.rst +++ b/docs/source/contributing.rst @@ -17,7 +17,7 @@ Installing a Development Version 1. Fork both the ``scico`` and ``scico-data`` repositories, creating copies of these repositories in your own git account. -2. Make sure that you have Python 3.8 or later installed in order to +2. Make sure that you have Python 3.10 or later installed in order to create a conda virtual environment. 3. Clone your fork from the source repo. @@ -26,11 +26,11 @@ Installing a Development Version git clone --recurse-submodules git@github.com:/scico.git -4. Create a conda environment using Python 3.8 or later, e.g.: +4. Create a conda environment using Python 3.10 or later, e.g.: :: - conda create -n scico python=3.9 + conda create -n scico python=3.12 5. Activate the created conda virtual environment: diff --git a/docs/source/install.rst b/docs/source/install.rst index dcff541ea..0931ef820 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -3,22 +3,28 @@ Installing SCICO ================ -SCICO requires Python version 3.8 or later. (Version 3.10 is -recommended as it is the version under which SCICO has been most -thoroughly tested.) It is supported on both Linux and MacOS, but is -not currently supported on Windows due to the limited support for -``jaxlib`` on Windows. However, Windows users can use SCICO via the -`Windows Subsystem for Linux +SCICO requires Python version 3.8 or later. (Version 3.12 is +recommended as it is the version under which SCICO is tested in GitHub +continuous integration, and since the most recent versions of JAX require +version 3.10 or later.) SCICO is supported on both Linux and +MacOS, but is not currently supported on Windows due to the limited +support for ``jaxlib`` on Windows. However, Windows users can use +SCICO via the `Windows Subsystem for Linux `_ (WSL). Guides -exist for using WSL with `CPU only -`_ and -with `GPU support -`_. - -While not required, installation of SCICO and its dependencies within a `Conda `_ environment -is recommended. `Scripts `_ -are provided for creating a `miniconda `_ installation and an environment including all primary SCICO dependencies as well as dependencies -for usage example, testing, and building the documentation. +exist for using WSL with +`CPU only `_ +and with +`GPU support `_. + +While not required, installation of SCICO and its dependencies within a +`Conda `_ +environment is recommended. +`Scripts `_ +are provided for creating a +`miniconda `_ +installation and an environment including all primary SCICO dependencies +as well as dependencies for usage example, testing, and building the +documentation. From PyPI @@ -108,31 +114,22 @@ a version with GPU support: 2. Install the version of jaxlib with GPU support, as described in the `JAX installation instructions `_. In the simplest case, the appropriate command is - :: - - pip install --upgrade "jax[cuda11]" - - for CUDA 11, or :: pip install --upgrade "jax[cuda12]" for CUDA 12, but it may be necessary to explicitly specify the ``jaxlib`` version if the most recent release is not yet supported - by SCICO (as specified in the ``requirements.txt`` file), or if - using a version of CUDA older than 11.4, or CuDNN older than 8.2, - in which case the command would be of the form - :: - - pip install --upgrade "jaxlib==0.4.2+cuda11.cudnn82" -f https://storage.googleapis.com/jax-releases/jax_cuda_releases.html - - with appropriate substitution of ``jaxlib``, CUDA, and CuDNN version - numbers. + by SCICO (as specified in the ``requirements.txt`` file). -The script `misc/envinfo.py `_ +The script +`misc/gpu/envinfo.py `_ in the source distribution is provided as an aid to debugging GPU support -issues. +issues. The script +`misc/gpu/availgpu.py `_ +can be used to automatically recommend a setting of the CUDA_VISIBLE_DEVICES +environment variable that excludes GPUs that are already in use. diff --git a/misc/README.rst b/misc/README.rst index beda2928c..70b86b530 100644 --- a/misc/README.rst +++ b/misc/README.rst @@ -1,5 +1,9 @@ Miscellaneous ============= -This directory is a temporary location for content for which there is no immediately obvious correct location. +This directory is a temporary location for content for which there is no +obviously more appropriate location: +- ``conda``: Scripts intended to faciliate the installation of miniconda and an environment with all SCICO requirements. +- ``gpu``: Scripts for debugging and managing JAX use of GPUs. +- ``pytest``: Scripts for specialized use of ``pytest``. diff --git a/misc/conda/README.rst b/misc/conda/README.rst index 0388434ff..9df4e6c5e 100644 --- a/misc/conda/README.rst +++ b/misc/conda/README.rst @@ -19,11 +19,11 @@ To install miniconda in ``/opt/conda`` do ./install_conda.sh -y /opt/conda -To create a conda environment called ``py39`` with default Python version (3.9) and without GPU support +To create a conda environment called ``scico`` with Python version 3.12 and without GPU support :: - ./make_conda_env.sh -y -e py39 + ./make_conda_env.sh -y -p 3.12 -e scico To include GPU support, follow the `jax installation instructions `__ after diff --git a/misc/conda/make_conda_env.sh b/misc/conda/make_conda_env.sh index 34b48c615..b5aa4411d 100755 --- a/misc/conda/make_conda_env.sh +++ b/misc/conda/make_conda_env.sh @@ -28,7 +28,7 @@ Usage: $SCRIPT [-h] [-y] [-g] [-p python_version] [-e env_name] [-v] Verbose operation [-t] Display actions that would be taken but do nothing [-y] Do not ask for confirmation - [-p python_version] Specify Python version (e.g. 3.9) + [-p python_version] Specify Python version (e.g. 3.12) [-e env_name] Specify conda environment name EOF ) @@ -36,7 +36,7 @@ EOF AGREE=no VERBOSE=no TEST=no -PYVER="3.9" +PYVER="3.12" ENVNM=py$(echo $PYVER | sed -e 's/\.//g') # Project requirements files @@ -50,7 +50,7 @@ EOF ) # Requirements that cannot be installed via conda (i.e. have to use pip) NOCONDA=$(cat <<-EOF -flax bm3d bm4d py2jn colour_demosaicing ray[tune,train] +flax bm3d bm4d py2jn colour_demosaicing hyperopt ray[tune,train] EOF ) @@ -257,6 +257,9 @@ echo echo "JAX installed without GPU support. To enable GPU support, install a" echo "version of jaxlib with CUDA support following the instructions at" echo " https://jax.readthedocs.io/en/latest/installation.html#nvidia-gpu" +echo "In most cases this just requires the command" +echo " pip install -U \"jax[cuda12]\"" +echo echo "ASTRA Toolbox installed without GPU support if this script was" echo "run on a host without CUDA drivers installed. To enable GPU support," echo "uninstall and then reinstall the astra-toolbox conda package on a" diff --git a/misc/gpu/README.rst b/misc/gpu/README.rst new file mode 100644 index 000000000..a2e229262 --- /dev/null +++ b/misc/gpu/README.rst @@ -0,0 +1,7 @@ +GPU Utility Scripts +=================== + +These scripts are intended for debugging and managing JAX use of GPUs: + +- ``availgpu.py``: Automatically recommend a setting of the ``CUDA_VISIBLE_DEVICES`` environment variable that excludes GPUs that are already in use. +- ``envinfo.py``: An aid to debugging JAX GPU access. diff --git a/misc/availgpu.py b/misc/gpu/availgpu.py similarity index 100% rename from misc/availgpu.py rename to misc/gpu/availgpu.py diff --git a/misc/envinfo.py b/misc/gpu/envinfo.py similarity index 100% rename from misc/envinfo.py rename to misc/gpu/envinfo.py diff --git a/misc/pytest/README.rst b/misc/pytest/README.rst new file mode 100644 index 000000000..e03e6e474 --- /dev/null +++ b/misc/pytest/README.rst @@ -0,0 +1,10 @@ +Specialized Pytest Usage +======================== + +These scripts support specialized ``pytest`` usage: + +- ``pytest_cov.sh``: This script runs ``scico`` unit tests using the ``pytest-cov`` plugin for test coverage analysis. +- ``pytest_fast.sh``: This script runs ``pytest`` tests in parallel using the ``pytest-xdist`` plugin. Some tests (those that do not function correctly when run in parallel) are run separately. +- ``pytest_time.sh``: This script runs each ``scico`` unit test module and lists them all in order of decreasing run time. + +All of these scripts must be run from the repository root directory. diff --git a/scico/_version.py b/scico/_version.py index 86c7b18fc..bf9f9d6f6 100644 --- a/scico/_version.py +++ b/scico/_version.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2020-2022 by SCICO Developers +# Copyright (C) 2020-2024 by SCICO Developers # All rights reserved. BSD 3-clause License. # This file is part of the SCICO package. Details of the copyright and # user license can be found in the 'LICENSE' file distributed with the @@ -7,10 +7,8 @@ """Support functions for determining the package version.""" -import ast import os import re -import sys from ast import parse from subprocess import PIPE, Popen from typing import Any, Optional, Tuple, Union @@ -41,16 +39,9 @@ def variable_assign_value(path: str, var: str) -> Any: """ with open(path) as f: try: - # See http://stackoverflow.com/questions/2058802 + # See https://stackoverflow.com/a/30471662 value_obj = parse(next(filter(lambda line: line.startswith(var), f))).body[0].value # type: ignore - if sys.version_info.major == 3 and sys.version_info.minor == 7: - if isinstance(value_obj, ast.Num): - value = value_obj.n # type: ignore - elif isinstance(value_obj, ast.Str): - value = value_obj.s # type: ignore - else: - value = value_obj.s # type: ignore - + value = value_obj.value # type: ignore except StopIteration: raise RuntimeError(f"Could not find initialization of variable {var}") return value