Merge pull request #16082 from nsoranzo/osx_startup_py3.11

Fix and test startup with Python 3.11 on macOS

.github/workflows/osx_startup.yaml

@@ -15,6 +15,10 @@ jobs:
   test:
     name: Startup test
     runs-on: macos-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.7', '3.11']
     defaults:
       run:
         shell: bash -l {0}
@@ -36,19 +40,25 @@ jobs:
         id: pip-cache
         with:
           path: ~/Library/Caches/pip
-          # scripts/common_startup.sh creates a conda env for Galaxy containing Python 3.7
-          key: pip-cache-3.7-${{ hashFiles('galaxy root/requirements.txt') }}
+          key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('galaxy root/requirements.txt') }}
       - name: Cache tox env
         uses: actions/cache@v3
         with:
           path: .tox
           key: tox-cache-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('galaxy root/requirements.txt') }}-osx
-      - name: Install and activate miniconda  # use this job to test using Python from a conda environment
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '18.12.1'
+          cache: 'yarn'
+          cache-dependency-path: 'galaxy root/client/yarn.lock'
+      - name: Install miniconda  # use this job to test using Python from a conda environment
         uses: conda-incubator/setup-miniconda@v2
         with:
           activate-environment: ''
       - name: Install tox
         run: pip install tox
-      - name: run tests
+      - name: Run tests
         run: tox -e first_startup
         working-directory: 'galaxy root'
+        env:
+          GALAXY_CONDA_PYTHON_VERSION: "${{ matrix.python-version }}"

lib/galaxy/dependencies/dev-requirements.txt

@@ -54,7 +54,8 @@ lxml==4.9.2 ; python_version >= "3.7" and python_version < "3.12"
 markdown-it-py==2.2.0 ; python_version >= "3.7" and python_version < "3.12"
 markdown-it-reporter==0.0.2 ; python_version >= "3.7" and python_version < "3.12"
 markupsafe==2.1.2 ; python_version >= "3.7" and python_version < "3.12"
-matplotlib==3.5.3 ; python_version >= "3.7" and python_version < "3.12"
+matplotlib==3.5.3 ; python_version >= "3.7" and python_version < "3.8"
+matplotlib==3.7.1 ; python_version >= "3.8" and python_version < "3.12"
 mdit-py-plugins==0.3.5 ; python_version >= "3.7" and python_version < "3.12"
 mdurl==0.1.2 ; python_version >= "3.7" and python_version < "3.12"
 mirakuru==2.5.1 ; python_version >= "3.7" and python_version < "3.12"

lib/galaxy/dependencies/pinned-requirements.txt

@@ -53,6 +53,7 @@ cryptography==40.0.2 ; python_version >= "3.7" and python_version < "3.12"
 cwl-upgrader==1.2.7 ; python_version >= "3.7" and python_version < "3.12"
 cwl-utils==0.26 ; python_version >= "3.7" and python_version < "3.12"
 cwltool==3.1.20221109155812 ; python_version >= "3.7" and python_version < "3.12"
+cython==0.29.34 ; python_version >= "3.7" and python_version < "3.12"
 decorator==5.1.1 ; python_version >= "3.7" and python_version < "3.12"
 defusedxml==0.7.1 ; python_version >= "3.7" and python_version < "3.12"
 deprecation==2.1.0 ; python_version >= "3.7" and python_version < "3.12"
@@ -139,7 +140,7 @@ pyparsing==3.0.9 ; python_version >= "3.7" and python_version < "3.12"
 pyreadline3==3.4.1 ; sys_platform == "win32" and python_version >= "3.8" and python_version < "3.12"
 pyreadline==2.1 ; sys_platform == "win32" and python_version < "3.8" and python_version >= "3.7"
 pyrsistent==0.19.3 ; python_version >= "3.7" and python_version < "3.12"
-pysam==0.20.0 ; python_version >= "3.7" and python_version < "3.12"
+pysam==0.21.0 ; python_version >= "3.7" and python_version < "3.12"
 python-dateutil==2.8.2 ; python_version >= "3.7" and python_version < "3.12"
 python-jose==3.3.0 ; python_version >= "3.7" and python_version < "3.12"
 python-magic==0.4.27 ; python_version >= "3.7" and python_version < "3.12"

lib/galaxy/dependencies/update.sh

@@ -88,5 +88,6 @@ split_requirement () {
 # supported by Galaxy, so Poetry resorts to an old version that didn't have a
 # maximum Python version pin. Here we replace any such requirement with multiple
 # Python-version-specific requirements.
+split_requirement matplotlib
 split_requirement numpy
 split_requirement scipy

pyproject.toml

@@ -85,7 +85,7 @@ PyJWT = "*"
 pykwalify = "*"
 pylibmagic = "*"
 pyparsing = "*"
-pysam = ">=0.20, !=0.21.0"  # https://github.com/pysam-developers/pysam/pull/1186
+pysam = ">=0.21"  # for Python 3.11 support on macOS
 python = ">=3.7,<3.12"  # Keep maximum version strict to appease numpy and scipy
 python-dateutil = "*"
 python-magic = "*"

scripts/common_startup.sh

@@ -20,7 +20,8 @@ SKIP_CLIENT_BUILD=${GALAXY_SKIP_CLIENT_BUILD:-0}
 SKIP_NODE=${GALAXY_SKIP_NODE:-0}
 INSTALL_PREBUILT_CLIENT=${GALAXY_INSTALL_PREBUILT_CLIENT:-0}
 NODE_VERSION=${GALAXY_NODE_VERSION:-"$(cat client/.node_version)"}
-YARN_INSTALL_OPTS=${YARN_INSTALL_OPTS:-"--network-timeout 300000 --check-files"}
+: "${YARN_INSTALL_OPTS:=--network-timeout 300000 --check-files}"
+: "${GALAXY_CONDA_PYTHON_VERSION:=3.7}"
 
 for arg in "$@"; do
     [ "$arg" = "--skip-eggs" ] && FETCH_WHEELS=0
@@ -101,7 +102,7 @@ else
     GIT_BRANCH=0
 fi
 
-: ${GALAXY_VIRTUAL_ENV:=.venv}
+: "${GALAXY_VIRTUAL_ENV:=.venv}"
 # GALAXY_CONDA_ENV is not set here because we don't want to execute the Galaxy version check if we don't need to
 
 if [ $SET_VENV -eq 1 ] && [ $CREATE_VENV -eq 1 ]; then
@@ -112,13 +113,13 @@ if [ $SET_VENV -eq 1 ] && [ $CREATE_VENV -eq 1 ]; then
         if [ -n "$CONDA_EXE" ]; then
             echo "Found Conda, will set up a virtualenv using conda."
             echo "To use a virtualenv instead, create one with a non-Conda Python at $GALAXY_VIRTUAL_ENV"
-            : ${GALAXY_CONDA_ENV:="_galaxy_"}
+            : "${GALAXY_CONDA_ENV:=_galaxy_}"
             if [ "$CONDA_DEFAULT_ENV" != "$GALAXY_CONDA_ENV" ]; then
                 if ! check_conda_env "$GALAXY_CONDA_ENV"; then
                     echo "Creating Conda environment for Galaxy: $GALAXY_CONDA_ENV"
                     echo "To avoid this, use the --no-create-venv flag or set \$GALAXY_CONDA_ENV to an"
                     echo "existing environment before starting Galaxy."
-                    $CONDA_EXE create --yes --override-channels --channel conda-forge --name "$GALAXY_CONDA_ENV" "python=${MIN_PYTHON_VERSION}" "pip>=${MIN_PIP_VERSION}"
+                    $CONDA_EXE create --yes --override-channels --channel conda-forge --name "$GALAXY_CONDA_ENV" "python=${GALAXY_CONDA_PYTHON_VERSION}" "pip>=${MIN_PIP_VERSION}"
                     unset __CONDA_INFO
                 fi
                 conda_activate
@@ -187,6 +188,7 @@ fi
 
 if [ $FETCH_WHEELS -eq 1 ]; then
     python -m pip install "pip>=$MIN_PIP_VERSION" wheel
+    # shellcheck disable=SC2086
     pip install $requirement_args --index-url "${GALAXY_WHEELS_INDEX_URL}" --extra-index-url "${PYPI_INDEX_URL}"
     GALAXY_CONDITIONAL_DEPENDENCIES=$(PYTHONPATH=lib python -c "from __future__ import print_function; import galaxy.dependencies; print('\n'.join(galaxy.dependencies.optional('$GALAXY_CONFIG_FILE')))")
     if [ -n "$GALAXY_CONDITIONAL_DEPENDENCIES" ]; then
@@ -265,6 +267,7 @@ if [ $SKIP_CLIENT_BUILD -eq 0 ]; then
     if [ $INSTALL_PREBUILT_CLIENT -eq 0 ]; then
         # If we have not opted to use a prebuilt client, then build client.
         cd client
+        # shellcheck disable=SC2086
         if yarn install $YARN_INSTALL_OPTS; then
             if ! (export GALAXY_PLUGIN_PATH="$GALAXY_PLUGIN_PATH"; yarn run build-production-maps;) then
                 echo "ERROR: Galaxy client build failed. See ./client/README.md for more information, including how to get help."
@@ -277,6 +280,7 @@ if [ $SKIP_CLIENT_BUILD -eq 0 ]; then
         cd -
     else
         # Install prebuilt client
+        # shellcheck disable=SC2086
         if yarn install $YARN_INSTALL_OPTS; then
             if ! (yarn run stage) then
                 echo "ERROR: Galaxy prebuilt client install failed. See ./client/README.md for more information, including how to get help."

tox.ini

@@ -25,14 +25,13 @@ allowlist_externals =
 passenv =
     CI
     CONDA_EXE
+    GALAXY_CONDA_PYTHON_VERSION
     GALAXY_CONFIG_OVERRIDE_DATABASE_CONNECTION
     APP_WEBSERVER
     TERM
 setenv =
     coverage: GALAXY_TEST_COVERAGE=1
-    first_startup: GALAXY_PYTHON=python
     first_startup: GALAXY_CONFIG_DATABASE_AUTO_MIGRATE=true
-    py{36,37,38}-first_startup: GALAXY_VIRTUAL_ENV=.venv3
     mulled: marker=external_dependency_management
     mulled,unit: GALAXY_VIRTUAL_ENV={envdir}
     unit: GALAXY_ENABLE_BETA_COMPRESSED_GENBANK_SNIFFING=1