Merge branch 't/29500/install_all_python_packages_via_pip_wheel__crea…

…te_pep_503_simple_repository_for_wheels' into t/30657/fix_up__sage__p_

build/bin/sage-dist-helpers

@@ -1,4 +1,4 @@
-# Shell functions for making spkg-install scripts a little easier to write,
+# -*- shell-script -*- functions for making spkg-install scripts a little easier to write,
 # eliminating duplication.  All Sage helper functions begin with sdh_ (for
 # Sage-distribution helper).  Consult the below documentation for the list of
 # available helper functions.
@@ -205,15 +205,51 @@ sdh_pip_install() {
     echo "Installing $PKG_NAME"
     if [ -n "$SAGE_DESTDIR" ]; then
         local sudo=""
-        local root="--root=$SAGE_DESTDIR"
     else
         local sudo="$SAGE_SUDO"
-        local root=""
     fi
-    $sudo sage-pip-install $root "$@" || \
-        sdh_die "Error installing $PKG_NAME"
+    $sudo sage-pip-uninstall "$@" || \
+        sdh_die "Error uninstalling a previous version of $PKG_NAME"
+
+    mkdir -p dist
+    rm -f dist/*.whl
+    sage-python23 -m pip wheel --wheel-dir=dist --no-binary :all: --verbose --no-deps --no-index --isolated --no-build-isolation "$@" || \
+        sdh_die "Error building a wheel for $PKG_NAME"
+
+    sdh_store_and_pip_install_wheel .
 }
 
+sdh_store_and_pip_install_wheel() {
+    if [ -n "$SAGE_DESTDIR" ]; then
+        local sudo=""
+        # --no-warn-script-location: Suppress a warning caused by --root
+        local root="--root=$SAGE_DESTDIR --no-warn-script-location"
+    else
+        local sudo="$SAGE_SUDO"
+        local root=""
+    fi
+    if [ "$*" != "." ]; then
+        sdh_die "Error: sdh_store_and_pip_install_wheel requires . as only argument"
+    fi
+    wheel=""
+    for w in dist/*.whl; do
+        if [ -n "$wheel" ]; then
+            sdh_die "Error: more than one wheel found after building"
+        fi
+        if [ -f "$w" ]; then
+            wheel="$w"
+        fi
+    done
+    if [ -z "$wheel" ]; then
+        sdh_die "Error: no wheel found after building"
+    fi
+
+    $sudo sage-pip-install $root "$wheel" || \
+        sdh_die "Error installing $wheel"
+    mkdir -p "${SAGE_DESTDIR}${SAGE_SPKG_WHEELS}" && \
+        $sudo mv "$wheel" "${SAGE_DESTDIR}${SAGE_SPKG_WHEELS}/" || \
+        sdh_die "Error storing $wheel"
+}
 
 sdh_cmake() {
     echo "Configuring $PKG_NAME with cmake"

build/bin/sage-pip-install

@@ -1,11 +1,6 @@
 #!/usr/bin/env bash
-# This command is specifically for pip-installing from a local
-# source directory, as opposed to from a package index via package
-# name.  That is, it is for pip-installing Sage spkgs from their
-# extracted upstream sources.
-#
-# This ensures that any previous installations of the same package
-# are uninstalled first.
+# This command is specifically for pip-installing a previously
+# built wheel.
 
 # Default arguments for all packages installed with `pip install`
 # --ignore-installed : Force pip to re-install package even if it thinks it's
@@ -17,81 +12,25 @@
 #                      This also disables pip's version self-check.
 # --isolated         : Don't read configuration files such as
 #                      ~/.pydistutils.cfg
-# --no-build-isolation:Build the package in the usual Python environment
-#                      (containing the dependencies) instead of an
-#                      "isolated" environment
-pip_install_flags="--ignore-installed --verbose --no-deps --no-index --isolated --no-build-isolation"
-
-# Consume any additional pip install arguments except the last one
-while [ $# -gt 1 ]; do
-    pip_install_flags="$pip_install_flags $1"
-    shift
-done
-
-# Last argument must be "." and will be ignored
-if [ "$1" != "." ]; then
-    echo >&2 "$0 requires . as final argument"
-    exit 1
-fi
-
+pip_install_flags="--ignore-installed --verbose --no-deps --no-index --isolated"
 
 # Note: We need to take care to specify the full path to Sage's Python here
-# to emphasize that this command hould use it, and not the system Python;
+# to emphasize that this command should use it, and not the system Python;
 # see https://trac.sagemath.org/ticket/18438
 # But now we delegate this to sage-python23.
 PYTHON=sage-python23
 
 # The PIP variable is only used to determine the name of the lock file.
-if [ "$SAGE_PYTHON3" = yes ]; then
-    PIP=pip3
-else
-    PIP=pip2
-fi
+PIP=pip3
 
-
-# Find out the name of the package that we are installing
-name="$($PYTHON setup.py --name)"
-
-if [ $? -ne 0 ]; then
-    echo >&2 "Error: could not determine package name"
-    exit 1
-fi
-
-if [ $(echo "$name" | wc -l) -gt 1 ]; then
-    name="$(echo "$name" | tail -1)"
-    echo >&2 "Warning: This package has a badly-behaved setup.py which outputs"
-    echo >&2 "more than the package name for 'setup.py --name'; using the last"
-    echo >&2 "line as the package name: $name"
-fi
-
-
-# We should avoid running pip2/3 while uninstalling a package because that
+# We should avoid running pip while installing a package because that
 # is prone to race conditions. Therefore, we use a lockfile while
 # running pip. This is implemented in the Python script sage-flock
 LOCK="$SAGE_LOCAL/var/lock/$PIP.lock"
 
-# Keep uninstalling as long as it succeeds
-while true; do
-    out=$(sage-flock -x $LOCK $PYTHON -m pip uninstall --disable-pip-version-check -y "$name" 2>&1)
-    if [ $? -ne 0 ]; then
-        # Uninstall failed
-        echo >&2 "$out"
-        exit 1
-    fi
-
-    # Uninstall succeeded, which may mean that the package was not
-    # installed to begin with.
-    if [[ "$out" != *"not installed" ]]; then
-        break
-    fi
-done
-
-
 # Finally actually do the installation (the "SHARED" tells pip2/3-lock
 # to apply a shared lock)
-echo "Installing package $name using $PIP"
-
-sage-flock -s $LOCK $PYTHON -m pip install $pip_install_flags .
+sage-flock -s $LOCK $PYTHON -m pip install $pip_install_flags "$@"
 if [ $? -ne 0 ]; then
     echo >&2 "Error: installing with $PIP failed"
     exit 3

build/bin/sage-pip-uninstall

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+# This command ensures that any previous installations of the same package
+# are uninstalled.
+
+# Only argument must be "." and will be ignored.
+if [ $# -gt 1 ]; then
+    echo >&2 "$0 requires . as only argument"
+    exit 1
+fi
+if [ "$1" != "." ]; then
+    echo >&2 "$0 requires . as final argument"
+    exit 1
+fi
+
+# Note: We need to take care to specify the full path to Sage's Python here
+# to emphasize that this command should use it, and not the system Python;
+# see https://trac.sagemath.org/ticket/18438
+# But now we delegate this to sage-python23.
+PYTHON=sage-python23
+
+# The PIP variable is only used to determine the name of the lock file.
+PIP=pip3
+
+# Find out the name of the package that we are installing
+name="$($PYTHON setup.py --name)"
+
+if [ $? -ne 0 ]; then
+    echo >&2 "Error: could not determine package name"
+    exit 1
+fi
+
+if [ $(echo "$name" | wc -l) -gt 1 ]; then
+    name="$(echo "$name" | tail -1)"
+    echo >&2 "Warning: This package has a badly-behaved setup.py which outputs"
+    echo >&2 "more than the package name for 'setup.py --name'; using the last"
+    echo >&2 "line as the package name: $name"
+fi
+
+# We should avoid running pip while uninstalling a package because that
+# is prone to race conditions. Therefore, we use a lockfile while
+# running pip. This is implemented in the Python script sage-flock
+LOCK="$SAGE_LOCAL/var/lock/$PIP.lock"
+
+# Keep uninstalling as long as it succeeds
+while true; do
+    out=$(sage-flock -x $LOCK $PYTHON -m pip uninstall --disable-pip-version-check -y "$name" 2>&1)
+    if [ $? -ne 0 ]; then
+        # Uninstall failed
+        echo >&2 "$out"
+        exit 1
+    fi
+
+    # Uninstall succeeded, which may mean that the package was not
+    # installed to begin with.
+    if [[ "$out" != *"not installed" ]]; then
+        break
+    fi
+done

build/make/install

@@ -20,6 +20,7 @@ export SAGE_PKGCONFIG="$SAGE_LOCAL/lib/pkgconfig"
 export SAGE_LOGS="$SAGE_ROOT/logs/pkgs"
 export SAGE_SPKG_INST="$SAGE_LOCAL/var/lib/sage/installed"
 export SAGE_SPKG_SCRIPTS="$SAGE_LOCAL/var/lib/sage/scripts"
+export SAGE_SPKG_WHEELS="$SAGE_LOCAL/var/lib/sage/wheels"
 
 if [ -z "${SAGE_ORIG_PATH_SET}" ]; then
     SAGE_ORIG_PATH=$PATH && export SAGE_ORIG_PATH

build/pkgs/gambit/spkg-install.in

@@ -11,8 +11,9 @@ cd src/python
 rm gambit/lib/libgambit.cpp
 
 # pip doesn't work (https://github.com/gambitproject/gambit/issues/207)
-sage-python23 setup.py --no-user-cfg build install
+sage-python23 setup.py --no-user-cfg bdist_wheel
 if [ $? -ne 0 ]; then
     echo "Error installing Python API"
     exit 1
 fi
+sdh_store_and_pip_install_wheel .

build/pkgs/numpy/spkg-install.in

@@ -23,7 +23,7 @@ export NUMPY_FCONFIG="config_fc --noopt --noarch"
 
 sage-python23 setup.py \
         --no-user-cfg \
-       install \
-       --single-version-externally-managed \
-       --root "$SAGE_DESTDIR" \
-       ${NUMPY_FCONFIG} || sdh_die "Error building / installing numpy"
+       bdist_wheel \
+       ${NUMPY_FCONFIG} || sdh_die "Error building wheel for numpy"
+
+sdh_store_and_pip_install_wheel .

build/pkgs/pillow/spkg-install.in

@@ -24,6 +24,6 @@ sage-python23 setup.py \
        --debug \
        --disable-jpeg \
        $extra_build_ext \
-       install \
-       --single-version-externally-managed \
-       --root "$SAGE_DESTDIR" || sdh_die "Error building/installing Pillow"
+       bdist_wheel || sdh_die "Error building/installing Pillow"
+
+sdh_store_and_pip_install_wheel .

src/doc/en/developer/packaging.rst

@@ -372,12 +372,26 @@ begin with ``sdh_``, which stands for "Sage-distribution helper".
    arguments. If ``$SAGE_DESTDIR`` is not set then the command is run
    with ``$SAGE_SUDO``, if set.
 
-- ``sdh_pip_install [...]``: Runs ``pip install`` with the given
-   arguments, as well as additional default arguments used for
-   installing packages into Sage with pip. Currently this is just a
-   wrapper around the ``sage-pip-install`` command. If
-   ``$SAGE_DESTDIR`` is not set then the command is run with
-   ``$SAGE_SUDO``, if set.
+- ``sdh_pip_install [...]``: The equivalent of running ``pip install``
+   with the given arguments, as well as additional default arguments used for
+   installing packages into Sage with pip. The last argument must be
+   ``.`` to indicate installation from the current directory.
+
+   ``sdh_pip_install`` actually does the installation via ``pip wheel``,
+   creating a wheel file in ``dist/``, followed by
+   ``sdh_store_and_pip_install_wheel`` (see below).
+
+- ``sdh_store_and_pip_install_wheel .``: The current directory,
+   indicated by the required argument ``.``, must have a subdirectory
+   ``dist`` containing a unique wheel file (``*.whl``).
+
+   This command (1) moves this wheel file to the
+   directory ``$SAGE_SPKG_WHEELS`` (``$SAGE_LOCAL/var/lib/sage/wheels``)
+   and then (2) installs the wheel in ``$SAGE_LOCAL``.
+
+   Both of these steps, instead of writing directly into ``$SAGE_LOCAL``,
+   use the staging directory ``$SAGE_DESTDIR`` if set; otherwise, they
+   use ``$SAGE_SUDO`` (if set).
 
 - ``sdh_install [-T] SRC [SRC...] DEST``: Copies one or more files or
    directories given as ``SRC`` (recursively in the case of