sagemathgh-37926: Makefile, .ci/write-dockerfile.sh: Update for `…

…src/pyproject.toml` after sagemath#36982

    
<!-- ^ Please provide a concise and informative title. -->
<!-- ^ Don't put issue numbers in the title, do this in the PR
description below. -->
<!-- ^ For example, instead of "Fixes sagemath#12345" use "Introduce new method
to calculate 1 + 2". -->
<!-- v Describe your changes below in detail. -->
<!-- v Why is this change required? What problem does it solve? -->
<!-- v If this PR resolves an open issue, please link to it here. For
example, "Fixes sagemath#12345". -->

Also some unrelated fixes:
- fix for `centos-7-devtoolset` needed to adjust for a small change in
how Docker files are parsed (https://github.com/sagemath/sage/actions/ru
ns/8931531493/job/24538795775).
- fix for `ubuntu-xenial-...` and other platforms that need special
`configure` args
- fix for current macOS Docker Desktop

### 📝 Checklist

<!-- Put an `x` in all the boxes that apply. -->

- [x] The title is concise and informative.
- [ ] The description explains in detail what this PR is about.
- [x] I have linked a relevant issue or discussion.
- [ ] I have created tests covering the changes.
- [ ] I have updated the documentation and checked the documentation
preview.

### ⌛ Dependencies

<!-- List all open PRs that this PR logically depends on. For example,
-->
<!-- - sagemath#12345: short description why this is a dependency -->
<!-- - sagemath#34567: ... -->
    
URL: sagemath#37926
Reported by: Matthias Köppe
Reviewer(s): Kwankyu Lee, Matthias Köppe

.ci/retrofit-worktree.sh

@@ -12,11 +12,13 @@ export GIT_AUTHOR_EMAIL="[email protected]"
 export GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME"
 export GIT_COMMITTER_EMAIL="$GIT_AUTHOR_EMAIL"
 
+set -e
+
 # Set globally for other parts of the workflow
 git config --global user.name "$GIT_AUTHOR_NAME"
 git config --global user.email "$GIT_AUTHOR_EMAIL"
 
-set -ex
+set -x
 
 # If actions/checkout downloaded our source tree using the GitHub REST API
 # instead of with git (because do not have git installed in our image),
@@ -36,6 +38,10 @@ git tag -f new
 # The changed files now show up as uncommitted changes.
 # The final "git add -N" makes sure that files that were added in "new" do not show
 # as untracked files, which would be removed by "git clean -fx".
+if [ -L $WORKTREE_NAME ]; then
+    rm -f $WORKTREE_NAME
+    git worktree prune --verbose
+fi
 git worktree add --detach $WORKTREE_NAME
 rm -rf $WORKTREE_DIRECTORY/.git && mv $WORKTREE_NAME/.git $WORKTREE_DIRECTORY/
 rm -rf $WORKTREE_NAME && ln -s $WORKTREE_DIRECTORY $WORKTREE_NAME

.ci/write-dockerfile.sh

@@ -1,20 +1,41 @@
 #! /usr/bin/env bash
-## Write a Dockerfile to stdout that tests that the packages listed in the debian.txt/fedora.txt files of standard spkg exist
-## and satisfy the requirements tested by spkg-configure.m4
-## This is called by $SAGE_ROOT/tox.ini
 set -e
-SYSTEM="${1:-debian}"
 shopt -s extglob
+##
+## Write a Dockerfile for portability testing to stdout.
+##
+## This script needs to be run from SAGE_ROOT (root of the Sage repository).
+## It is called by $SAGE_ROOT/tox.ini for all environments 'tox -e docker-...'
+##
+## Positional arguments:
+##
+SYSTEM="${1:-debian}"
 SAGE_PACKAGE_LIST_ARGS="${2:-:standard:}"
 WITH_SYSTEM_SPKG="${3:-yes}"
 IGNORE_MISSING_SYSTEM_PACKAGES="${4:-no}"
 EXTRA_SAGE_PACKAGES="${5:-_bootstrap}"
-#
+##
+## Environment variables that take influence:
+##
+## - BOOTSTRAP
+## - CONFIGURE_ARGS
+## - DEVTOOLSET
+## - DIST_UPGRADE
+## - DOCKER_BUILDKIT
+## - EXTRA_PATH
+## - EXTRA_REPOSITORIES
+## - EXTRA_SYSTEM_PACKAGES
+## - FULL_BASE_IMAGE_AND_TAG
+## - SKIP_SYSTEM_PKG_INSTALL
+## - USE_CONDARC
+## - __CHOWN
+## - __SUDO
+##
 STRIP_COMMENTS="sed s/#.*//;"
 SAGE_ROOT=.
 export PATH="$SAGE_ROOT"/build/bin:$PATH
 SYSTEM_PACKAGES=$EXTRA_SYSTEM_PACKAGES
-CONFIGURE_ARGS="--enable-option-checking "
+SYSTEM_CONFIGURE_ARGS="--enable-option-checking "
 for SPKG in $(sage-package list --has-file=spkg-configure.m4 $SAGE_PACKAGE_LIST_ARGS) $EXTRA_SAGE_PACKAGES; do
     SYSTEM_PACKAGE=$(sage-get-system-packages $SYSTEM $SPKG)
     if [ -n "${SYSTEM_PACKAGE}" ]; then
@@ -24,7 +45,7 @@ for SPKG in $(sage-package list --has-file=spkg-configure.m4 $SAGE_PACKAGE_LIST_
             # shell-quote package if necessary
             SYSTEM_PACKAGES+=$(printf " %q" "$a")
         done
-        CONFIGURE_ARGS+="--with-system-${SPKG}=${WITH_SYSTEM_SPKG} "
+        SYSTEM_CONFIGURE_ARGS+="--with-system-${SPKG}=${WITH_SYSTEM_SPKG} "
     fi
 done
 echo "# Automatically generated by SAGE_ROOT/.ci/write-dockerfile.sh"
@@ -247,19 +268,17 @@ $ADD Makefile VERSION.txt COPYING.txt condarc.yml README.md bootstrap bootstrap-
 $ADD config/config.rpath /new/config/config.rpath
 $ADD src/doc/bootstrap /new/src/doc/bootstrap
 $ADD src/bin /new/src/bin
-$ADD src/Pipfile.m4 src/pyproject.toml.m4 src/requirements.txt.m4 src/setup.cfg.m4 src/VERSION.txt /new/src/
+$ADD src/Pipfile.m4 src/pyproject.toml src/requirements.txt.m4 src/setup.cfg.m4 src/VERSION.txt /new/src/
 $ADD m4 /new/m4
 $ADD pkgs /new/pkgs
 $ADD build /new/build
 $ADD .ci /new/.ci
 $ADD .upstream.d /new/.upstream.d
 RUN if [ -d /sage ]; then                                               \
         echo "### Incremental build from \$(cat /sage/VERSION.txt)" &&  \
-        if command -v git; then                                         \
-            (cd /new &&                                                 \
-             echo /src >> .gitignore &&                                 \
-             ./.ci/retrofit-worktree.sh worktree-image /sage);          \
-        else                                                            \
+        printf '/src\n!/src/doc/bootstrap\n!/src/bin\n!/src/*.m4\n!/src/*.toml\n!/src/VERSION.txt\n' >> /sage/.gitignore && \
+        printf '/src\n!/src/doc/bootstrap\n!/src/bin\n!/src/*.m4\n!/src/*.toml\n!/src/VERSION.txt\n' >> /new/.gitignore && \
+        if ! (cd /new && ./.ci/retrofit-worktree.sh worktree-image /sage); then \
             for a in local logs; do                                     \
                 if [ -d /sage/\$a ]; then mv /sage/\$a /new/; fi;       \
             done;                                                       \
@@ -271,21 +290,21 @@ RUN if [ -d /sage ]; then                                               \
     fi
 WORKDIR /sage
 
-ARG BOOTSTRAP=${BOOTSTRAP-./bootstrap}
+ARG BOOTSTRAP="${BOOTSTRAP-./bootstrap}"
 $RUN sh -x -c "\${BOOTSTRAP}" $ENDRUN $THEN_SAVE_STATUS
 
 FROM bootstrapped as configured
 #:configuring:
 RUN $CHECK_STATUS_THEN mkdir -p logs/pkgs; rm -f config.log; ln -s logs/pkgs/config.log config.log
-ARG EXTRA_CONFIGURE_ARGS="${CONFIGURE_ARGS}"
+ARG CONFIGURE_ARGS="${CONFIGURE_ARGS:---enable-build-as-root}"
 EOF
 if [ ${WITH_SYSTEM_SPKG} = "force" ]; then
     cat <<EOF
-$RUN echo "****** Configuring: ./configure --enable-build-as-root $CONFIGURE_ARGS \${EXTRA_CONFIGURE_ARGS} *******"; ./configure --enable-build-as-root $CONFIGURE_ARGS \${EXTRA_CONFIGURE_ARGS} || (echo "********** configuring without forcing ***********"; echo "::group::config.log"; cat config.log; echo "::endgroup::"; ./configure --enable-build-as-root; echo "::group::config.log"; cat config.log; echo "::endgroup::"; exit 1) $ENDRUN $THEN_SAVE_STATUS
+$RUN ./configure $SYSTEM_CONFIGURE_ARGS \${CONFIGURE_ARGS} || (echo "::group::config.log"; cat config.log; echo "::endgroup::"; echo "********** configuring without forcing ***********"; ./configure \${CONFIGURE_ARGS}; echo "::group::config.log"; cat config.log; echo "::endgroup::"; exit 1) $ENDRUN $THEN_SAVE_STATUS
 EOF
 else
     cat <<EOF
-$RUN echo "****** Configuring: ./configure --enable-build-as-root $CONFIGURE_ARGS \${EXTRA_CONFIGURE_ARGS} *******"; ./configure --enable-build-as-root $CONFIGURE_ARGS \${EXTRA_CONFIGURE_ARGS} || (echo "::group::config.log"; cat config.log; echo "::endgroup::"; exit 1) $ENDRUN $THEN_SAVE_STATUS
+$RUN ./configure $SYSTEM_CONFIGURE_ARGS \${CONFIGURE_ARGS} || (echo "::group::config.log"; cat config.log; echo "::endgroup::"; exit 1) $ENDRUN $THEN_SAVE_STATUS
 EOF
 fi
 cat <<EOF
@@ -318,12 +337,11 @@ ENV SAGE_CHECK=warn
 ENV SAGE_CHECK_PACKAGES="!cython,!r,!python3,!gap,!cysignals,!linbox,!git,!ppl,!cmake,!rpy2,!sage_sws2rst"
 $ADD .gitignore /new/.gitignore
 $ADD src /new/src
-RUN if command -v git; then                             \
-        cd /new && rm -rf .git &&                       \
-        ./.ci/retrofit-worktree.sh worktree-pre /sage;  \
-    else                                                \
-        rm -rf /sage/src;                               \
-        mv /new/src /sage/src;                          \
+RUN cd /new && rm -rf .git && \
+    if ! /sage/.ci/retrofit-worktree.sh worktree-pre /sage; then \
+        rm -rf /sage/src;                                    \
+        mv src /sage/src;                                    \
+        cd /sage && ./bootstrap && ./config.status;          \
     fi
 
 ARG TARGETS="build"

.gitignore

@@ -204,6 +204,8 @@ build/pkgs/wheel/version_requirements.txt
 /pkgs/*/MANIFEST
 /pkgs/*/*.egg-info
 /pkgs/*/.tox
+/pkgs/*/venv*
+/pkgs/*/.venv*
 
 /pkgs/sage-conf_pypi/sage_root/config.log
 /pkgs/sage-conf_pypi/sage_root/config.status

Makefile

@@ -351,6 +351,7 @@ ptestoptionallong-nodoc:
 # CONFIGURE_DEPENDENCIES is the list of files that influence the generation of 'configure'.
 CONFIGURE_DEPENDENCIES =							\
 	configure.ac src/bin/sage-version.sh m4/*.m4				\
+	src/pyproject.toml							\
 	build/pkgs/*/spkg-configure.m4						\
 	build/pkgs/*/type build/pkgs/*/SPKG.rst					\
 	build/pkgs/*/checksums.ini build/pkgs/*/requirements.txt		\

src/sage/misc/package_dir.py

@@ -273,14 +273,14 @@ def is_package_or_sage_namespace_package_dir(path, *, distribution_filter=None):
 
         sage: directory = os.path.join(sage.libs.__path__[0], 'mpfr'); directory
         '.../sage/libs/mpfr'
-        sage: is_package_or_sage_namespace_package_dir(directory)
+        sage: is_package_or_sage_namespace_package_dir(directory)       # known bug (seen in build.yml)
         True
 
     :mod:`sage` is designated to become an implicit namespace package::
 
         sage: directory = sage.__path__[0]; directory
         '.../sage'
-        sage: is_package_or_sage_namespace_package_dir(directory)
+        sage: is_package_or_sage_namespace_package_dir(directory)       # known bug (seen in build.yml)
         True
 
     Not a package::

tox.ini

@@ -148,6 +148,12 @@ passenv =
                   # Use DOCKER_BUILDKIT=0 for legacy builder
     docker:       DOCKER_BUILDKIT
     docker:       BUILDKIT_INLINE_CACHE
+    docker:       DOCKER_HOST
+                  # On macOS, the docker CLI may need $HOME to find the socket
+                  # $HOME/Library/Containers/com.docker.docker/Data/docker-cli.sock
+                  # Otherwise may get the error "Cannot connect to the Docker daemon at
+                  # unix:///var/run/docker.sock. Is the docker daemon running?"
+    docker:       HOME
                   # Set for example to "with-system-packages configured with-targets-pre with-targets"
                   # to tag intermediate images.
     docker:       DOCKER_TARGETS
@@ -198,8 +204,8 @@ setenv =
     # Whether to add the system packages needed for bootstrapping
                            EXTRA_SAGE_PACKAGES_0=_bootstrap
     nobootstrap:           EXTRA_SAGE_PACKAGES_0=
-    # local envs need HOME set, also Docker 19.03 needs HOME
-    {local,docker}: HOME={envdir}
+    # local envs need HOME set
+    local:            HOME={envdir}
     # for local envs we can guess the package system if it is not provided as a factor
     local:          SYSTEM=auto
     #
@@ -507,6 +513,7 @@ setenv =
     docker-incremental-sitepackages:                    SKIP_SYSTEM_PKG_INSTALL=no
     #
     docker-nobootstrap: BOOTSTRAP=./bootstrap -D
+    docker:           CONFIG_CONFIGURE_ARGS_ROOT=--enable-build-as-root
     ###
     ### "local" envs
     ###
@@ -758,7 +765,7 @@ commands =
     local:         bash -c 'if [ ! -d prefix -o -L prefix ]; then rm -f prefix; ln -sf {env:PREFIX:{envdir}/local} prefix; fi'
 
 ##commands =
-    docker:        bash -c '.ci/write-dockerfile.sh {env:SYSTEM} "{env:SAGE_PACKAGE_LIST_ARGS:}" {env:WITH_SYSTEM_SPKG} {env:IGNORE_MISSING_SYSTEM_PACKAGES} "{env:ALL_EXTRA_SAGE_PACKAGES}" > {envdir}/Dockerfile'
+    docker:        bash -c 'BUILD_TAG={env:DOCKER_TAG:$(git describe --dirty --always)} .ci/write-dockerfile.sh {env:SYSTEM} "{env:SAGE_PACKAGE_LIST_ARGS:}" {env:WITH_SYSTEM_SPKG} {env:IGNORE_MISSING_SYSTEM_PACKAGES} "{env:ALL_EXTRA_SAGE_PACKAGES}" > {envdir}/Dockerfile'
     # From https://hub.docker.com/r/multiarch/ubuntu-core/
     # configure binfmt-support on the Docker host (works locally or remotely, i.e: using boot2docker)
     docker-{arm64,armhf}:  docker run --rm --privileged multiarch/qemu-user-static:register --reset