diff --git a/.ci/retrofit-worktree.sh b/.ci/retrofit-worktree.sh index b9c880a839f..5dcb6e2b88f 100755 --- a/.ci/retrofit-worktree.sh +++ b/.ci/retrofit-worktree.sh @@ -12,11 +12,13 @@ export GIT_AUTHOR_EMAIL="ci-sage@example.com" export GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME" export GIT_COMMITTER_EMAIL="$GIT_AUTHOR_EMAIL" +set -e + # Set globally for other parts of the workflow git config --global user.name "$GIT_AUTHOR_NAME" git config --global user.email "$GIT_AUTHOR_EMAIL" -set -ex +set -x # If actions/checkout downloaded our source tree using the GitHub REST API # instead of with git (because do not have git installed in our image), @@ -36,6 +38,10 @@ git tag -f new # The changed files now show up as uncommitted changes. # The final "git add -N" makes sure that files that were added in "new" do not show # as untracked files, which would be removed by "git clean -fx". +if [ -L $WORKTREE_NAME ]; then + rm -f $WORKTREE_NAME + git worktree prune --verbose +fi git worktree add --detach $WORKTREE_NAME rm -rf $WORKTREE_DIRECTORY/.git && mv $WORKTREE_NAME/.git $WORKTREE_DIRECTORY/ rm -rf $WORKTREE_NAME && ln -s $WORKTREE_DIRECTORY $WORKTREE_NAME diff --git a/.ci/write-dockerfile.sh b/.ci/write-dockerfile.sh index 0aa53414553..578680d76f2 100755 --- a/.ci/write-dockerfile.sh +++ b/.ci/write-dockerfile.sh @@ -1,20 +1,41 @@ #! /usr/bin/env bash -## Write a Dockerfile to stdout that tests that the packages listed in the debian.txt/fedora.txt files of standard spkg exist -## and satisfy the requirements tested by spkg-configure.m4 -## This is called by $SAGE_ROOT/tox.ini set -e -SYSTEM="${1:-debian}" shopt -s extglob +## +## Write a Dockerfile for portability testing to stdout. +## +## This script needs to be run from SAGE_ROOT (root of the Sage repository). +## It is called by $SAGE_ROOT/tox.ini for all environments 'tox -e docker-...' +## +## Positional arguments: +## +SYSTEM="${1:-debian}" SAGE_PACKAGE_LIST_ARGS="${2:-:standard:}" WITH_SYSTEM_SPKG="${3:-yes}" IGNORE_MISSING_SYSTEM_PACKAGES="${4:-no}" EXTRA_SAGE_PACKAGES="${5:-_bootstrap}" -# +## +## Environment variables that take influence: +## +## - BOOTSTRAP +## - CONFIGURE_ARGS +## - DEVTOOLSET +## - DIST_UPGRADE +## - DOCKER_BUILDKIT +## - EXTRA_PATH +## - EXTRA_REPOSITORIES +## - EXTRA_SYSTEM_PACKAGES +## - FULL_BASE_IMAGE_AND_TAG +## - SKIP_SYSTEM_PKG_INSTALL +## - USE_CONDARC +## - __CHOWN +## - __SUDO +## STRIP_COMMENTS="sed s/#.*//;" SAGE_ROOT=. export PATH="$SAGE_ROOT"/build/bin:$PATH SYSTEM_PACKAGES=$EXTRA_SYSTEM_PACKAGES -CONFIGURE_ARGS="--enable-option-checking " +SYSTEM_CONFIGURE_ARGS="--enable-option-checking " for SPKG in $(sage-package list --has-file=spkg-configure.m4 $SAGE_PACKAGE_LIST_ARGS) $EXTRA_SAGE_PACKAGES; do SYSTEM_PACKAGE=$(sage-get-system-packages $SYSTEM $SPKG) if [ -n "${SYSTEM_PACKAGE}" ]; then @@ -24,7 +45,7 @@ for SPKG in $(sage-package list --has-file=spkg-configure.m4 $SAGE_PACKAGE_LIST_ # shell-quote package if necessary SYSTEM_PACKAGES+=$(printf " %q" "$a") done - CONFIGURE_ARGS+="--with-system-${SPKG}=${WITH_SYSTEM_SPKG} " + SYSTEM_CONFIGURE_ARGS+="--with-system-${SPKG}=${WITH_SYSTEM_SPKG} " fi done echo "# Automatically generated by SAGE_ROOT/.ci/write-dockerfile.sh" @@ -247,7 +268,7 @@ $ADD Makefile VERSION.txt COPYING.txt condarc.yml README.md bootstrap bootstrap- $ADD config/config.rpath /new/config/config.rpath $ADD src/doc/bootstrap /new/src/doc/bootstrap $ADD src/bin /new/src/bin -$ADD src/Pipfile.m4 src/pyproject.toml.m4 src/requirements.txt.m4 src/setup.cfg.m4 src/VERSION.txt /new/src/ +$ADD src/Pipfile.m4 src/pyproject.toml src/requirements.txt.m4 src/setup.cfg.m4 src/VERSION.txt /new/src/ $ADD m4 /new/m4 $ADD pkgs /new/pkgs $ADD build /new/build @@ -255,11 +276,9 @@ $ADD .ci /new/.ci $ADD .upstream.d /new/.upstream.d RUN if [ -d /sage ]; then \ echo "### Incremental build from \$(cat /sage/VERSION.txt)" && \ - if command -v git; then \ - (cd /new && \ - echo /src >> .gitignore && \ - ./.ci/retrofit-worktree.sh worktree-image /sage); \ - else \ + printf '/src\n!/src/doc/bootstrap\n!/src/bin\n!/src/*.m4\n!/src/*.toml\n!/src/VERSION.txt\n' >> /sage/.gitignore && \ + printf '/src\n!/src/doc/bootstrap\n!/src/bin\n!/src/*.m4\n!/src/*.toml\n!/src/VERSION.txt\n' >> /new/.gitignore && \ + if ! (cd /new && ./.ci/retrofit-worktree.sh worktree-image /sage); then \ for a in local logs; do \ if [ -d /sage/\$a ]; then mv /sage/\$a /new/; fi; \ done; \ @@ -271,21 +290,21 @@ RUN if [ -d /sage ]; then \ fi WORKDIR /sage -ARG BOOTSTRAP=${BOOTSTRAP-./bootstrap} +ARG BOOTSTRAP="${BOOTSTRAP-./bootstrap}" $RUN sh -x -c "\${BOOTSTRAP}" $ENDRUN $THEN_SAVE_STATUS FROM bootstrapped as configured #:configuring: RUN $CHECK_STATUS_THEN mkdir -p logs/pkgs; rm -f config.log; ln -s logs/pkgs/config.log config.log -ARG EXTRA_CONFIGURE_ARGS="${CONFIGURE_ARGS}" +ARG CONFIGURE_ARGS="${CONFIGURE_ARGS:---enable-build-as-root}" EOF if [ ${WITH_SYSTEM_SPKG} = "force" ]; then cat < {envdir}/Dockerfile' + docker: bash -c 'BUILD_TAG={env:DOCKER_TAG:$(git describe --dirty --always)} .ci/write-dockerfile.sh {env:SYSTEM} "{env:SAGE_PACKAGE_LIST_ARGS:}" {env:WITH_SYSTEM_SPKG} {env:IGNORE_MISSING_SYSTEM_PACKAGES} "{env:ALL_EXTRA_SAGE_PACKAGES}" > {envdir}/Dockerfile' # From https://hub.docker.com/r/multiarch/ubuntu-core/ # configure binfmt-support on the Docker host (works locally or remotely, i.e: using boot2docker) docker-{arm64,armhf}: docker run --rm --privileged multiarch/qemu-user-static:register --reset