Trac #30845: Fixes for the conda-for-Sage-developers installation met…

…hod, add GH Actions workflow

Add github action workflow that checks the build of sage in a conda
environment, completely bypassing the installation of any sage package.
This is based on the steps outlined at https://wiki.sagemath.org/Conda.

This also tests that the generated `src/environment*.yml` files work
correctly. (See documentation added in #28745)

Run: https://github.com/sagemath/sagetrac-mirror/actions/workflows/ci-
conda.yml

Fixes:
- #33330: primecountpy is not installed in the conda env because its not
listed in `environment.yml`
- environment-optional: lcalc 1 is installed and then not recognized /
leads to error
{{{
building 'sage.libs.lcalc.lcalc_Lfunction' extension
    cc1plus: warning: command line option '-Wstrict-prototypes' is valid
for C/ObjC but not for C++
    In file included from sage/libs/lcalc/lcalc_Lfunction.cpp:740:
    sage/libs/lcalc/lcalc_sage.h:1:10: fatal error: lcalc/L.h: No such
file or directory
        1 | #include "lcalc/L.h"
          |          ^~~~~~~~~~~
    compilation terminated.
}}}
- environment-optional: probably related, pari 2.11.4 is installed by
conda instead of the newest 2.13.2. This version is then rejected by
`configure` as being too old.
- Disable conda distro information for `gdb` because it is broken on
macOS.
- Add conda distro information for `lrcalc_python`
- Filter out conda-specific `ld` warnings in doctests
- In the feature test for `sage_spkg`, check whether any Sage packages
are actually installed
- Include an upper version bound for `ptyprocess` in sagemath-standard's
install-requires

Follow-ups: See Meta-ticket #33331

URL: https://trac.sagemath.org/30845
Reported by: mkoeppe
Ticket author(s): Tobias Diez, Matthias Koeppe
Reviewer(s): Matthias Koeppe, Dima Pasechnik, Tobias Diez

.github/workflows/ci-conda.yml

@@ -0,0 +1,107 @@
+name: Build & Test using Conda
+
+on:
+  pull_request:
+    types: [opened, synchronize]
+  push:
+    tags:
+      - '*'
+    branches:
+      - 'public/build/**-runci'
+  workflow_dispatch:
+    # Allow to run manually
+
+concurrency:
+  # Cancel previous runs of this workflow for the same branch
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: Conda
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        python: [3.8, 3.9]
+        conda-env: [environment, environment-optional]
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Check for Miniconda
+        id: check_conda
+        run: echo ::set-output name=installed::$CONDA
+
+      # Miniconda is installed by default in the ubuntu-latest, however not in the act-image to run it locally
+      - name: Install Miniconda
+        if: steps.check_conda.outputs.installed == ''
+        run: |
+          wget https://repo.anaconda.com/miniconda/Miniconda3-py39_4.10.3-Linux-x86_64.sh -O ~/miniconda.sh
+          bash ~/miniconda.sh -b -p $HOME/miniconda
+          echo "CONDA=$HOME/miniconda" >> $GITHUB_ENV
+
+      - name: Install bootstrap prerequisites
+        run: |
+          export PATH="$(pwd)/build/bin:$PATH"
+          SYSTEM=$(sage-guess-package-system)
+          eval $(sage-print-system-package-command $SYSTEM --sudo install $(sage-get-system-packages $SYSTEM _bootstrap))
+
+      # Create conda environment file
+      - name: Bootstrap
+        run: ./bootstrap
+
+      - name: Cache conda packages
+        uses: actions/cache@v2
+        with:
+          path: ~/conda_pkgs_dir
+          key:
+            ${{ runner.os }}-conda-${{ hashFiles('src/environment.yml') }}
+
+      - name: Setup Conda
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          python-version: ${{ matrix.python }}
+          mamba-version: "*"
+          channels: conda-forge,defaults
+          channel-priority: true
+          activate-environment: sage-build
+          environment-file: src/${{ matrix.conda-env }}.yml
+
+      - name: Print Conda environment
+        shell: bash -l {0}
+        run: |
+          conda info
+          conda list
+
+      - name: Configure
+        shell: bash -l {0}
+        continue-on-error: true
+        run: |
+          echo "::add-matcher::.github/workflows/configure-systempackage-problem-matcher.json"
+          ./configure --enable-build-as-root --with-python=$CONDA_PREFIX/bin/python --prefix=$CONDA_PREFIX $(for pkg in $(./sage -package list :standard: --has-file spkg-configure.m4 --has-file distros/conda.txt); do echo --with-system-$pkg=force; done)
+          echo "::remove-matcher owner=configure-system-package-warning::"
+          echo "::remove-matcher owner=configure-system-package-error::"
+
+      - name: Build
+        shell: bash -l {0}
+        run: |
+          pip install --no-build-isolation -v -v -e pkgs/sage-conf pkgs/sage-setup 
+          pip install --no-build-isolation -v -v -e src
+        env:
+          SAGE_NUM_THREADS: 2
+
+      - name: Test
+        shell: bash -l {0}
+        run: ./sage -t --all -p0
+
+      - name: Print logs
+        run: |
+          for file in $(find . -type f -name "*.log"); do
+              echo "::group::$file"
+              cat "$file"
+              echo "::endgroup::"
+          done
+        if: always()

.github/workflows/configure-systempackage-problem-matcher.json

@@ -0,0 +1,27 @@
+{
+    "problemMatcher": [
+        {
+            "owner": "configure-system-package-warning",
+            "severity": "warning",
+            "pattern": [
+                {
+                    "regexp": "((\\S*)\\:(\\s*)(no suitable system package; standard, will be installed as an SPKG))",
+                    "file": 2,
+                    "message": 1,
+                    "kind": "file"
+                }
+            ]
+        },
+        {
+            "owner": "configure-system-package-error",
+            "pattern": [
+                {
+                    "regexp": "((\\S*)\\:(\\s*)(no suitable system package; this is an error))",
+                    "file": 2,
+                    "message": 1,
+                    "kind": "file"
+                }
+            ]
+        }
+    ]
+}

build/pkgs/gdb/distros/conda.txt

@@ -1 +1,2 @@
-gdb
+# Disabled for now because of https://trac.sagemath.org/ticket/30845#comment:269
+# gdb

build/pkgs/lrcalc_python/distros/conda.txt

@@ -0,0 +1 @@
+python-lrcalc

src/sage/doctest/parsing.py

@@ -41,6 +41,8 @@
 glpk_simplex_warning_regex = re.compile(r'(Long-step dual simplex will be used)')
 # :trac:`31204` -- suppress warning about ld and OS version for dylib files.
 ld_warning_regex = re.compile(r'^.*dylib.*was built for newer macOS version.*than being linked.*')
+# :trac:`30845` -- suppress warning on conda about ld
+ld_pie_warning_regex = re.compile(r'ld: warning: -pie being ignored. It is only used when linking a main executable')
 find_sage_prompt = re.compile(r"^(\s*)sage: ", re.M)
 find_sage_continuation = re.compile(r"^(\s*)\.\.\.\.:", re.M)
 find_python_continuation = re.compile(r"^(\s*)\.\.\.([^\.])", re.M)
@@ -115,7 +117,10 @@ def fake_RIFtol(*args):
      lambda g, w: (glpk_simplex_warning_regex.sub('', g), w)),
 
     (lambda g, w: "dylib" in g,
-     lambda g, w: (ld_warning_regex.sub('', g), w))
+     lambda g, w: (ld_warning_regex.sub('', g), w)),
+
+    (lambda g, w: "pie being ignored" in g,
+     lambda g, w: (ld_pie_warning_regex.sub('', g), w))
 ]
 
 

src/sage/features/pkg_systems.py

@@ -115,9 +115,16 @@ def _is_present(self):
         try:
             # "sage -p" is a fast way of checking whether sage-spkg is available.
             run('sage -p', shell=True, stdout=DEVNULL, stderr=DEVNULL, check=True)
-            return True
         except CalledProcessError:
             return False
+        # Check if there are any installation records.
+        try:
+            from sage.misc.package import installed_packages
+        except ImportError:
+            return False
+        for pkg in installed_packages(exclude_pip=True):
+            return True
+        return False
 
     def _spkg_installation_hint(self, spkgs, prompt, feature):
         r"""

src/setup.cfg.m4

@@ -70,6 +70,10 @@ dnl Other Python packages that are standard spkg, used in doctests
         fpylll         \
         | sed "2,\$s/^/    /;"')dnl'
 dnl pycryptosat  # Sage distribution installs it as part of cryptominisat. According to its README on https://pypi.org/project/pycryptosat/: "The pycryptosat python package compiles while compiling CryptoMiniSat. It cannot be compiled on its own, it must be compiled at the same time as CryptoMiniSat."
+dnl Packages with important upper version bounds
+    esyscmd(`sage-get-system-packages install-requires \
+        ptyprocess     \
+        | sed "2,\$s/^/    /;"')dnl'
 
 scripts =
     # The sage script