[ci] use conda-forge in Linux and macOS CI jobs (#4953)

* [ci] use conda-forge in CI jobs (fixes #4948) * comment out more jobs * try reverting graphviz patch, running more cuda jobs * get graphviz from PyPI and try removing some patches for r-lintr * start running appveyor again * use conda-forge if using conda * fix commands * conda install graphviz * try newer openmp * pin below openmp 11.x * focus on gpu task * trying to narrow down error * maybe gcc11 is the issue * start adding other tests back * pin openmp too * maybe need to pin to gcc less than 10.x * pin libgfortran and libstdcxx as well * pin to gcc 9.3.0 * move constraints up to initial environment * add all CI jobs back * try installing python-graphviz separately * try new lightgbm/vsts-agent image * fix typo * test if pinning gcc for linux gpu_source build is still necessary * ok yes, pinning gcc is necessary * test if Linux gpu_source works with Python 3.9.6 * no special exception for Linux gpu_source job * pin to Python 3.9.6 in Linux gpu_source * try explicitly asking for libstdcxx-ng for every linux build * swap compilers * switch compilers back * revert accidental whitespace change * comment out CI * try Linux gpu_source with different Python versions * Revert "try Linux gpu_source with different Python versions" This reverts commit f6f63cb. * Revert "comment out CI" This reverts commit ece191f. * remove libxml2 install, change CONDA path * avoid installing conda in rchk job * empty commit 1 * empty commit 2 * empty commit 3 * empty commit 4 * add more verbose logging around installation of python-graphviz * empty commit 1 * get mamba info * get more conda info * add another mamba info call * allow for other macOS environments in GHA configuration * Revert "allow for other macOS environments in GHA configuration" This reverts commit a3c7a19. * get more logs from mamba * get Build.ArtifactsStagingDirectory * get more logs and try to force re-installing everything * clean cache after every step * remove --update-all and make logs less verbose * remove more print statements and uncomment jobs * test if conda-clean issue fixes segfaults for gpu_source * pin python version for gpu_source * empty commit 1 * use miniforge instead * empty commit 1 * Apply suggestions from code review * bring workarounds back * remove duplicated graphviz system-wide installation (reverts #4095, #4097, #4238) * empty commit 1 * empty commit 2 * empty commit 3 * empty commit 4 * empty commit 5 * empty commit 6 * empty commit 7 * empty commit 8 * empty commit 9 * empty commit 10 * empty commit 10 * empty commit 10 * empty commit 10 * empty commit 11 * one more try * try to downgrade Python version for Linux GPU job * swap compilers * Revert "swap compilers" This reverts commit f04dc27. Co-authored-by: Nikita Titov <[email protected]> Co-authored-by: Nikita Titov <[email protected]>
microsoft · Feb 11, 2022 · 3500cb6 · 3500cb6
1 parent 0688f47
commit 3500cb6
Show file tree

Hide file tree

Showing 7 changed files with 48 additions and 46 deletions.
diff --git a/.ci/setup.sh b/.ci/setup.sh
@@ -17,8 +17,10 @@ if [[ $OS_NAME == "macos" ]]; then
     if [[ $TASK == "swig" ]]; then
         brew install swig
     fi
-    brew install graphviz
-    curl -sL -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
+    curl \
+        -sL \
+        -o miniforge.sh \
+        https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-MacOSX-x86_64.sh
 else  # Linux
     if [[ $IN_UBUNTU_LATEST_CONTAINER == "true" ]]; then
         # fixes error "unable to initialize frontend: Dialog"
@@ -42,9 +44,6 @@ else  # Linux
             libicu66 \
             libssl1.1 \
             libunwind8 \
-            libxau6 \
-            libxext6 \
-            libxrender1 \
             locales \
             netcat \
             unzip \
@@ -81,16 +80,11 @@ else  # Linux
         mv $AMDAPPSDK_PATH/lib/x86_64/sdk/* $AMDAPPSDK_PATH/lib/x86_64/
         echo libamdocl64.so > $OPENCL_VENDOR_PATH/amdocl64.icd
     fi
-    ARCH=$(uname -m)
     if [[ $TASK == "cuda" ]]; then
         echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
         apt-get update
         apt-get install --no-install-recommends -y \
             curl \
-            graphviz \
-            libxau6 \
-            libxext6 \
-            libxrender1 \
             lsb-release \
             software-properties-common
         if [[ $COMPILER == "clang" ]]; then
@@ -103,29 +97,19 @@ else  # Linux
         apt-get update
         apt-get install --no-install-recommends -y \
             cmake
-    else
-        if [[ $ARCH != "x86_64" ]]; then
-            yum update -y
-            yum install -y \
-                graphviz
-        else
-            sudo apt-get update
-            sudo apt-get install --no-install-recommends -y \
-                graphviz
-        fi
     fi
     if [[ $SETUP_CONDA != "false" ]]; then
-        if [[ $ARCH == "x86_64" ]]; then
-            curl -sL -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
-        else
-            curl -sL -o conda.sh https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-${ARCH}.sh
-        fi
+        ARCH=$(uname -m)
+        curl \
+            -sL \
+            -o miniforge.sh \
+            https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-${ARCH}.sh
     fi
 fi
 
-if [[ "${TASK}" != "r-package" ]]; then
+if [[ "${TASK}" != "r-package" ]] && [[ "${TASK}" != "r-rchk" ]]; then
     if [[ $SETUP_CONDA != "false" ]]; then
-        sh conda.sh -b -p $CONDA
+        sh miniforge.sh -b -p $CONDA
     fi
     conda config --set always_yes yes --set changeps1 no
     conda update -q -y conda

diff --git a/.ci/test.sh b/.ci/test.sh
@@ -62,16 +62,13 @@ fi
 
 if [[ $TASK == "lint" ]]; then
     conda install -q -y -n $CONDA_ENV \
+        cmakelint \
+        cpplint \
+        isort \
+        mypy \
         pycodestyle \
         pydocstyle \
-        r-stringi  # stringi needs to be installed separate from r-lintr to avoid issues like 'unable to load shared object stringi.so'
-    # r-xfun below has to be upgraded because lintr requires > 0.19 for that package
-    conda install -q -y -n $CONDA_ENV \
-        -c conda-forge \
-            libxml2 \
-            "r-xfun>=0.19" \
-            "r-lintr>=2.0"
-    pip install --user cmakelint cpplint isort mypy
+        "r-lintr>=2.0"
     echo "Linting Python code"
     pycodestyle --ignore=E501,W503 --exclude=./.nuget,./external_libs . || exit -1
     pydocstyle --convention=numpy --add-ignore=D105 --match-dir="^(?!^external_libs|test|example).*" --match="(?!^test_|setup).*\.py" . || exit -1
@@ -114,8 +111,22 @@ if [[ $TASK == "swig" ]]; then
     exit 0
 fi
 
-conda install -q -y -n $CONDA_ENV cloudpickle "dask=2021.9.1" "distributed=2021.9.1" joblib matplotlib numpy pandas psutil pytest scikit-learn scipy
-pip install graphviz  # python-graphviz from Anaconda is not allowed to be installed with Python 3.9
+conda install -q -y -n $CONDA_ENV \
+    cloudpickle \
+    dask \
+    distributed \
+    joblib \
+    matplotlib \
+    numpy \
+    pandas \
+    psutil \
+    pytest \
+    scikit-learn \
+    scipy || exit -1
+
+# python-graphviz has to be installed separately to prevent conda from downgrading to pypy
+conda install -q -y -n $CONDA_ENV \
+    python-graphviz || exit -1
 
 if [[ $OS_NAME == "macos" ]] && [[ $COMPILER == "clang" ]]; then
     # fix "OMP: Error #15: Initializing libiomp5.dylib, but found libomp.dylib already initialized." (OpenMP library conflict due to conda's MKL)
@@ -229,7 +240,11 @@ import matplotlib\
 matplotlib.use\(\"Agg\"\)\
 ' plot_example.py  # prevent interactive window mode
     sed -i'.bak' 's/graph.render(view=True)/graph.render(view=False)/' plot_example.py
-    conda install -q -y -n $CONDA_ENV h5py ipywidgets notebook  # requirements for examples
+    # requirements for examples
+    conda install -q -y -n $CONDA_ENV \
+        h5py \
+        ipywidgets \
+        notebook
     for f in *.py **/*.py; do python $f || exit -1; done  # run all examples
     cd $BUILD_DIRECTORY/examples/python-guide/notebooks
     sed -i'.bak' 's/INTERACTIVE = False/assert False, \\"Interactive mode disabled\\"/' interactive_plot_example.ipynb

diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml
@@ -81,7 +81,7 @@ jobs:
             LGB_VER=$(head -n 1 VERSION.txt)
             EOF
             cat > docker-script.sh <<EOF
-            export CONDA=\$HOME/miniconda
+            export CONDA=\$HOME/miniforge
             export PATH=\$CONDA/bin:\$PATH
             nvidia-smi
             $ROOT_DOCKER_FOLDER/.ci/setup.sh || exit -1

diff --git a/.github/workflows/linkchecker.yml b/.github/workflows/linkchecker.yml
@@ -27,7 +27,7 @@ jobs:
       - name: Setup and run tests
         run: |
           export BUILD_DIRECTORY="$GITHUB_WORKSPACE"
-          export CONDA=${HOME}/miniconda
+          export CONDA=${HOME}/miniforge
           export PATH=${CONDA}/bin:${HOME}/.local/bin:${PATH}
           $GITHUB_WORKSPACE/.ci/setup.sh || exit -1
           $GITHUB_WORKSPACE/.ci/test.sh || exit -1
diff --git a/.github/workflows/python_package.yml b/.github/workflows/python_package.yml
@@ -66,7 +66,7 @@ jobs:
           fi
           export BUILD_DIRECTORY="$GITHUB_WORKSPACE"
           export LGB_VER=$(head -n 1 VERSION.txt)
-          export CONDA=${HOME}/miniconda
+          export CONDA=${HOME}/miniforge
           export PATH=${CONDA}/bin:${PATH}
           $GITHUB_WORKSPACE/.ci/setup.sh || exit -1
           $GITHUB_WORKSPACE/.ci/test.sh || exit -1

diff --git a/.github/workflows/static_analysis.yml b/.github/workflows/static_analysis.yml
@@ -39,7 +39,7 @@ jobs:
         run: |
           export TASK="${{ matrix.task }}"
           export BUILD_DIRECTORY="$GITHUB_WORKSPACE"
-          export CONDA=${HOME}/miniconda
+          export CONDA=${HOME}/miniforge
           export PATH=${CONDA}/bin:$HOME/.local/bin:${PATH}
           $GITHUB_WORKSPACE/.ci/setup.sh || exit -1
           $GITHUB_WORKSPACE/.ci/test.sh || exit -1

diff --git a/.vsts-ci.yml b/.vsts-ci.yml
@@ -14,7 +14,7 @@ variables:
 resources:
   containers:
   - container: ubuntu1404
-    image: lightgbm/vsts-agent:ubuntu-14.04
+    image: lightgbm/vsts-agent:ubuntu-14.04-dev
   - container: ubuntu-latest
     image: 'ubuntu:latest'
     options: "--name ci-container -v /usr/bin/docker:/tmp/docker:ro"
@@ -49,6 +49,9 @@ jobs:
       gpu_source:
         TASK: gpu
         METHOD: source
+        # on Ubuntu 14.04, gpu_source build segfaults with newer version of Python
+        # (and newer version of scipy as a consequence)
+        PYTHON_VERSION: 3.7
       swig:
         TASK: swig
   steps:
@@ -122,7 +125,7 @@ jobs:
   - script: |
       echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
       echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
-      CONDA=$HOME/miniconda
+      CONDA=$HOME/miniforge
       echo "##vso[task.setvariable variable=CONDA]$CONDA"
       echo "##vso[task.prependpath]$CONDA/bin"
       AMDAPPSDK_PATH=$BUILD_SOURCESDIRECTORY/AMDAPPSDK
@@ -183,7 +186,7 @@ jobs:
       BUILD_ARTIFACTSTAGINGDIRECTORY=$BUILD_ARTIFACTSTAGINGDIRECTORY
       EOF
       cat > docker-script.sh <<EOF
-      export CONDA=\$HOME/miniconda
+      export CONDA=\$HOME/miniforge
       export PATH=\$CONDA/bin:\$PATH
       $ROOT_DOCKER_FOLDER/.ci/setup.sh || exit -1
       $ROOT_DOCKER_FOLDER/.ci/test.sh || exit -1
@@ -236,7 +239,7 @@ jobs:
   - script: |
       echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
       echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
-      CONDA=$AGENT_HOMEDIRECTORY/miniconda
+      CONDA=$AGENT_HOMEDIRECTORY/miniforge
       echo "##vso[task.setvariable variable=CONDA]$CONDA"
       echo "##vso[task.prependpath]$CONDA/bin"
       echo "##vso[task.setvariable variable=JAVA_HOME]$JAVA_HOME_8_X64"