From 2d4f3909e8fe1f03bced5afc2c6afcb81eb2e5b3 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 1 Apr 2020 18:38:34 +0100 Subject: [PATCH] [ci] reduce duplication in Python Windows CI (#2962) * [ci] reduced duplication in Python Windows CI * fixed typo * reorganized * fixed task name * Update .ci/test_windows.ps1 Co-Authored-By: Nikita Titov * changes from code review * added conda init powershell back * Update .ci/test_windows.ps1 Co-Authored-By: Nikita Titov * Switched to absolute paths * fixed env variables Co-authored-by: Nikita Titov --- .appveyor.yml | 45 +++++++++++++------------------------------- .ci/test_windows.ps1 | 32 ++++++++++++++++++++++++++++--- .vsts-ci.yml | 12 +++++++----- 3 files changed, 49 insertions(+), 40 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 990d36953dd1..f8cd85f17dcb 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -8,7 +8,9 @@ configuration: # a trick to construct a build matrix with multiple Python versi environment: matrix: - COMPILER: MSVC + TASK: python - COMPILER: MINGW + TASK: python clone_depth: 5 @@ -17,41 +19,20 @@ install: - set PATH=%PATH:C:\Program Files\Git\usr\bin;=% # delete sh.exe from PATH (mingw32-make fix) - set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH% - set PYTHON_VERSION=%CONFIGURATION% + - set CONDA_ENV="test-env" - ps: >- switch ($env:PYTHON_VERSION) { - "2.7" {$env:MINICONDA = """C:\Miniconda-x64"""} - "3.5" {$env:MINICONDA = """C:\Miniconda35-x64"""} - "3.6" {$env:MINICONDA = """C:\Miniconda36-x64"""} - "3.7" {$env:MINICONDA = """C:\Miniconda37-x64"""} - default {$env:MINICONDA = """C:\Miniconda37-x64"""} + "2.7" {$env:MINICONDA = "C:\Miniconda-x64"} + "3.5" {$env:MINICONDA = "C:\Miniconda35-x64"} + "3.6" {$env:MINICONDA = "C:\Miniconda36-x64"} + "3.7" {$env:MINICONDA = "C:\Miniconda37-x64"} + default {$env:MINICONDA = "C:\Miniconda37-x64"} } - - set PATH=%MINICONDA%;%MINICONDA%\Scripts;%PATH% - - ps: $env:LGB_VER = (Get-Content VERSION.txt).trim() - - activate - - conda config --set always_yes yes --set changeps1 no - - conda update -q -y conda - - conda create -q -y -n test-env python=%PYTHON_VERSION% joblib matplotlib numpy pandas psutil pytest python-graphviz "scikit-learn<=0.21.3" scipy - - activate test-env + $env:PATH="$env:MINICONDA;$env:MINICONDA\Scripts;$env:PATH" + - ps: $env:LGB_VER = (Get-Content $env:APPVEYOR_BUILD_FOLDER\VERSION.txt).trim() -build_script: - - cd %APPVEYOR_BUILD_FOLDER%\python-package - - IF "%COMPILER%"=="MINGW" ( - python setup.py install --mingw) - ELSE ( - python setup.py install) +build: false test_script: - - pytest %APPVEYOR_BUILD_FOLDER%\tests\python_package_test - - cd %APPVEYOR_BUILD_FOLDER%\examples\python-guide - - ps: >- - @("import matplotlib", "matplotlib.use('Agg')") + (Get-Content "plot_example.py") | Set-Content "plot_example.py" # prevent interactive window mode - (Get-Content "plot_example.py").replace('graph.render(view=True)', 'graph.render(view=False)') | Set-Content "plot_example.py" - - ps: >- - foreach ($file in @(Get-ChildItem *.py)) { - @("import sys, warnings", "warnings.showwarning = lambda message, category, filename, lineno, file=None, line=None: sys.stdout.write(warnings.formatwarning(message, category, filename, lineno, line))") + (Get-Content $file) | Set-Content $file - python $file - if (!$?) { $host.SetShouldExit(-1) } - } # run all examples - - cd %APPVEYOR_BUILD_FOLDER%\examples\python-guide\notebooks - - conda install -q -y -n test-env ipywidgets notebook - - jupyter nbconvert --ExecutePreprocessor.timeout=180 --to notebook --execute --inplace *.ipynb # run all notebooks + - conda init powershell + - powershell.exe -ExecutionPolicy Bypass -File %APPVEYOR_BUILD_FOLDER%\.ci\test_windows.ps1 diff --git a/.ci/test_windows.ps1 b/.ci/test_windows.ps1 index cf34af079268..1d6624fa344c 100644 --- a/.ci/test_windows.ps1 +++ b/.ci/test_windows.ps1 @@ -6,6 +6,20 @@ function Check-Output { } } +# unify environment variables for Azure devops and AppVeyor +if (Test-Path env:APPVEYOR) { + $env:APPVEYOR = "true" + $env:BUILD_SOURCESDIRECTORY = $env:APPVEYOR_BUILD_FOLDER +} + +# setup for Python +conda init powershell +conda activate +conda config --set always_yes yes --set changeps1 no +conda update -q -y conda +conda create -q -y -n $env:CONDA_ENV python=$env:PYTHON_VERSION joblib matplotlib numpy pandas psutil pytest python-graphviz "scikit-learn<=0.21.3" scipy ; Check-Output $? +conda activate $env:CONDA_ENV + if ($env:TASK -eq "regular") { mkdir $env:BUILD_SOURCESDIRECTORY/build; cd $env:BUILD_SOURCESDIRECTORY/build cmake -A x64 .. ; cmake --build . --target ALL_BUILD --config Release ; Check-Output $? @@ -33,15 +47,27 @@ elseif ($env:TASK -eq "bdist") { python setup.py bdist_wheel --plat-name=win-amd64 --universal ; Check-Output $? cd dist; pip install @(Get-ChildItem *.whl) ; Check-Output $? cp @(Get-ChildItem *.whl) $env:BUILD_ARTIFACTSTAGINGDIRECTORY +} elseif (($env:APPVEYOR -eq "true") -and ($env:TASK -eq "python")) { + cd $env:BUILD_SOURCESDIRECTORY\python-package + if ($env:COMPILER -eq "MINGW") { + python setup.py install --mingw ; Check-Output $? + } else { + python setup.py install ; Check-Output $? + } } -$tests = $env:BUILD_SOURCESDIRECTORY + $(If ($env:TASK -eq "sdist") {"/tests/python_package_test"} Else {"/tests"}) # cannot test C API with "sdist" task +if (($env:TASK -eq "sdist") -or (($env:APPVEYOR -eq "true") -and ($env:TASK -eq "python"))) { + $tests = $env:BUILD_SOURCESDIRECTORY + "/tests/python_package_test" +} else { + # cannot test C API with "sdist" task + $tests = $env:BUILD_SOURCESDIRECTORY + "/tests" +} pytest $tests ; Check-Output $? -if ($env:TASK -eq "regular") { +if (($env:TASK -eq "regular") -or (($env:APPVEYOR -eq "true") -and ($env:TASK -eq "python"))) { cd $env:BUILD_SOURCESDIRECTORY/examples/python-guide @("import matplotlib", "matplotlib.use('Agg')") + (Get-Content "plot_example.py") | Set-Content "plot_example.py" - (Get-Content "plot_example.py").replace('graph.render(view=True)', 'graph.render(view=False)') | Set-Content "plot_example.py" + (Get-Content "plot_example.py").replace('graph.render(view=True)', 'graph.render(view=False)') | Set-Content "plot_example.py" # prevent interactive window mode foreach ($file in @(Get-ChildItem *.py)) { @("import sys, warnings", "warnings.showwarning = lambda message, category, filename, lineno, file=None, line=None: sys.stdout.write(warnings.formatwarning(message, category, filename, lineno, line))") + (Get-Content $file) | Set-Content $file python $file ; Check-Output $? diff --git a/.vsts-ci.yml b/.vsts-ci.yml index e004b56d5500..74d8d0ee4f82 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -108,7 +108,7 @@ jobs: - task: PublishBuildArtifacts@1 condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))) inputs: - pathtoPublish: '$(Build.ArtifactStagingDirectory)' + pathtoPublish: '$(Build.ArtifactStagingDirectory)' artifactName: PackageAssets artifactType: container ########################################### @@ -129,11 +129,13 @@ jobs: TASK: bdist PYTHON_VERSION: 3.5 steps: - - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts" - displayName: Enable conda + - powershell: | + Write-Host "##vso[task.prependpath]$env:CONDA\Scripts" + Write-Host "##vso[task.setvariable variable=AZURE]true" + displayName: 'Set Variables' - script: | - cmd /c 'activate & conda config --set always_yes yes --set changeps1 no & conda update -q -y conda & conda create -q -y -n %CONDA_ENV% python=%PYTHON_VERSION% joblib matplotlib numpy pandas psutil pytest python-graphviz "scikit-learn<=0.21.3" scipy' - cmd /c "activate %CONDA_ENV% & powershell -ExecutionPolicy Bypass -File %BUILD_SOURCESDIRECTORY%/.ci/test_windows.ps1" + cmd /c "conda init powershell" + cmd /c "powershell -ExecutionPolicy Bypass -File %BUILD_SOURCESDIRECTORY%/.ci/test_windows.ps1" displayName: Test - task: PublishBuildArtifacts@1 condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))