[ci] reduce duplication in Python Windows CI (#2962)

* [ci] reduced duplication in Python Windows CI * fixed typo * reorganized * fixed task name * Update .ci/test_windows.ps1 Co-Authored-By: Nikita Titov <[email protected]> * changes from code review * added conda init powershell back * Update .ci/test_windows.ps1 Co-Authored-By: Nikita Titov <[email protected]> * Switched to absolute paths * fixed env variables Co-authored-by: Nikita Titov <[email protected]>
microsoft · Apr 1, 2020 · 2d4f390 · 2d4f390
1 parent 84bdf25
commit 2d4f390
Show file tree

Hide file tree

Showing 3 changed files with 49 additions and 40 deletions.
diff --git a/.appveyor.yml b/.appveyor.yml
@@ -8,7 +8,9 @@ configuration:  # a trick to construct a build matrix with multiple Python versi
 environment:
   matrix:
     - COMPILER: MSVC
+      TASK: python
     - COMPILER: MINGW
+      TASK: python
 
 clone_depth: 5
 
@@ -17,41 +19,20 @@ install:
   - set PATH=%PATH:C:\Program Files\Git\usr\bin;=%  # delete sh.exe from PATH (mingw32-make fix)
   - set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH%
   - set PYTHON_VERSION=%CONFIGURATION%
+  - set CONDA_ENV="test-env"
   - ps: >-
       switch ($env:PYTHON_VERSION) {
-          "2.7" {$env:MINICONDA = """C:\Miniconda-x64"""}
-          "3.5" {$env:MINICONDA = """C:\Miniconda35-x64"""}
-          "3.6" {$env:MINICONDA = """C:\Miniconda36-x64"""}
-          "3.7" {$env:MINICONDA = """C:\Miniconda37-x64"""}
-          default {$env:MINICONDA = """C:\Miniconda37-x64"""}
+          "2.7" {$env:MINICONDA = "C:\Miniconda-x64"}
+          "3.5" {$env:MINICONDA = "C:\Miniconda35-x64"}
+          "3.6" {$env:MINICONDA = "C:\Miniconda36-x64"}
+          "3.7" {$env:MINICONDA = "C:\Miniconda37-x64"}
+          default {$env:MINICONDA = "C:\Miniconda37-x64"}
       }
-  - set PATH=%MINICONDA%;%MINICONDA%\Scripts;%PATH%
-  - ps: $env:LGB_VER = (Get-Content VERSION.txt).trim()
-  - activate
-  - conda config --set always_yes yes --set changeps1 no
-  - conda update -q -y conda
-  - conda create -q -y -n test-env python=%PYTHON_VERSION% joblib matplotlib numpy pandas psutil pytest python-graphviz "scikit-learn<=0.21.3" scipy
-  - activate test-env
+      $env:PATH="$env:MINICONDA;$env:MINICONDA\Scripts;$env:PATH"
+  - ps: $env:LGB_VER = (Get-Content $env:APPVEYOR_BUILD_FOLDER\VERSION.txt).trim()
 
-build_script:
-  - cd %APPVEYOR_BUILD_FOLDER%\python-package
-  - IF "%COMPILER%"=="MINGW" (
-    python setup.py install --mingw)
-    ELSE (
-    python setup.py install)
+build: false
 
 test_script:
-  - pytest %APPVEYOR_BUILD_FOLDER%\tests\python_package_test
-  - cd %APPVEYOR_BUILD_FOLDER%\examples\python-guide
-  - ps: >-
-      @("import matplotlib", "matplotlib.use('Agg')") + (Get-Content "plot_example.py") | Set-Content "plot_example.py"  # prevent interactive window mode
-      (Get-Content "plot_example.py").replace('graph.render(view=True)', 'graph.render(view=False)') | Set-Content "plot_example.py"
-  - ps: >-
-      foreach ($file in @(Get-ChildItem *.py)) {
-        @("import sys, warnings", "warnings.showwarning = lambda message, category, filename, lineno, file=None, line=None: sys.stdout.write(warnings.formatwarning(message, category, filename, lineno, line))") + (Get-Content $file) | Set-Content $file
-        python $file
-        if (!$?) { $host.SetShouldExit(-1) }
-      }  # run all examples
-  - cd %APPVEYOR_BUILD_FOLDER%\examples\python-guide\notebooks
-  - conda install -q -y -n test-env ipywidgets notebook
-  - jupyter nbconvert --ExecutePreprocessor.timeout=180 --to notebook --execute --inplace *.ipynb  # run all notebooks
+  - conda init powershell
+  - powershell.exe -ExecutionPolicy Bypass -File %APPVEYOR_BUILD_FOLDER%\.ci\test_windows.ps1
diff --git a/.ci/test_windows.ps1 b/.ci/test_windows.ps1
@@ -6,6 +6,20 @@ function Check-Output {
   }
 }
 
+# unify environment variables for Azure devops and AppVeyor
+if (Test-Path env:APPVEYOR) {
+  $env:APPVEYOR = "true"
+  $env:BUILD_SOURCESDIRECTORY = $env:APPVEYOR_BUILD_FOLDER
+}
+
+# setup for Python
+conda init powershell
+conda activate
+conda config --set always_yes yes --set changeps1 no
+conda update -q -y conda
+conda create -q -y -n $env:CONDA_ENV python=$env:PYTHON_VERSION joblib matplotlib numpy pandas psutil pytest python-graphviz "scikit-learn<=0.21.3" scipy ; Check-Output $?
+conda activate $env:CONDA_ENV
+
 if ($env:TASK -eq "regular") {
   mkdir $env:BUILD_SOURCESDIRECTORY/build; cd $env:BUILD_SOURCESDIRECTORY/build
   cmake -A x64 .. ; cmake --build . --target ALL_BUILD --config Release ; Check-Output $?
@@ -33,15 +47,27 @@ elseif ($env:TASK -eq "bdist") {
   python setup.py bdist_wheel --plat-name=win-amd64 --universal ; Check-Output $?
   cd dist; pip install @(Get-ChildItem *.whl) ; Check-Output $?
   cp @(Get-ChildItem *.whl) $env:BUILD_ARTIFACTSTAGINGDIRECTORY
+} elseif (($env:APPVEYOR -eq "true") -and ($env:TASK -eq "python")) {
+  cd $env:BUILD_SOURCESDIRECTORY\python-package
+  if ($env:COMPILER -eq "MINGW") {
+    python setup.py install --mingw ; Check-Output $?
+  } else {
+    python setup.py install ; Check-Output $?
+  }
 }
 
-$tests = $env:BUILD_SOURCESDIRECTORY + $(If ($env:TASK -eq "sdist") {"/tests/python_package_test"} Else {"/tests"})  # cannot test C API with "sdist" task
+if (($env:TASK -eq "sdist") -or (($env:APPVEYOR -eq "true") -and ($env:TASK -eq "python"))) {
+  $tests = $env:BUILD_SOURCESDIRECTORY + "/tests/python_package_test"
+} else {
+  # cannot test C API with "sdist" task
+  $tests = $env:BUILD_SOURCESDIRECTORY + "/tests"
+}
 pytest $tests ; Check-Output $?
 
-if ($env:TASK -eq "regular") {
+if (($env:TASK -eq "regular") -or (($env:APPVEYOR -eq "true") -and ($env:TASK -eq "python"))) {
   cd $env:BUILD_SOURCESDIRECTORY/examples/python-guide
   @("import matplotlib", "matplotlib.use('Agg')") + (Get-Content "plot_example.py") | Set-Content "plot_example.py"
-  (Get-Content "plot_example.py").replace('graph.render(view=True)', 'graph.render(view=False)') | Set-Content "plot_example.py"
+  (Get-Content "plot_example.py").replace('graph.render(view=True)', 'graph.render(view=False)') | Set-Content "plot_example.py"  # prevent interactive window mode
   foreach ($file in @(Get-ChildItem *.py)) {
     @("import sys, warnings", "warnings.showwarning = lambda message, category, filename, lineno, file=None, line=None: sys.stdout.write(warnings.formatwarning(message, category, filename, lineno, line))") + (Get-Content $file) | Set-Content $file
     python $file ; Check-Output $?

diff --git a/.vsts-ci.yml b/.vsts-ci.yml
@@ -108,7 +108,7 @@ jobs:
   - task: PublishBuildArtifacts@1
     condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
     inputs:
-      pathtoPublish: '$(Build.ArtifactStagingDirectory)' 
+      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
       artifactName: PackageAssets
       artifactType: container
 ###########################################
@@ -129,11 +129,13 @@ jobs:
         TASK: bdist
         PYTHON_VERSION: 3.5
   steps:
-  - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
-    displayName: Enable conda
+  - powershell: |
+      Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
+      Write-Host "##vso[task.setvariable variable=AZURE]true"
+    displayName: 'Set Variables'
   - script: |
-      cmd /c 'activate & conda config --set always_yes yes --set changeps1 no & conda update -q -y conda & conda create -q -y -n %CONDA_ENV% python=%PYTHON_VERSION% joblib matplotlib numpy pandas psutil pytest python-graphviz "scikit-learn<=0.21.3" scipy'
-      cmd /c "activate %CONDA_ENV% & powershell -ExecutionPolicy Bypass -File %BUILD_SOURCESDIRECTORY%/.ci/test_windows.ps1"
+      cmd /c "conda init powershell"
+      cmd /c "powershell -ExecutionPolicy Bypass -File %BUILD_SOURCESDIRECTORY%/.ci/test_windows.ps1"
     displayName: Test
   - task: PublishBuildArtifacts@1
     condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))