Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[ci] reduce duplication in Python Windows CI #2962

Merged
merged 10 commits into from
Apr 1, 2020
Merged

[ci] reduce duplication in Python Windows CI #2962

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
137e68f
[ci] reduced duplication in Python Windows CI
jameslamb Mar 31, 2020
69c280a
fixed typo
jameslamb Mar 31, 2020
79c48ab
reorganized
jameslamb Mar 31, 2020
a331e2b
fixed task name
jameslamb Mar 31, 2020
eaac8bf
Update .ci/test_windows.ps1
jameslamb Apr 1, 2020
64ef6ac
changes from code review
jameslamb Apr 1, 2020
109e152
added conda init powershell back
jameslamb Apr 1, 2020
4a194da
Update .ci/test_windows.ps1
jameslamb Apr 1, 2020
70525bd
Switched to absolute paths
jameslamb Apr 1, 2020
6ee3bd1
fixed env variables
jameslamb Apr 1, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 13 additions & 31 deletions .appveyor.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,11 @@ configuration: # a trick to construct a build matrix with multiple Python versi
environment:
matrix:
- COMPILER: MSVC
TASK: python
CONDA_ENV: test-env
StrikerRUS marked this conversation as resolved.
Show resolved Hide resolved
- COMPILER: MINGW
TASK: python
CONDA_ENV: test-env

clone_depth: 5

Expand All @@ -19,39 +23,17 @@ install:
- set PYTHON_VERSION=%CONFIGURATION%
- ps: >-
switch ($env:PYTHON_VERSION) {
"2.7" {$env:MINICONDA = """C:\Miniconda-x64"""}
"3.5" {$env:MINICONDA = """C:\Miniconda35-x64"""}
"3.6" {$env:MINICONDA = """C:\Miniconda36-x64"""}
"3.7" {$env:MINICONDA = """C:\Miniconda37-x64"""}
default {$env:MINICONDA = """C:\Miniconda37-x64"""}
"2.7" {$env:MINICONDA = "C:\Miniconda-x64"}
"3.5" {$env:MINICONDA = "C:\Miniconda35-x64"}
"3.6" {$env:MINICONDA = "C:\Miniconda36-x64"}
"3.7" {$env:MINICONDA = "C:\Miniconda37-x64"}
default {$env:MINICONDA = "C:\Miniconda37-x64"}
}
- set PATH=%MINICONDA%;%MINICONDA%\Scripts;%PATH%
$env:PATH="$env:MINICONDA;$env:MINICONDA\Scripts;$env:PATH"
- ps: $env:LGB_VER = (Get-Content VERSION.txt).trim()
- activate
- conda config --set always_yes yes --set changeps1 no
- conda update -q -y conda
- conda create -q -y -n test-env python=%PYTHON_VERSION% joblib matplotlib numpy pandas psutil pytest python-graphviz "scikit-learn<=0.21.3" scipy
- activate test-env

build_script:
- cd %APPVEYOR_BUILD_FOLDER%\python-package
- IF "%COMPILER%"=="MINGW" (
python setup.py install --mingw)
ELSE (
python setup.py install)
build: false

test_script:
- pytest %APPVEYOR_BUILD_FOLDER%\tests\python_package_test
- cd %APPVEYOR_BUILD_FOLDER%\examples\python-guide
- ps: >-
@("import matplotlib", "matplotlib.use('Agg')") + (Get-Content "plot_example.py") | Set-Content "plot_example.py" # prevent interactive window mode
(Get-Content "plot_example.py").replace('graph.render(view=True)', 'graph.render(view=False)') | Set-Content "plot_example.py"
- ps: >-
foreach ($file in @(Get-ChildItem *.py)) {
@("import sys, warnings", "warnings.showwarning = lambda message, category, filename, lineno, file=None, line=None: sys.stdout.write(warnings.formatwarning(message, category, filename, lineno, line))") + (Get-Content $file) | Set-Content $file
python $file
if (!$?) { $host.SetShouldExit(-1) }
} # run all examples
- cd %APPVEYOR_BUILD_FOLDER%\examples\python-guide\notebooks
- conda install -q -y -n test-env ipywidgets notebook
- jupyter nbconvert --ExecutePreprocessor.timeout=180 --to notebook --execute --inplace *.ipynb # run all notebooks
- conda init powershell
- powershell.exe -ExecutionPolicy Bypass -File .\.ci\test_windows.ps1
StrikerRUS marked this conversation as resolved.
Show resolved Hide resolved
32 changes: 29 additions & 3 deletions .ci/test_windows.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,20 @@ function Check-Output {
}
}

# unify environment variables for Azure devops and AppVeyor
if (Test-Path env:APPVEYOR) {
$env:APPVEYOR = "true"
$env:BUILD_SOURCESDIRECTORY = $env:APPVEYOR_BUILD_FOLDER
}

# setup for Python
conda init powershell
conda activate
conda config --set always_yes yes --set changeps1 no
conda update -q -y conda
conda create -q -y -n $env:CONDA_ENV python=$env:PYTHON_VERSION joblib matplotlib numpy pandas psutil pytest python-graphviz "scikit-learn<=0.21.3" scipy wheel ; Check-Output $?
StrikerRUS marked this conversation as resolved.
Show resolved Hide resolved
conda activate $env:CONDA_ENV

if ($env:TASK -eq "regular") {
mkdir $env:BUILD_SOURCESDIRECTORY/build; cd $env:BUILD_SOURCESDIRECTORY/build
cmake -A x64 .. ; cmake --build . --target ALL_BUILD --config Release ; Check-Output $?
Expand Down Expand Up @@ -33,15 +47,27 @@ elseif ($env:TASK -eq "bdist") {
python setup.py bdist_wheel --plat-name=win-amd64 --universal ; Check-Output $?
cd dist; pip install @(Get-ChildItem *.whl) ; Check-Output $?
cp @(Get-ChildItem *.whl) $env:BUILD_ARTIFACTSTAGINGDIRECTORY
} elseif (($env:APPVEYOR -eq "true") -and ($env:TASK -eq "python")) {
cd $env:BUILD_SOURCESDIRECTORY\python-package
if ($env:COMPILER -eq "MINGW") {
python setup.py install --mingw | Check-Output $?
jameslamb marked this conversation as resolved.
Show resolved Hide resolved
} else {
python setup.py install | Check-Output $?
jameslamb marked this conversation as resolved.
Show resolved Hide resolved
}
}

$tests = $env:BUILD_SOURCESDIRECTORY + $(If ($env:TASK -eq "sdist") {"/tests/python_package_test"} Else {"/tests"}) # cannot test C API with "sdist" task
if (($env:TASK -eq "sdist") -or (($env:APPVEYOR -eq "true") -and ($env:TASK -eq "python"))) {
$tests = $env:BUILD_SOURCESDIRECTORY + "/tests/python_package_test"
} else {
# cannot test C API with "sdist" task
$tests = $env:BUILD_SOURCESDIRECTORY + "/tests"
}
pytest $tests ; Check-Output $?

if ($env:TASK -eq "regular") {
if (($env:TASK -eq "regular") -or (($env:APPVEYOR -eq "true") -and ($env:TASK -eq "python"))) {
cd $env:BUILD_SOURCESDIRECTORY/examples/python-guide
@("import matplotlib", "matplotlib.use('Agg')") + (Get-Content "plot_example.py") | Set-Content "plot_example.py"
(Get-Content "plot_example.py").replace('graph.render(view=True)', 'graph.render(view=False)') | Set-Content "plot_example.py"
(Get-Content "plot_example.py").replace('graph.render(view=True)', 'graph.render(view=False)') | Set-Content "plot_example.py" # prevent interactive window mode
foreach ($file in @(Get-ChildItem *.py)) {
@("import sys, warnings", "warnings.showwarning = lambda message, category, filename, lineno, file=None, line=None: sys.stdout.write(warnings.formatwarning(message, category, filename, lineno, line))") + (Get-Content $file) | Set-Content $file
python $file ; Check-Output $?
Expand Down
12 changes: 7 additions & 5 deletions .vsts-ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ jobs:
- task: PublishBuildArtifacts@1
condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
inputs:
pathtoPublish: '$(Build.ArtifactStagingDirectory)'
pathtoPublish: '$(Build.ArtifactStagingDirectory)'
artifactName: PackageAssets
artifactType: container
###########################################
Expand All @@ -129,11 +129,13 @@ jobs:
TASK: bdist
PYTHON_VERSION: 3.5
steps:
- powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
displayName: Enable conda
- powershell: |
Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
Write-Host "##vso[task.setvariable variable=AZURE]true"
StrikerRUS marked this conversation as resolved.
Show resolved Hide resolved
displayName: 'Set Variables'
- script: |
cmd /c 'activate & conda config --set always_yes yes --set changeps1 no & conda update -q -y conda & conda create -q -y -n %CONDA_ENV% python=%PYTHON_VERSION% joblib matplotlib numpy pandas psutil pytest python-graphviz "scikit-learn<=0.21.3" scipy'
cmd /c "activate %CONDA_ENV% & powershell -ExecutionPolicy Bypass -File %BUILD_SOURCESDIRECTORY%/.ci/test_windows.ps1"
cmd /c "conda init powershell"
StrikerRUS marked this conversation as resolved.
Show resolved Hide resolved
cmd /c "powershell -ExecutionPolicy Bypass -File %BUILD_SOURCESDIRECTORY%/.ci/test_windows.ps1"
displayName: Test
- task: PublishBuildArtifacts@1
condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
Expand Down