Skip to content

Commit

Permalink
CI: update appveyor to build conda packages as artifacts
Browse files Browse the repository at this point in the history
  • Loading branch information
jreback committed Dec 5, 2015
1 parent 925f62a commit 24fce07
Show file tree
Hide file tree
Showing 7 changed files with 268 additions and 39 deletions.
9 changes: 8 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,14 @@
<td>Build Status</td>
<td>
<a href="https://travis-ci.org/pydata/pandas">
<img src="https://travis-ci.org/pydata/pandas.svg?branch=master" alt="build status" />
<img src="https://travis-ci.org/pydata/pandas.svg?branch=master" alt="travis build status" />
</a>
</td>
</tr>
<td></td>
<td>
<a href="https://ci.appveyor.com/project/jreback/pandas-465">
<img src="https://ci.appveyor.com/api/projects/status/iblk29s98quexwxi/branch/master?svg=true" alt="appveyor build status" />
</a>
</td>
</tr>
Expand Down
67 changes: 52 additions & 15 deletions appveyor.yml
Original file line number Diff line number Diff line change
@@ -1,3 +1,13 @@
# With infos from
# http://tjelvarolsson.com/blog/how-to-continuously-test-your-python-code-on-windows-using-appveyor/
# https://packaging.python.org/en/latest/appveyor/
# https://github.com/rmcgibbo/python-appveyor-conda-example

# Backslashes in quotes need to be escaped: \ -> "\\"

matrix:
fast_finish: true # immediately finish build once one of the jobs fails.

environment:
global:
# SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the
Expand All @@ -6,33 +16,60 @@ environment:
CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\ci\\run_with_env.cmd"

matrix:
- PYTHON: "C:\\Python27_32"
PYTHON_VERSION: "2.7"
PYTHON_ARCH: "32"
- PYTHON: "C:\\Python35_64"
PYTHON_VERSION: "3.5"
PYTHON_ARCH: "64"
CONDA_PY: "35"
CONDA_NPY: "110"

- PYTHON: "C:\\Python27_64"
PYTHON_VERSION: "2.7"
PYTHON_ARCH: "64"
CONDA_PY: "27"
CONDA_NPY: "110"

- PYTHON: "C:\\Python34_32"
PYTHON_VERSION: "3.4"
PYTHON_ARCH: "32"
# We always use a 64-bit machine, but can build x86 distributions
# with the PYTHON_ARCH variable (which is used by CMD_IN_ENV).
platform:
- x64

- PYTHON: "C:\\Python34_64"
PYTHON_VERSION: "3.4"
PYTHON_ARCH: "64"
# all our python builds have to happen in tests_script...
build: false

init:
- "ECHO %PYTHON_VERSION% %PYTHON%"

install:
# this installs the appropriate Miniconda (Py2/Py3, 32/64 bit),
# as well as pip, conda-build, and the binstar CLI
- powershell .\ci\install.ps1
- SET PATH=%PYTHON%;%PYTHON%\Scripts;%PATH%
- echo "install"
- cd
- ls -ltr
- powershell .\\ci\\install_appveyor.ps1
- "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
- git tag --sort v:refname

build: false
# install our build environment
- cmd: conda config --set show_channel_urls yes --set always_yes yes --set changeps1 no
- cmd: conda update -q conda
- cmd: conda config --add channels http://conda.anaconda.org/pandas
- cmd: conda config --set ssl_verify false

# this is now the downloaded conda...
- conda info -a

# build em using the local source checkout in the correct windows env
- conda install conda-build
- cmd: '%CMD_IN_ENV% conda build conda.recipe -q --no-test'

# create our env
- SET REQ=ci\requirements-%PYTHON_VERSION%-%PYTHON_ARCH%.run
- cmd: conda create -q -n pandas python=%PYTHON_VERSION% nose
- cmd: activate pandas
- cmd: conda install -q --file=%REQ%
- ps: conda install -q (conda build conda.recipe -q --output --no-test)

test_script:
- "%CMD_IN_ENV% %PYTHON%/python.exe setup.py build_ext --inplace"
- "%PYTHON%/Scripts/nosetests -A \"not slow and not network and not disabled\" pandas"
# tests
- cd \
- conda list pandas
- nosetests --exe -A "not slow and not network and not disabled" pandas
96 changes: 96 additions & 0 deletions ci/install.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
# Sample script to install Miniconda under Windows
# Authors: Olivier Grisel, Jonathan Helmus and Kyle Kastner, Robert McGibbon
# License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/

$MINICONDA_URL = "http://repo.continuum.io/miniconda/"


function DownloadMiniconda ($python_version, $platform_suffix) {
$webclient = New-Object System.Net.WebClient
if ($python_version -match "3.4") {
$filename = "Miniconda3-latest-Windows-" + $platform_suffix + ".exe"
} else {
$filename = "Miniconda-latest-Windows-" + $platform_suffix + ".exe"
}
$url = $MINICONDA_URL + $filename

$basedir = $pwd.Path + "\"
$filepath = $basedir + $filename
if (Test-Path $filename) {
Write-Host "Reusing" $filepath
return $filepath
}

# Download and retry up to 3 times in case of network transient errors.
Write-Host "Downloading" $filename "from" $url
$retry_attempts = 2
for($i=0; $i -lt $retry_attempts; $i++){
try {
$webclient.DownloadFile($url, $filepath)
break
}
Catch [Exception]{
Start-Sleep 1
}
}
if (Test-Path $filepath) {
Write-Host "File saved at" $filepath
} else {
# Retry once to get the error message if any at the last try
$webclient.DownloadFile($url, $filepath)
}
return $filepath
}


function InstallMiniconda ($python_version, $architecture, $python_home) {
Write-Host "Installing Python" $python_version "for" $architecture "bit architecture to" $python_home
if (Test-Path $python_home) {
Write-Host $python_home "already exists, skipping."
return $false
}
if ($architecture -match "32") {
$platform_suffix = "x86"
} else {
$platform_suffix = "x86_64"
}

$filepath = DownloadMiniconda $python_version $platform_suffix
Write-Host "Installing" $filepath "to" $python_home
$install_log = $python_home + ".log"
$args = "/S /D=$python_home"
Write-Host $filepath $args
Start-Process -FilePath $filepath -ArgumentList $args -Wait -Passthru
if (Test-Path $python_home) {
Write-Host "Python $python_version ($architecture) installation complete"
} else {
Write-Host "Failed to install Python in $python_home"
Get-Content -Path $install_log
Exit 1
}
}


function InstallCondaPackages ($python_home, $spec) {
$conda_path = $python_home + "\Scripts\conda.exe"
$args = "install --yes " + $spec
Write-Host ("conda " + $args)
Start-Process -FilePath "$conda_path" -ArgumentList $args -Wait -Passthru
}

function UpdateConda ($python_home) {
$conda_path = $python_home + "\Scripts\conda.exe"
Write-Host "Updating conda..."
$args = "update --yes conda"
Write-Host $conda_path $args
Start-Process -FilePath "$conda_path" -ArgumentList $args -Wait -Passthru
}


function main () {
InstallMiniconda $env:PYTHON_VERSION $env:PYTHON_ARCH $env:PYTHON
UpdateConda $env:PYTHON
InstallCondaPackages $env:PYTHON "conda-build jinja2 anaconda-client"
}

main
22 changes: 22 additions & 0 deletions ci/requirements-2.7-64.run
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
dateutil
pytz
numpy
xlwt
numexpr
pytables
matplotlib
openpyxl
xlrd
sqlalchemy
lxml=3.2.1
scipy
xlsxwriter
boto
bottleneck
patsy
html5lib
beautiful-soup
jinja2=2.8

#pymysql=0.6.3
#psycopg2=2.5.2
25 changes: 25 additions & 0 deletions ci/requirements-3.5-64.run
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
python-dateutil
pytz
numpy
openpyxl
xlsxwriter
xlrd
xlwt
patsy
scipy
numexpr
pytables
html5lib
lxml
matplotlib
jinja2
blosc

# currently causing some warnings
#sqlalchemy
#pymysql
#psycopg2

# not available from conda
#beautiful-soup
#bottleneck
80 changes: 64 additions & 16 deletions ci/run_with_env.cmd
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
:: EXPECTED ENV VARS: PYTHON_ARCH (either x86 or x64)
:: CONDA_PY (either 27, 33, 35 etc. - only major version is extracted)
::
::
:: To build extensions for 64 bit Python 3, we need to configure environment
:: variables to use the MSVC 2010 C++ compilers from GRMSDKX_EN_DVD.iso of:
:: MS Windows SDK for Windows 7 and .NET Framework 4 (SDK v7.1)
Expand All @@ -6,7 +10,8 @@
:: variables to use the MSVC 2008 C++ compilers from GRMSDKX_EN_DVD.iso of:
:: MS Windows SDK for Windows 7 and .NET Framework 3.5 (SDK v7.0)
::
:: 32 bit builds do not require specific environment configurations.
:: 32 bit builds, and 64-bit builds for 3.5 and beyond, do not require specific
:: environment configurations.
::
:: Note: this script needs to be run with the /E:ON and /V:ON flags for the
:: cmd interpreter, at least for (SDK v7.0)
Expand All @@ -15,33 +20,76 @@
:: https://github.com/cython/cython/wiki/64BitCythonExtensionsOnWindows
:: http://stackoverflow.com/a/13751649/163740
::
:: Author: Olivier Grisel
:: Author: Phil Elson
:: Original Author: Olivier Grisel (https://github.com/ogrisel/python-appveyor-demo)
:: License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/
::
:: Notes about batch files for Python people:
::
:: Quotes in values are literally part of the values:
:: SET FOO="bar"
:: FOO is now five characters long: " b a r "
:: If you don't want quotes, don't include them on the right-hand side.
::
:: The CALL lines at the end of this file look redundant, but if you move them
:: outside of the IF clauses, they do not run properly in the SET_SDK_64==Y
:: case, I don't know why.
:: originally from https://github.com/pelson/Obvious-CI/blob/master/scripts/obvci_appveyor_python_build_env.cmd
@ECHO OFF

SET COMMAND_TO_RUN=%*
SET WIN_SDK_ROOT=C:\Program Files\Microsoft SDKs\Windows

SET MAJOR_PYTHON_VERSION="%PYTHON_VERSION:~0,1%"
IF %MAJOR_PYTHON_VERSION% == "2" (
:: Extract the major and minor versions, and allow for the minor version to be
:: more than 9. This requires the version number to have two dots in it.
SET MAJOR_PYTHON_VERSION=%CONDA_PY:~0,1%

IF "%CONDA_PY:~2,1%" == "" (
:: CONDA_PY style, such as 27, 34 etc.
SET MINOR_PYTHON_VERSION=%CONDA_PY:~1,1%
) ELSE (
IF "%CONDA_PY:~3,1%" == "." (
SET MINOR_PYTHON_VERSION=%CONDA_PY:~2,1%
) ELSE (
SET MINOR_PYTHON_VERSION=%CONDA_PY:~2,2%
)
)

:: Based on the Python version, determine what SDK version to use, and whether
:: to set the SDK for 64-bit.
IF %MAJOR_PYTHON_VERSION% == 2 (
SET WINDOWS_SDK_VERSION="v7.0"
) ELSE IF %MAJOR_PYTHON_VERSION% == "3" (
SET WINDOWS_SDK_VERSION="v7.1"
SET SET_SDK_64=Y
) ELSE (
ECHO Unsupported Python version: "%MAJOR_PYTHON_VERSION%"
EXIT 1
IF %MAJOR_PYTHON_VERSION% == 3 (
SET WINDOWS_SDK_VERSION="v7.1"
IF %MINOR_PYTHON_VERSION% LEQ 4 (
SET SET_SDK_64=Y
) ELSE (
SET SET_SDK_64=N
)
) ELSE (
ECHO Unsupported Python version: "%MAJOR_PYTHON_VERSION%"
EXIT /B 1
)
)

IF "%PYTHON_ARCH%"=="64" (
ECHO Configuring Windows SDK %WINDOWS_SDK_VERSION% for Python %MAJOR_PYTHON_VERSION% on a 64 bit architecture
SET DISTUTILS_USE_SDK=1
SET MSSdk=1
"%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Setup\WindowsSdkVer.exe" -q -version:%WINDOWS_SDK_VERSION%
"%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Bin\SetEnv.cmd" /x64 /release
ECHO Executing: %COMMAND_TO_RUN%
call %COMMAND_TO_RUN% || EXIT 1
IF %SET_SDK_64% == Y (
ECHO Configuring Windows SDK %WINDOWS_SDK_VERSION% for Python %MAJOR_PYTHON_VERSION% on a 64 bit architecture
SET DISTUTILS_USE_SDK=1
SET MSSdk=1
"%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Setup\WindowsSdkVer.exe" -q -version:%WINDOWS_SDK_VERSION%
"%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Bin\SetEnv.cmd" /x64 /release
ECHO Executing: %COMMAND_TO_RUN%
call %COMMAND_TO_RUN% || EXIT /B 1
) ELSE (
ECHO Using default MSVC build environment for 64 bit architecture
ECHO Executing: %COMMAND_TO_RUN%
call %COMMAND_TO_RUN% || EXIT /B 1
)
) ELSE (
ECHO Using default MSVC build environment for 32 bit architecture
ECHO Executing: %COMMAND_TO_RUN%
call %COMMAND_TO_RUN% || EXIT 1
call %COMMAND_TO_RUN% || EXIT /B 1
)
8 changes: 1 addition & 7 deletions conda.recipe/meta.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ build:
number: {{ environ.get('GIT_DESCRIBE_NUMBER', 0) }}

source:
git_url: ../
path: ../../

requirements:
build:
Expand All @@ -28,12 +28,6 @@ test:
imports:
- pandas

#requires:
# - nose

#commands:
# - nosetests --exe -A "not slow and not network and not disabled" pandas

about:
home: http://pandas.pydata.org
license: BSD

0 comments on commit 24fce07

Please sign in to comment.