automated build tweaks

README.md

@@ -28,6 +28,15 @@ To run the Docker script you need to have two files in your `recipes` directory:
 and password. The build scripts will authenticate with these, but will require a
 manual okay. This is the only non-automated step in the build and upload process.
 
+In some cases, building the docker container can take more time than building
+the conda package. If so, you can build a container locally and use that for
+building instead of the centos:centos5 container. Use the
+`update_binstar_packages_docker_prebuilt.sh` script when using this strategy:
+
+    (cd docker && docker build -t bioconda-builder .)
+    docker run --net=host --rm=true -i -t -v `pwd`:/tmp/conda-recipes bioconda-builder /bin/bash /tmp/conda-recipes/update_binstar_packages_docker.sh your_package
+
+
 ## OSX
 
 For packages that build on OSX, run:

docker/Dockerfile

@@ -0,0 +1,9 @@
+FROM centos:centos5
+RUN rpm -Uvh http://dl.fedoraproject.org/pub/epel/5/x86_64/epel-release-5-4.noarch.rpm
+RUN yum install -y wget bzip2 bzip2-devel git gcc gcc-c++ patch zlib-devel make gcc44 gcc44-c++ cmake ncurses-devel
+RUN wget http://people.centos.org/tru/devtools-2/devtools-2.repo -O /etc/yum.repos.d/devtools-2.repo
+RUN yum install -y devtoolset-2-gcc devtoolset-2-binutils devtoolset-2-gcc-c++
+RUN mkdir -p /tmp/conda-build
+RUN wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh && bash Miniconda-latest-Linux-x86_64.sh -b -p /tmp/conda-build/anaconda
+ENV PATH=/tmp/conda-build/anaconda/bin:$PATH
+RUN conda install -y conda conda-build anaconda-client pyyaml toolz jinja2

update_binstar_packages.py

@@ -6,6 +6,7 @@
 import re
 import subprocess
 import sys
+import argparse
 
 import toolz as tz
 import yaml
@@ -14,14 +15,20 @@
 CONFIG = {
     "remote_user": "bioconda",
     "remote_repo": "bioconda",
-    #"targets": ["linux-64", "linux-32", "osx-64"],
-    "targets": ["linux-64", "osx-64"],
     "numpy": "19"}
 
+# Add targets depending on detected current OS
+if sys.platform == "darwin":
+    CONFIG['targets'] = ['osx-64']
+elif sys.platform in ['linux', 'linux2']:
+    CONFIG['targets'] = ['linux-64']
+else:
+    raise ValueError("%s not in [linux, linux2, darwin]: can't determine OS" % sys.platform)
+
 CUSTOM_TARGETS = {
     "hap.py": ["linux-64"]}
 
-def main(pkgs):
+def main(pkgs, force_build=True):
     config = CONFIG
     subprocess.check_call(["conda", "config", "--add", "channels", config["remote_repo"]])
     if not pkgs:
@@ -31,7 +38,8 @@ def main(pkgs):
         if os.path.exists(meta_file):
             version, build = _meta_to_version(meta_file)
             needs_build = _needs_upload(name, version, build, config)
-            if needs_build:
+            print("Need to build these packages: {0}".format(needs_build))
+            if needs_build or force_build:
                 _build_and_upload(name, needs_build, config, _should_convert(os.path.join(name, "build.sh")))
 
 def _build_and_upload(name, platforms, config, do_convert):
@@ -111,4 +119,8 @@ def _should_convert(build_file):
     return False
 
 if __name__ == "__main__":
-    main(sys.argv[1:])
+    ap = argparse.ArgumentParser()
+    ap.add_argument('packages', nargs="+", help="Package to build")
+    ap.add_argument('--force-build', action='store_true', help='Force local build even though the latest version already exists on binstar')
+    args = ap.parse_args()
+    main(args.packages, force_build=args.force_build)

update_binstar_packages_docker_prebuilt.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -eu -o pipefail
+
+# enable gcc-4.8
+export MANPATH=""
+source /opt/rh/devtoolset-2/enable
+cd /tmp/conda-recipes
+anaconda login --hostname bcbio-conda-auto --username `cat anaconda-user.txt` --password `cat anaconda-pass.txt`
+cd /tmp/conda-recipes
+export PATH=/tmp/conda-build/anaconda/bin:$PATH
+/tmp/conda-build/anaconda/bin/python update_binstar_packages.py $*