Extract astrometry installation into a script.

.travis.yml

@@ -1,38 +1,33 @@
+# Style note: we are using YAML's "literal style" block scalars for
+# commands that require multiple lines, such as if-then-else-fi
+# statements. For example:
+#
+#   - |
+#     if condition-1 ; then
+#       stmt-1
+#       stmt-2 || alt-stmt-2
+#     elif condition-2
+#     then
+#       stmt-3 && stmt-4
+#     fi
+#
+# For commands that don't require multiple lines, just use the single
+# line literal form, such as:
+#
+#   - install-some-software
+#   - PATH="$PWD/some-software/bin:$PATH"
+#
+# When debugging errors, it is easier to determine which line was
+# the source if Travis is executing them one at a time.
+
 dist: trusty
-sudo: required
+
+# TODO Does it matter which language we're using?
+# Can we use no language at all since we are installing
+# miniconda to get our choice of python version?
 language: python
-services:
-  - mongodb
-python:
-  - "3.6"
-env:
-    - PANDIR=$HOME POCS=$TRAVIS_BUILD_DIR PANUSER=$USER ARDUINO_VERSION=1.8.1
-before_install:
-    - mkdir -p $PANDIR/logs
-    - mkdir -p $PANDIR/astrometry/data
-    - ln -s $POCS $PANDIR/POCS
-    - pip install -U pip
-    - pip install coveralls
-
-    # Install cfitsio
-    - cd $PANDIR
-    - wget http://heasarc.gsfc.nasa.gov/FTP/software/fitsio/c/cfitsio_latest.tar.gz
-    - tar zxf cfitsio_latest.tar.gz
-    - cd cfitsio
-    - ./configure
-    - make
-    - make fpack
-    - make funpack
-    - sudo make install
-
-    # Install arudino files
-    - cd $PANDIR
-    - export DISPLAY=:1.0
-    - export
-    - wget http://downloads.arduino.cc/arduino-${ARDUINO_VERSION}-linux64.tar.xz
-    - tar xf arduino-${ARDUINO_VERSION}-linux64.tar.xz
-    - sudo mv arduino-${ARDUINO_VERSION} /usr/local/share/arduino
-    - sudo ln -s /usr/local/share/arduino/arduino /usr/local/bin/arduino
+
+# apt-get runs first during build.
 addons:
   apt:
     packages:
@@ -49,19 +44,112 @@ addons:
     - libbz2-dev
     - swig
     - cfitsio-dev
+
+# Services are started second.
+services:
+  - mongodb
+
+# After services are started, the git repo is cloned.
+
+python:
+  - "3.6"
+
+# Environment variables from the Travis repo config, then from .travis.yml, are exported.
+env:
+  - PANDIR=$HOME 
+    POCS=$TRAVIS_BUILD_DIR
+    PANUSER=$USER
+    ARDUINO_VERSION=1.8.7
+    ASTROMETRY_VERSION=0.76
+
+# Caches are restored next, if they are available. Note that it is
+# important to avoid modifying the cached directories during most
+# builds, else caching the directory is possibly a loss because we
+# have to restore it from cache at the beginning, possibly rebuild
+# it, then save it at the end.
+# For example, a problem with astroplan's caching of the IERS Bulletin A
+# is that it does so under conda's site-packages directory, so caching of
+# the miniconda directory is more expensive than it should be.
+
+cache:
+  pip: true
+  directories:
+    - $PANDIR/arduino/
+    - $PANDIR/astrometry/
+    - $PANDIR/cfitsio/
+
+# Before_install commands run after caches have been restored.
+before_install:
+  - echo PANDIR=$PANDIR
+  - echo POCS=$POCS
+  - echo PANUSER=$PANUSER
+  - echo ARDUINO_VERSION=$ARDUINO_VERSION
+  - mkdir -p $PANDIR/logs
+  - ln -s $POCS $PANDIR/POCS
+
+  # TODO Why install (with) pip when we're going to use Anaconda later?
+
+  - pip install -U pip
+  - pip install coveralls
+
 install:
   - cd $PANDIR
 
-  # Install astrometry.net
-  - wget http://astrometry.net/downloads/astrometry.net-0.72.tar.gz
-  - tar zxvf astrometry.net-0.72.tar.gz
-  - cd astrometry.net-0.72 && make && make py && make install INSTALL_DIR=$PANDIR/astrometry
-  - echo 'add_path $PANDIR/astrometry/data' | sudo tee --append $PANDIR/astrometry/etc/astrometry.cfg
+  # Install cfitsio, if not already cached.
+  # TODO Install a specific version so that we can better handle
+  # version changes.
+  - |
+    if [ -x $PANDIR/cfitsio/bin/fpack ] && [ -x $PANDIR/cfitsio/bin/funpack ] ; then
+      echo "Can reuse built cfitsio"
+    else
+      echo "Fetching and building cfitsio"
+      cd $PANDIR
+      wget http://heasarc.gsfc.nasa.gov/FTP/software/fitsio/c/cfitsio_latest.tar.gz
+      tar zxf cfitsio_latest.tar.gz
+      cd $PANDIR/cfitsio
+      ./configure
+      make
+      make fpack
+      make funpack
+      make install
+    fi
+  - find $PANDIR/cfitsio/bin -type f -executable -ls
+  - PATH="$PANDIR/cfitsio/bin:$PATH"
+  - cd $HOME
+  - which fpack
+  - which funpack
+
+  # Install Arduino tools, if not already cached.
+  - |
+    if [ -x $PANDIR/arduino/arduino-${ARDUINO_VERSION}/arduino ] ; then
+      echo "Can reuse cached arduino version."
+    else
+      echo "Removing any other cached arduino versions."
+      find $PANDIR/arduino -delete || /bin/true
+      mkdir -p $PANDIR/arduino
+      cd $PANDIR
+      wget http://downloads.arduino.cc/arduino-${ARDUINO_VERSION}-linux64.tar.xz
+      tar -xf arduino-${ARDUINO_VERSION}-linux64.tar.xz -C $PANDIR/arduino
+    fi
+  - export DISPLAY=:1.0
+  - PATH="$PANDIR/arduino/arduino-${ARDUINO_VERSION}:$PATH"
+  - which arduino
+  - arduino --version
+
+  # Install astrometry.net software.
+  - $POCS/scripts/travis-ci/install-astrometry.sh
+  - PATH="$PANDIR/astrometry/bin:$PATH"
+  - |
+    if [ -d $PANDIR/astrometry/data ] ; then
+      ls -l $PANDIR/astrometry/data
+    else
+      echo "Astrometry data is missing and will need to be fetched."
+    fi
 
   # Install miniconda
   - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh;
   - bash miniconda.sh -b -p $HOME/miniconda
-  - export PATH="$HOME/miniconda/bin:$HOME/cfitsio/bin:$PANDIR/astrometry/bin:$PATH"
+  - PATH="$HOME/miniconda/bin:$PATH"
   - hash -r
 
   - conda config --set always_yes yes --set changeps1 no
@@ -73,9 +161,9 @@ install:
   # install POCS and requirements
   - cd $POCS
   - pip install -r requirements.txt
-  - pip install -r docs/requirements.txt
   - pip install -e .
   - python pocs/utils/data.py --folder $PANDIR/astrometry/data
+
 script:
   - export BOARD="arduino:avr:micro"
   - arduino --verify --board $BOARD resources/arduino_files/camera_board/camera_board.ino
@@ -85,9 +173,11 @@ script:
   - export COVERAGE_PROCESS_START=.coveragerc
   - coverage run $(which pytest) -v --test-databases all
   - coverage combine
-cache:
-  pip: true
-  directories:
-    - $PANDIR/astrometry/
+
+# Travis detects whether any cached files changed, and if so packs them
+# up into a new archive for use in a future build.
+
 after_success:
   - bash <(curl -s https://codecov.io/bash)
+  - lscpu
+

requirements.txt

@@ -8,7 +8,10 @@ gcloud
 google-cloud-storage
 matplotlib >= 2.0.0,<3.0.0
 mocket
-numpy >= 1.12.1
+
+# numpy/numpy#12248 - it has been fixed and will be released in 1.15.4
+numpy >= 1.12.1, != 1.15.3
+
 pycodestyle == 2.3.1
 pymongo >= 3.2.2
 pyserial >= 3.1.1

scripts/travis-ci/install-astrometry.sh

@@ -0,0 +1,47 @@
+#!/bin/bash -ex
+
+ASTROMETRY_VERSION="${ASTROMETRY_VERSION:-0.72}"
+ASTROMETRY_DIR="astrometry.net-${ASTROMETRY_VERSION}"
+
+TAR_NAME="astrometry.net-${ASTROMETRY_VERSION}.tar.gz"
+TAR_URL="http://astrometry.net/downloads/${TAR_NAME}"
+
+ZIP_NAME="astrometry.net-${ASTROMETRY_VERSION}.zip"
+ZIP_URL="https://github.com/dstndstn/astrometry.net/archive/${ASTROMETRY_VERSION}.zip"
+
+DO_WGET="wget --dns-timeout=30 --connect-timeout=30 --read-timeout=60 --tries=1"
+
+if [ -x $PANDIR/astrometry/bin/solve-field ] ; then
+  echo "Astrometry has been cached:"
+  find $PANDIR/astrometry -type f -executable -ls
+  exit
+fi
+
+cd $PANDIR
+echo "Downloading ${TAR_URL} ..."
+$DO_WGET "${TAR_URL}" || /bin/true
+if [ -f "${TAR_NAME}" ] ; then
+  echo "Unpacking ${TAR_NAME} ..."
+  tar zxf "${TAR_NAME}"
+else
+  echo "Unable to download ${TAR_URL}"
+  echo "Downloading ${ZIP_URL} ..."
+  $DO_WGET --output-document="${ZIP_NAME}" "${ZIP_URL}"
+  echo "Unpacking ${ZIP_NAME} ..."
+  unzip "${ZIP_NAME}"
+fi
+
+echo "Building astrometry in ${ASTROMETRY_DIR}..."
+cd "${ASTROMETRY_DIR}"
+
+make
+make py
+find . -type f -executable -ls
+
+echo "Installing astrometry into $PANDIR/astrometry..."
+
+make install INSTALL_DIR=$PANDIR/astrometry
+find $PANDIR/astrometry -type f -executable -ls
+
+mkdir -p $PANDIR/astrometry/data
+echo 'add_path $PANDIR/astrometry/data' >> $PANDIR/astrometry/etc/astrometry.cfg