From 11d26a122eae2166d25deec8992cefb03f3f485d Mon Sep 17 00:00:00 2001 From: jamessynge Date: Sun, 19 Nov 2017 17:23:41 -0500 Subject: [PATCH 01/15] wip - Starting on creating some setup scripts. Need to coordinate with Wilfred who is doing a similar exercise with the Raspberry Pi. --- scripts/apt-packages-list.txt | 45 ++++++++++++++ scripts/install-dependencies.sh | 103 ++++++++++++++++++++++++++++++++ 2 files changed, 148 insertions(+) create mode 100644 scripts/apt-packages-list.txt create mode 100755 scripts/install-dependencies.sh diff --git a/scripts/apt-packages-list.txt b/scripts/apt-packages-list.txt new file mode 100644 index 000000000..cb6066c7c --- /dev/null +++ b/scripts/apt-packages-list.txt @@ -0,0 +1,45 @@ +# This file contains a list of apt packages to install. +# Lines starting with # are comments. +# astrometry.net # No, will install from source into /var/panoptes/astrometry. +# libcfitsio3-dev # Do we need this, or should we build it as in .travis.yml +# And if we should build it, should it be in /var/pantoptes or in $HOME? +gphoto2 +libbz2-dev +libcairo2-dev +libjpeg-dev +libnetpbm10-dev +libpng12-dev +netpbm +python-dev +python-numpy +python-pyfits +swig +zlib1g-dev +# These additional packages may or may not be needed, I simply installed them on +# the NUC for PAN006 and need to determine if they are needed. +dcraw + +# Is exiftool needed? It gets converted to libimage-exiftool-perl, which I'm +# guessing we don't need. +# exiftool + +graphviz +iw +libgeos-3.* +libgeos-c1v5 +libgeos-dev +libgraphviz-dev +libncurses5-dev +lm-sensors +mongodb +openssh-client +openssh-server +pkg-config +python-psutil + +# I use tmux (in lieu of screen) to start the various Panoptes programs at +# system startup. +tmux + +xbase-clients + diff --git a/scripts/install-dependencies.sh b/scripts/install-dependencies.sh new file mode 100755 index 000000000..3e48e8c74 --- /dev/null +++ b/scripts/install-dependencies.sh @@ -0,0 +1,103 @@ +#!/bin/bash -e + +if [[ -z "${PANDIR}" || -z "${POCS}" || -z "${PAWS}" || -z "${PANLOG}" || + -z "${PANUSER}" || ! -d "${PANDIR}" ]] ; then + echo "Please set the Panoptes environment variables, then re-run this script." + exit 1 +fi + +THIS_DIR="$(dirname $(readlink -f "${0}"))" +APT_PKGS="$(egrep -v '^#|^ *$' "${THIS_DIR}/apt-packages-list.txt"|sort|uniq)" + +cd "${PANDIR}" +mkdir -p tmp + +echo +echo "Running sudo apt-get install, you may be prompted for your password." +echo + +(set -x ; sudo apt-get install --yes ${APT_PKGS}) + +exit + +# If conda is not present, install latest version of Miniconda (Anaconda with +# fewer packages; any that are needed can then be installed). +set -x +CONDA_BIN=$(which conda) # Assuming is for python3 if found. +if [[ -z "${CONDA_BIN}" ]] ; then + wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O tmp/miniconda.sh + bash miniconda.sh -b -p "${PANDIR}/miniconda" +fi + +exit + +dist: trusty +sudo: required +language: python +services: + - mongodb +python: + - "3.6" +cache: + - pip +env: + - POCS=$TRAVIS_BUILD_DIR PANDIR=/var/panoptes +before_install: + - sudo mkdir /var/panoptes && sudo chmod 777 /var/panoptes + - mkdir $PANDIR/logs + - ln -s $TRAVIS_BUILD_DIR /var/panoptes/POCS + - pip install coveralls + - pip install -U pip + - cd $HOME + - wget http://heasarc.gsfc.nasa.gov/FTP/software/fitsio/c/cfitsio_latest.tar.gz + - tar zxf cfitsio_latest.tar.gz + - cd cfitsio + - ./configure + - make + - make fpack + - make funpack + - sudo make install + - sudo mkdir -p /var/panoptes/astrometry/data + - sudo chmod -R 777 /var/panoptes/astrometry/ +addons: + apt: + packages: + - gphoto2 + - libcairo2-dev + - libnetpbm10-dev + - netpbm + - libpng12-dev + - libjpeg-dev + - python-numpy + - python-pyfits + - python-dev + - zlib1g-dev + - libbz2-dev + - swig + - cfitsio-dev +install: + - wget http://astrometry.net/downloads/astrometry.net-0.72.tar.gz + - tar zxvf astrometry.net-0.72.tar.gz + - cd astrometry.net-0.72 && make && make py && make install INSTALL_DIR=/var/panoptes/astrometry + - echo 'add_path /var/panoptes/astrometry/data' | sudo tee --append /var/panoptes/astrometry/etc/astrometry.cfg + - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; + - bash miniconda.sh -b -p $HOME/miniconda + - export PATH="$HOME/miniconda/bin:$HOME/cfitsio/bin:/var/panoptes/astrometry/bin:$PATH" + - hash -r + + - conda config --set always_yes yes --set changeps1 no + - conda update -q conda + - conda info -a # Useful for debugging any issues with conda + - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION + - source activate test-environment + + - cd $TRAVIS_BUILD_DIR + - pip install -r requirements.txt + - python setup.py install +script: + - coverage run setup.py test + - coverage combine .coverage* +after_success: + - if [[ $TRAVIS_PYTHON_VERSION == 3.6* ]]; then + bash <(curl -s https://codecov.io/bash); + fi From 8dc2835b6dce5dab678232c802251bc4ff62e525 Mon Sep 17 00:00:00 2001 From: jamessynge Date: Sun, 19 Nov 2017 21:27:36 -0500 Subject: [PATCH 02/15] wip - Install cfitsio. --- requirements.txt | 4 ++- scripts/install-dependencies.sh | 46 ++++++++++++++++++++++++++++++--- 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index d7a63b628..5e9c54577 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,4 +22,6 @@ codecov ffmpy pygraphviz google-cloud-storage -dateparser \ No newline at end of file +dateparser +coveralls + diff --git a/scripts/install-dependencies.sh b/scripts/install-dependencies.sh index 3e48e8c74..361c2695a 100755 --- a/scripts/install-dependencies.sh +++ b/scripts/install-dependencies.sh @@ -1,5 +1,7 @@ #!/bin/bash -e +# TODO(jamessynge): Add flags to control behavior, such as skipping apt-get. + if [[ -z "${PANDIR}" || -z "${POCS}" || -z "${PAWS}" || -z "${PANLOG}" || -z "${PANUSER}" || ! -d "${PANDIR}" ]] ; then echo "Please set the Panoptes environment variables, then re-run this script." @@ -18,17 +20,55 @@ echo (set -x ; sudo apt-get install --yes ${APT_PKGS}) -exit +add_to_PATH() { + local -r the_dir="$(readlink -f "${1}")" + echo >>"${HOME}/.bashrc" " +# Added by PANOPTES install-dependencies.sh +PATH=\"${the_dir}:\${PATH}\"" + PATH="${the_dir}:${PATH}" +} # If conda is not present, install latest version of Miniconda (Anaconda with # fewer packages; any that are needed can then be installed). set -x CONDA_BIN=$(which conda) # Assuming is for python3 if found. if [[ -z "${CONDA_BIN}" ]] ; then - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O tmp/miniconda.sh - bash miniconda.sh -b -p "${PANDIR}/miniconda" + wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \ + -O tmp/miniconda.sh + bash tmp/miniconda.sh -b -p "${PANDIR}/miniconda" + rm tmp/miniconda.sh + add_to_PATH miniconda/bin +fi + +if [[ ! -d cfitsio ]] ; then + wget \ + http://heasarc.gsfc.nasa.gov/FTP/software/fitsio/c/cfitsio_latest.tar.gz \ + -O tmp/cfitsio_latest.tar.gz + tar zxf tmp/cfitsio_latest.tar.gz + cd cfitsio + ./configure + make + make utils stand_alone shared fi +# Upgrade pip itself before installing other python packages. +pip install -U pip +pip install -r "${THIS_DIR}/../requirements.txt" + +exit + +- cd cfitsio +- ./configure +- make +- make fpack +- make funpack +- sudo make install + + + + + + exit dist: trusty From 2f539f816f7150aaec5643fa99b2c651ba071ad1 Mon Sep 17 00:00:00 2001 From: jamessynge Date: Tue, 21 Nov 2017 21:30:52 -0500 Subject: [PATCH 03/15] wip - Added download, build and install of astrometry.net, but it fails so far because fitsio headers can't be found. Also added download of index files. --- scripts/apt-packages-list.txt | 2 +- scripts/install-dependencies.sh | 35 +++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/scripts/apt-packages-list.txt b/scripts/apt-packages-list.txt index cb6066c7c..e81023cca 100644 --- a/scripts/apt-packages-list.txt +++ b/scripts/apt-packages-list.txt @@ -9,6 +9,7 @@ libcairo2-dev libjpeg-dev libnetpbm10-dev libpng12-dev +mongodb netpbm python-dev python-numpy @@ -31,7 +32,6 @@ libgeos-dev libgraphviz-dev libncurses5-dev lm-sensors -mongodb openssh-client openssh-server pkg-config diff --git a/scripts/install-dependencies.sh b/scripts/install-dependencies.sh index 361c2695a..db01c7d56 100755 --- a/scripts/install-dependencies.sh +++ b/scripts/install-dependencies.sh @@ -2,6 +2,8 @@ # TODO(jamessynge): Add flags to control behavior, such as skipping apt-get. +ASTROMETRY_VERSION="0.72" + if [[ -z "${PANDIR}" || -z "${POCS}" || -z "${PAWS}" || -z "${PANLOG}" || -z "${PANUSER}" || ! -d "${PANDIR}" ]] ; then echo "Please set the Panoptes environment variables, then re-run this script." @@ -51,6 +53,39 @@ if [[ ! -d cfitsio ]] ; then make utils stand_alone shared fi +if [[ ! -d astrometry/bin ]] ; then + # This uses a directory in tmp to download and build astrometry, then + # installs the binaries into $PANDIR/astrometry. + install_astrometry() { + local -r DIR="astrometry.net-${ASTROMETRY_VERSION}" + local -r FN="${DIR}.tar.gz" + local -r URL="http://astrometry.net/downloads/${FN}" + local -r SCRATCH_DIR="$(mktemp -d "${TMPDIR:-/tmp/}install-astrometry.XXXXXXXXXXXX")" + local -r INSTALL_TO="${PANDIR}/astrometry" + cd ${SCRATCH_DIR} + echo "Fetching astrometry into directory $(pwd)" + wget "${URL}" + tar zxvf "${FN}" + cd "${DIR}" && make && make py && make install "INSTALL_DIR=${INSTALL_TO}" + echo "add_path ${INSTALL_TO}/data" >> "${INSTALL_TO}/etc/astrometry.cfg" + cd "${PANDIR}" + } + mkdir astrometry + install_astrometry +fi + +if [[ ! -f astrometry/data/index-4107.fits ]] ; then + mkdir astrometry/data + (cd astrometry/data ; + curl --remote-name http://broiler.astrometry.net/~dstn/4100/index-41[07-19].fits ) +fi + +if [[ ! -f astrometry/data/index-4208.fits ]] ; then + mkdir astrometry/data + (cd astrometry/data ; + curl --remote-name http://broiler.astrometry.net/~dstn/4200/index-42[08-19].fits ) +fi + # Upgrade pip itself before installing other python packages. pip install -U pip pip install -r "${THIS_DIR}/../requirements.txt" From 17e1a809458f1b238432e07660bed6d66faeaa9b Mon Sep 17 00:00:00 2001 From: jamessynge Date: Tue, 21 Nov 2017 21:35:47 -0500 Subject: [PATCH 04/15] wip - Fix creation of astrometry/data directory. --- scripts/install-dependencies.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/install-dependencies.sh b/scripts/install-dependencies.sh index db01c7d56..76d7a38b6 100755 --- a/scripts/install-dependencies.sh +++ b/scripts/install-dependencies.sh @@ -75,13 +75,13 @@ if [[ ! -d astrometry/bin ]] ; then fi if [[ ! -f astrometry/data/index-4107.fits ]] ; then - mkdir astrometry/data + mkdir -p astrometry/data (cd astrometry/data ; curl --remote-name http://broiler.astrometry.net/~dstn/4100/index-41[07-19].fits ) fi if [[ ! -f astrometry/data/index-4208.fits ]] ; then - mkdir astrometry/data + mkdir -p astrometry/data (cd astrometry/data ; curl --remote-name http://broiler.astrometry.net/~dstn/4200/index-42[08-19].fits ) fi From 9d06b89cc43ccf356b4d8a5b773a717739c3c9e7 Mon Sep 17 00:00:00 2001 From: jamessynge Date: Thu, 23 Nov 2017 21:45:55 -0500 Subject: [PATCH 05/15] wip - Added functions for each type of dependenciy to be installed and command-line flags to disable each of the various types of dependencies from beig installed. Still need to remove excess crap at end of file, and to resolve a problem running pytest. --- scripts/install-dependencies.sh | 390 +++++++++++++++++++++++++++++--- 1 file changed, 363 insertions(+), 27 deletions(-) diff --git a/scripts/install-dependencies.sh b/scripts/install-dependencies.sh index 76d7a38b6..8b38ce87f 100755 --- a/scripts/install-dependencies.sh +++ b/scripts/install-dependencies.sh @@ -1,8 +1,7 @@ #!/bin/bash -e -# TODO(jamessynge): Add flags to control behavior, such as skipping apt-get. - -ASTROMETRY_VERSION="0.72" +THIS_DIR="$(dirname $(readlink -f "${0}"))" +THIS_PROGRAM="$(basename "${0}")" if [[ -z "${PANDIR}" || -z "${POCS}" || -z "${PAWS}" || -z "${PANLOG}" || -z "${PANUSER}" || ! -d "${PANDIR}" ]] ; then @@ -10,26 +9,346 @@ if [[ -z "${PANDIR}" || -z "${POCS}" || -z "${PAWS}" || -z "${PANLOG}" || exit 1 fi -THIS_DIR="$(dirname $(readlink -f "${0}"))" -APT_PKGS="$(egrep -v '^#|^ *$' "${THIS_DIR}/apt-packages-list.txt"|sort|uniq)" - cd "${PANDIR}" -mkdir -p tmp -echo -echo "Running sudo apt-get install, you may be prompted for your password." -echo +# TODO(jamessynge): Add flags to control behavior, such as skipping apt-get. -(set -x ; sudo apt-get install --yes ${APT_PKGS}) +ASTROMETRY_VERSION="0.72" +INSTALL_PREFIX="/usr/local" +DO_APT_GET=1 +DO_CONDA=1 +DO_CFITSIO=1 +DO_ASTROMETRY=1 +DO_ASTROMETRY_INDICES=1 +DO_PIP_REQUIREMENTS=1 -add_to_PATH() { +function echo_bar() { + echo <>"${HOME}/.bashrc" " # Added by PANOPTES install-dependencies.sh PATH=\"${the_dir}:\${PATH}\"" PATH="${the_dir}:${PATH}" + echo + echo "Added ${the_dir} to PATH in .bashrc" +} + +# Given the path to a pkg-config file (.pc), extract the version number. +function extract_version_from_pkg_config() { + if [[ -f "${1}" ]] ; then + egrep '^Version:' "${1}" | cut '-d:' -f2 + else + echo "" + fi +} + +# Install miniconda (conda with no additional packages); we can then select +# the set to install. +function install_conda() { + local -r the_script="${PANDIR}/tmp/miniconda.sh" + echo_bar + echo + echo "Installing miniconda" + wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \ + -O "${the_script}" + bash "${the_script}" -b -p "${PANDIR}/miniconda" + rm "${the_script}" + add_to_PATH miniconda/bin +} + +function install_conda_if_missing() { + # Intall latest version of Miniconda (Anaconda with + # fewer packages; any that are needed can then be installed). + CONDA_BIN="$(which conda)" # Assuming is a python3 version if found. + if [[ -z "${CONDA_BIN}" ]] ; then + install_conda + fi +} + +# Fetches and builds the latest version of cfitsio. +function fetch_and_build_cfitsio() { + echo_bar + echo + echo "Fetching and building latest cfitsio release." + echo + # Unpack into PANDIR/tmp/cfitsio + cd "${PANDIR}" + mkdir -p tmp/cfitsio + cd tmp + rm -rf cfitsio + wget \ + http://heasarc.gsfc.nasa.gov/FTP/software/fitsio/c/cfitsio_latest.tar.gz \ + -O cfitsio_latest.tar.gz + tar zxf cfitsio_latest.tar.gz + cd cfitsio + ./configure --prefix=${INSTALL_PREFIX} + make + make utils stand_alone shared +} + +# Print the version of the downloaded and built cfitsio version number, as in +# its pkg-config file (cfitsio.pc). +function latest_cfitsio_version() { + extract_version_from_pkg_config tmp/cfitsio/cfitsio.pc +} + +# Prints the installed cfitsio version number. +function installed_cfitsio_version() { + pkg-config --mod-version --silence-errors cfitsio || /bin/true } +# Install the downloaded and built cfitsio. +function sudo_install_cfitsio() { + echo + echo "Running sudo make install for cfitsio, you may be prompted for your password." + echo + cd "${PANDIR}/tmp/cfitsio" + (set -x ; sudo make install) +} + +# Install the latest versio of cfitsio if it is not already installed. +function install_latest_cfitsio() { + (fetch_and_build_cfitsio) + local -r LATEST_CFITSIO_VERSION="$(latest_cfitsio_version)" + local -r INSTALLED_CFITSIO_VERSION="$(installed_cfitsio_version)" + if [[ "${LATEST_CFITSIO_VERSION}" == "${INSTALLED_CFITSIO_VERSION}" ]] ; then + echo "Installed cfitsio is at latest version (${LATEST_CFITSIO_VERSION})." + else + if [[ -z "${INSTALLED_CFITSIO_VERSION}" ]] ; then + echo "Installing cfitsio version ${LATEST_CFITSIO_VERSION}." + else + echo "Installing cfitsio version ${LATEST_CFITSIO_VERSION}, replacing" \ + "installed version ${OLD_CFITSIO_VERSION}." + fi + (sudo_install_cfitsio) + fi + rm -rf tmp/cfitsio_latest.tar.gz tmp/cfitsio +} + +# Downloads astrometry version ${ASTROMETRY_VERSION} into a temp directory, +# then builds and installs into ${PANDIR}/astrometry. +# TODO(jamessynge): Discuss whether to build directly in ${PANDIR}/astrometry, +# or instead installing into /usr/local as we do with cfitsio. +function install_astrometry() { + local -r DIR="astrometry.net-${ASTROMETRY_VERSION}" + local -r FN="${DIR}.tar.gz" + local -r URL="http://astrometry.net/downloads/${FN}" + local -r SCRATCH_DIR="$(mktemp -d "${TMPDIR:-/tmp/}install-astrometry.XXXXXXXXXXXX")" + local -r INSTALL_TO="${PANDIR}/astrometry" + cd "${SCRATCH_DIR}" + + echo_bar + echo + echo "Fetching astrometry into directory $(pwd)" + + wget "${URL}" + tar zxvf "${FN}" + cd "${DIR}" + + echo "Building ${DIR}" + make + make py + if [[ -d "${INSTALL_TO}" ]] ; then + rm -rf "${INSTALL_TO}" + fi + mkdir -p "${INSTALL_TO}" + echo "Installing into ${INSTALL_TO}" + make install "INSTALL_DIR=${INSTALL_TO}" + echo "add_path ${INSTALL_TO}/data" >> "${INSTALL_TO}/etc/astrometry.cfg" + + add_to_PATH "${INSTALL_TO}/bin" + cd "${PANDIR}" + rm -rf "${SCRATCH_DIR}" +} + +function install_astrometry_indices() { + mkdir -p "${PANDIR}/astrometry/data" + cd "${PANDIR}/astrometry/data" + if [[ ! -f index-4107.fits ]] ; then + curl --remote-name \ + http://broiler.astrometry.net/~dstn/4100/index-41[07-19].fits + fi + if [[ ! -f astrometry/data/index-4208.fits ]] ; then + curl --remote-name \ + http://broiler.astrometry.net/~dstn/4200/index-42[08-19].fits + fi +} + +#------------------------------------------------------------------------------- + +function show_help() { + echo "${THIS_PROGRAM} - Install software needed for PANOPTES." + echo " " + echo "${THIS_PROGRAM} [options]" + echo " " + echo "options:" + echo "-h, --help show brief help" + echo "--no-apt-get don't run apt-get to install Linux packages" + echo "--no-cfitsio don't install the latest version of cfitsio" + echo "--no-astrometry don't install astrometry.net software" + echo "--no-astrometry-indices don't install astrometry.net indices" + echo "--no-pip-requirements don't install python packages" +} + +cd "${PANDIR}" +mkdir -p tmp + +while test ${#} -gt 0; do + case "${1}" in + -h|--help) + show_help + exit 0 + ;; + --no-apt-get) + DO_APT_GET=0 + shift + ;; + --no-cfitsio) + DO_CFITSIO=0 + shift + ;; + --no-conda) + DO_CONDA=0 + shift + ;; + --no-astrometry) + DO_ASTROMETRY=0 + shift + ;; + --no-astrometry-ind*) + DO_ASTROMETRY_INDICES=0 + shift + ;; + --no-pip-requirements) + DO_PIP_REQUIREMENTS=0 + shift + ;; + -x) + set -x + shift + ;; + *) + echo "Unknown parameter: ${1}" + echo + show_help + exit 1 + ;; + esac +done + + +if [[ "${DO_APT_GET}" -eq 1 ]] ; then + echo + echo "Running sudo apt-get install, you may be prompted for your password." + echo + (set -x ; sudo apt-get install --yes ${APT_PKGS}) +fi + + +if [[ "${DO_CONDA}" -eq 1 ]] ; then + install_conda_if_missing +fi + + +if [[ "${DO_CFITSIO}" -eq 1 ]] ; then + install_latest_cfitsio +fi + + +if [[ "${DO_ASTROMETRY}" -eq 1 ]] ; then + (install_astrometry) +fi + + +if [[ "${DO_ASTROMETRY_INDICES}" -eq 1 ]] ; then + (install_astrometry_indices) +fi + + +if [[ "${DO_PIP_REQUIREMENTS}" -eq 1 ]] ; then + # Upgrade pip itself before installing other python packages. + pip install -U pip + pip install -r "${THIS_DIR}/../requirements.txt" +fi + +cd "${PANDIR}" +rmdir --ignore-fail-on-non-empty "${PANDIR}/tmp" + +exit + + + + + + +exit + + + + + + install_astrometry + + + + + + + + + + + +fi + + +if [[ "${DO_ASTROMETRY_INDICES}" -eq 1 ]] ; then + install_astrometry_indices +fi + + +set -x +CONDA_BIN=$(which conda) # Assuming is for python3 if found. +fi + + + + + + echo + echo "Running sudo apt-get install, you may be prompted for your password." + echo + (set -x ; sudo apt-get install --yes ${APT_PKGS}) +fi + + + + + + + + + + + + +APT_PKGS="$(egrep -v '^#|^ *$' "${THIS_DIR}/apt-packages-list.txt"|sort|uniq)" + +cd "${PANDIR}" +mkdir -p tmp + +echo +echo "Running sudo apt-get install, you may be prompted for your password." +echo + +(set -x ; sudo apt-get install --yes ${APT_PKGS}) + # If conda is not present, install latest version of Miniconda (Anaconda with # fewer packages; any that are needed can then be installed). set -x @@ -42,17 +361,32 @@ if [[ -z "${CONDA_BIN}" ]] ; then add_to_PATH miniconda/bin fi -if [[ ! -d cfitsio ]] ; then - wget \ - http://heasarc.gsfc.nasa.gov/FTP/software/fitsio/c/cfitsio_latest.tar.gz \ - -O tmp/cfitsio_latest.tar.gz - tar zxf tmp/cfitsio_latest.tar.gz - cd cfitsio - ./configure - make - make utils stand_alone shared +if [[ ! -f /usr/local/include/fitsio.h ]] ; then + install_cfitsio() { + wget \ + http://heasarc.gsfc.nasa.gov/FTP/software/fitsio/c/cfitsio_latest.tar.gz \ + -O tmp/cfitsio_latest.tar.gz + tar zxf tmp/cfitsio_latest.tar.gz + cd cfitsio + ./configure --prefix=/usr/local + make + make utils stand_alone shared + echo + echo "Running sudo make install for cfitsio, you may be prompted for your password." + echo + (set -x ; sudo make install) + rm tmp/cfitsio_latest.tar.gz + rmdir --ignore-fail-on-non-empty tmp + } + if [[ -d cfitsio ]] ; then + rm -rf cfitsio + fi + (install_cfitsio) fi +exit + + if [[ ! -d astrometry/bin ]] ; then # This uses a directory in tmp to download and build astrometry, then # installs the binaries into $PANDIR/astrometry. @@ -60,7 +394,8 @@ if [[ ! -d astrometry/bin ]] ; then local -r DIR="astrometry.net-${ASTROMETRY_VERSION}" local -r FN="${DIR}.tar.gz" local -r URL="http://astrometry.net/downloads/${FN}" - local -r SCRATCH_DIR="$(mktemp -d "${TMPDIR:-/tmp/}install-astrometry.XXXXXXXXXXXX")" + local -r TMP_DIR="$(realpath "${TMPDIR:-${PANDIR}/tmp}")" + local -r SCRATCH_DIR="$(mktemp -d "${TMP_DIR}/install-astrometry.XXXXXXXXXXXX")" local -r INSTALL_TO="${PANDIR}/astrometry" cd ${SCRATCH_DIR} echo "Fetching astrometry into directory $(pwd)" @@ -68,22 +403,23 @@ if [[ ! -d astrometry/bin ]] ; then tar zxvf "${FN}" cd "${DIR}" && make && make py && make install "INSTALL_DIR=${INSTALL_TO}" echo "add_path ${INSTALL_TO}/data" >> "${INSTALL_TO}/etc/astrometry.cfg" - cd "${PANDIR}" } - mkdir astrometry - install_astrometry + mkdir -p astrometry + (install_astrometry) fi if [[ ! -f astrometry/data/index-4107.fits ]] ; then mkdir -p astrometry/data (cd astrometry/data ; - curl --remote-name http://broiler.astrometry.net/~dstn/4100/index-41[07-19].fits ) + curl --remote-name \ + http://broiler.astrometry.net/~dstn/4100/index-41[07-19].fits ) fi if [[ ! -f astrometry/data/index-4208.fits ]] ; then mkdir -p astrometry/data (cd astrometry/data ; - curl --remote-name http://broiler.astrometry.net/~dstn/4200/index-42[08-19].fits ) + curl --remote-name \ + http://broiler.astrometry.net/~dstn/4200/index-42[08-19].fits ) fi # Upgrade pip itself before installing other python packages. From 0dafb45afb9e48c3cb0491b001cef3df36e86f72 Mon Sep 17 00:00:00 2001 From: jamessynge Date: Fri, 24 Nov 2017 15:05:58 -0500 Subject: [PATCH 06/15] Installation script for PANOPTES. Ready for review AND testing (perhaps in a docker container). --- scripts/apt-packages-list.txt | 82 ++++--- scripts/install-dependencies.sh | 408 +++++++++++--------------------- 2 files changed, 196 insertions(+), 294 deletions(-) diff --git a/scripts/apt-packages-list.txt b/scripts/apt-packages-list.txt index e81023cca..92b6d9f09 100644 --- a/scripts/apt-packages-list.txt +++ b/scripts/apt-packages-list.txt @@ -1,45 +1,67 @@ # This file contains a list of apt packages to install. -# Lines starting with # are comments. -# astrometry.net # No, will install from source into /var/panoptes/astrometry. -# libcfitsio3-dev # Do we need this, or should we build it as in .travis.yml -# And if we should build it, should it be in /var/pantoptes or in $HOME? +# Lines starting with # are comments as are blank lines, +# and # starts trailing comments. + +# dcraw decodes Canon raw images, and gphoto2 takes pictures with +# Canon DSLRs, among others. +dcraw gphoto2 -libbz2-dev -libcairo2-dev + +# Packages needed according to http://astrometry.net/doc/build.html +libbz2-dev +libbz2-dev +libcfitsio-dev libjpeg-dev -libnetpbm10-dev +libnetpbm10-dev libpng12-dev -mongodb netpbm -python-dev +python-dev python-numpy python-pyfits -swig -zlib1g-dev -# These additional packages may or may not be needed, I simply installed them on -# the NUC for PAN006 and need to determine if they are needed. -dcraw +swig +zlib1g-dev + +# Cairo is a graphics library, and matplotlib can use it as a backend +# for rendering. +libcairo2-dev -# Is exiftool needed? It gets converted to libimage-exiftool-perl, which I'm -# guessing we don't need. -# exiftool +# exiftool is used in a couple of places to extract info from the .cr2 file +exiftool +# Linux monitoring sensors provides access to hardward sensors in the computer, +# such as CPU or GPU temperature. May require bios or other changes to make use +# of these. +# TODO(jamessynge): Discuss these with Wilfred as an additional way to get +# temperature data. +# lm-sensors + +# These are used if you want to setup SSH access into the computer. +# openssh-client +# openssh-server + +# Graphviz is used for rendering the state machine of POCS. +# TODO(wtgee): Are there any other uses? graphviz -iw -libgeos-3.* -libgeos-c1v5 -libgeos-dev libgraphviz-dev -libncurses5-dev -lm-sensors -openssh-client -openssh-server + +# Linux tool used during building and installing software (e.g. astrometry). pkg-config -python-psutil -# I use tmux (in lieu of screen) to start the various Panoptes programs at -# system startup. -tmux +# Cross platform python library for getting system info. +# TODO(wtgee): Is this used by any tools we use? It isn't directly used by +# POCS et al. +# python-psutil + +# These additional packages may or may not be needed, I simply installed them on +# the NUC for PAN006 and need to determine if they are needed. Commenting them +# out for now until a reason for their presence is established. + +# iw # Wireless networking config & info. + +# libgeos-3.* +# libgeos-c1v5 +# libgeos-dev +# libncurses5-dev -xbase-clients +# xbase-clients diff --git a/scripts/install-dependencies.sh b/scripts/install-dependencies.sh index 8b38ce87f..0d857145f 100755 --- a/scripts/install-dependencies.sh +++ b/scripts/install-dependencies.sh @@ -16,27 +16,32 @@ cd "${PANDIR}" ASTROMETRY_VERSION="0.72" INSTALL_PREFIX="/usr/local" DO_APT_GET=1 +DO_MONGODB=1 DO_CONDA=1 -DO_CFITSIO=1 +DO_CFITSIO=0 DO_ASTROMETRY=1 DO_ASTROMETRY_INDICES=1 +DO_ASTROMETRY_FULL_INDICES=0 DO_PIP_REQUIREMENTS=1 function echo_bar() { - echo <>"${HOME}/.bashrc" " + if [[ -z "$(egrep -- "PATH=.*${the_dir}" ~/.bashrc)" ]] ; then + echo >>"${HOME}/.bashrc" " # Added by PANOPTES install-dependencies.sh PATH=\"${the_dir}:\${PATH}\"" - PATH="${the_dir}:${PATH}" - echo - echo "Added ${the_dir} to PATH in .bashrc" + PATH="${the_dir}:${PATH}" + echo + echo "Added ${the_dir} to PATH in .bashrc" + else + echo ".bashrc already adds ${the_dir} to PATH" + fi } # Given the path to a pkg-config file (.pc), extract the version number. @@ -48,6 +53,55 @@ function extract_version_from_pkg_config() { fi } +function install_apt_packages() { + # Remove all the comments from the package list and install the packages whose + # names are left. + APT_PKGS="$(cut '-d#' -f1 "${THIS_DIR}/apt-packages-list.txt" | sort | uniq)" + echo + echo "Running sudo apt-get install, you may be prompted for your password." + echo + (set -x ; sudo apt-get update) + (set -x ; sudo apt-get install --yes ${APT_PKGS}) +} + +function install_mongodb() { + # This is based on https://www.howtoforge.com/tutorial/install-mongodb-on-ubuntu-16.04/ + # Note this function does not configure mongodb itself, i.e. no users or + # security settings. + echo_bar + echo " +Installing mongodb, for which several commands require sudo, so +you may be prompted for you password. +" + sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv EA312927 + echo "deb http://repo.mongodb.org/apt/ubuntu "$(lsb_release -sc)"/mongodb-org/3.2 multiverse" \ + | sudo tee /etc/apt/sources.list.d/mongodb-org-3.2.list + sudo apt-get update + sudo apt-get install -y mongodb-org + echo "[Unit] +Description=High-performance, schema-free document-oriented database +After=network.target +Documentation=https://docs.mongodb.org/manual + +[Service] +User=mongodb +Group=mongodb +ExecStart=/usr/bin/mongod --quiet --config /etc/mongod.conf + +[Install] +WantedBy=multi-user.target" | sudo tee /lib/systemd/system/mongod.service + + sudo systemctl daemon-reload + sudo systemctl start mongod + sudo systemctl enable mongod +} + +function maybe_install_mongodb() { + if [[ -z "$(which mongo)" || -z "$(systemctl | grep mongo)" ]] ; then + install_mongodb + fi +} + # Install miniconda (conda with no additional packages); we can then select # the set to install. function install_conda() { @@ -68,14 +122,20 @@ function install_conda_if_missing() { CONDA_BIN="$(which conda)" # Assuming is a python3 version if found. if [[ -z "${CONDA_BIN}" ]] ; then install_conda + else + echo_bar + echo + echo "Updating conda installation." + conda update conda fi } -# Fetches and builds the latest version of cfitsio. -function fetch_and_build_cfitsio() { +# Fetches and configures the latest version of cfitsio; this allows us to +# extract the version from the package config file (cfitsio.pc). +function fetch_and_configure_cfitsio() { echo_bar echo - echo "Fetching and building latest cfitsio release." + echo "Fetching and configuring latest cfitsio release..." echo # Unpack into PANDIR/tmp/cfitsio cd "${PANDIR}" @@ -88,6 +148,14 @@ function fetch_and_build_cfitsio() { tar zxf cfitsio_latest.tar.gz cd cfitsio ./configure --prefix=${INSTALL_PREFIX} +} + +# Builds the downloaded version of cfitsio. +function build_cfitsio() { + echo + echo "Building cfitsio..." + echo + cd "${PANDIR}/tmp/cfitsio" make make utils stand_alone shared } @@ -114,7 +182,7 @@ function sudo_install_cfitsio() { # Install the latest versio of cfitsio if it is not already installed. function install_latest_cfitsio() { - (fetch_and_build_cfitsio) + (fetch_and_configure_cfitsio) local -r LATEST_CFITSIO_VERSION="$(latest_cfitsio_version)" local -r INSTALLED_CFITSIO_VERSION="$(installed_cfitsio_version)" if [[ "${LATEST_CFITSIO_VERSION}" == "${INSTALLED_CFITSIO_VERSION}" ]] ; then @@ -126,6 +194,7 @@ function install_latest_cfitsio() { echo "Installing cfitsio version ${LATEST_CFITSIO_VERSION}, replacing" \ "installed version ${OLD_CFITSIO_VERSION}." fi + (build_cfitsio) (sudo_install_cfitsio) fi rm -rf tmp/cfitsio_latest.tar.gz tmp/cfitsio @@ -160,7 +229,6 @@ function install_astrometry() { mkdir -p "${INSTALL_TO}" echo "Installing into ${INSTALL_TO}" make install "INSTALL_DIR=${INSTALL_TO}" - echo "add_path ${INSTALL_TO}/data" >> "${INSTALL_TO}/etc/astrometry.cfg" add_to_PATH "${INSTALL_TO}/bin" cd "${PANDIR}" @@ -170,13 +238,23 @@ function install_astrometry() { function install_astrometry_indices() { mkdir -p "${PANDIR}/astrometry/data" cd "${PANDIR}/astrometry/data" - if [[ ! -f index-4107.fits ]] ; then - curl --remote-name \ - http://broiler.astrometry.net/~dstn/4100/index-41[07-19].fits - fi - if [[ ! -f astrometry/data/index-4208.fits ]] ; then - curl --remote-name \ - http://broiler.astrometry.net/~dstn/4200/index-42[08-19].fits + # See http://astrometry.net/doc/readme.html#getting-index-files + # for info on the index files you might need. + if [[ "${DO_ASTROMETRY_FULL_INDICES}" -eq 1 ]] ; then + if [[ ! -f index-4107.fits ]] ; then + curl --remote-name \ + http://broiler.astrometry.net/~dstn/4100/index-41[07-19].fits + fi + if [[ ! -f astrometry/data/index-4208.fits ]] ; then + curl --remote-name \ + http://broiler.astrometry.net/~dstn/4200/index-42[08-19].fits + fi + else + # Install wide field indices. If we change to using longer focal length + if [[ ! -f index-4116.fits ]] ; then + curl --remote-name \ + http://broiler.astrometry.net/~dstn/4100/index-411[6-9].fits + fi fi } @@ -188,12 +266,17 @@ function show_help() { echo "${THIS_PROGRAM} [options]" echo " " echo "options:" - echo "-h, --help show brief help" - echo "--no-apt-get don't run apt-get to install Linux packages" - echo "--no-cfitsio don't install the latest version of cfitsio" - echo "--no-astrometry don't install astrometry.net software" - echo "--no-astrometry-indices don't install astrometry.net indices" - echo "--no-pip-requirements don't install python packages" + echo "-h, --help show brief help" + echo "-x turn on bash debug output" + echo "--run run the named function and exit, for debugging" + echo "--no-apt-get don't run apt-get to install Linux packages" + echo "--no-mongodb don't install and start mongodb server" + echo "--no-cfitsio don't install the latest version of cfitsio" + echo "--no-conda don't install the latest version of Anaconda" + echo "--no-astrometry don't install astrometry.net software" + echo "--no-astrometry-indices don't install astrometry.net indices" + echo "--astrometry-full-indices install full indices, for narrow and wide fields" + echo "--no-pip-requirements don't install python packages" } cd "${PANDIR}" @@ -205,10 +288,28 @@ while test ${#} -gt 0; do show_help exit 0 ;; + -x) + set -x + shift + ;; + --run) + if [ $# -lt 2 ]; then + echo "Function name missing after ${1}." + show_help + exit 1 + fi + shift + (${1}) + exit + ;; --no-apt-get) DO_APT_GET=0 shift ;; + --no-mongodb) + DO_MONGODB=0 + shift + ;; --no-cfitsio) DO_CFITSIO=0 shift @@ -223,14 +324,16 @@ while test ${#} -gt 0; do ;; --no-astrometry-ind*) DO_ASTROMETRY_INDICES=0 + DO_ASTROMETRY_FULL_INDICES=0 shift ;; - --no-pip-requirements) - DO_PIP_REQUIREMENTS=0 + --astrometry-full-ind*) + DO_ASTROMETRY_FULL_INDICES=1 + DO_ASTROMETRY_INDICES=1 shift ;; - -x) - set -x + --no-pip-requirements) + DO_PIP_REQUIREMENTS=0 shift ;; *) @@ -244,10 +347,12 @@ done if [[ "${DO_APT_GET}" -eq 1 ]] ; then - echo - echo "Running sudo apt-get install, you may be prompted for your password." - echo - (set -x ; sudo apt-get install --yes ${APT_PKGS}) + install_apt_packages +fi + + +if [[ "${DO_MONGODB}" -eq 1 ]] ; then + maybe_install_mongodb fi @@ -274,6 +379,11 @@ fi if [[ "${DO_PIP_REQUIREMENTS}" -eq 1 ]] ; then # Upgrade pip itself before installing other python packages. pip install -U pip + # TODO(jamessynge): Move this script and needed text files into an install + # directory. + # TODO(wtgee): Consider whether to inline the needed text files into this + # file, and add git clone of the PANOPTES repos, so that the user can do + # an install with just about two commands (wget script, then run script). pip install -r "${THIS_DIR}/../requirements.txt" fi @@ -282,233 +392,3 @@ rmdir --ignore-fail-on-non-empty "${PANDIR}/tmp" exit - - - - - -exit - - - - - - install_astrometry - - - - - - - - - - - -fi - - -if [[ "${DO_ASTROMETRY_INDICES}" -eq 1 ]] ; then - install_astrometry_indices -fi - - -set -x -CONDA_BIN=$(which conda) # Assuming is for python3 if found. -fi - - - - - - echo - echo "Running sudo apt-get install, you may be prompted for your password." - echo - (set -x ; sudo apt-get install --yes ${APT_PKGS}) -fi - - - - - - - - - - - - -APT_PKGS="$(egrep -v '^#|^ *$' "${THIS_DIR}/apt-packages-list.txt"|sort|uniq)" - -cd "${PANDIR}" -mkdir -p tmp - -echo -echo "Running sudo apt-get install, you may be prompted for your password." -echo - -(set -x ; sudo apt-get install --yes ${APT_PKGS}) - -# If conda is not present, install latest version of Miniconda (Anaconda with -# fewer packages; any that are needed can then be installed). -set -x -CONDA_BIN=$(which conda) # Assuming is for python3 if found. -if [[ -z "${CONDA_BIN}" ]] ; then - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \ - -O tmp/miniconda.sh - bash tmp/miniconda.sh -b -p "${PANDIR}/miniconda" - rm tmp/miniconda.sh - add_to_PATH miniconda/bin -fi - -if [[ ! -f /usr/local/include/fitsio.h ]] ; then - install_cfitsio() { - wget \ - http://heasarc.gsfc.nasa.gov/FTP/software/fitsio/c/cfitsio_latest.tar.gz \ - -O tmp/cfitsio_latest.tar.gz - tar zxf tmp/cfitsio_latest.tar.gz - cd cfitsio - ./configure --prefix=/usr/local - make - make utils stand_alone shared - echo - echo "Running sudo make install for cfitsio, you may be prompted for your password." - echo - (set -x ; sudo make install) - rm tmp/cfitsio_latest.tar.gz - rmdir --ignore-fail-on-non-empty tmp - } - if [[ -d cfitsio ]] ; then - rm -rf cfitsio - fi - (install_cfitsio) -fi - -exit - - -if [[ ! -d astrometry/bin ]] ; then - # This uses a directory in tmp to download and build astrometry, then - # installs the binaries into $PANDIR/astrometry. - install_astrometry() { - local -r DIR="astrometry.net-${ASTROMETRY_VERSION}" - local -r FN="${DIR}.tar.gz" - local -r URL="http://astrometry.net/downloads/${FN}" - local -r TMP_DIR="$(realpath "${TMPDIR:-${PANDIR}/tmp}")" - local -r SCRATCH_DIR="$(mktemp -d "${TMP_DIR}/install-astrometry.XXXXXXXXXXXX")" - local -r INSTALL_TO="${PANDIR}/astrometry" - cd ${SCRATCH_DIR} - echo "Fetching astrometry into directory $(pwd)" - wget "${URL}" - tar zxvf "${FN}" - cd "${DIR}" && make && make py && make install "INSTALL_DIR=${INSTALL_TO}" - echo "add_path ${INSTALL_TO}/data" >> "${INSTALL_TO}/etc/astrometry.cfg" - } - mkdir -p astrometry - (install_astrometry) -fi - -if [[ ! -f astrometry/data/index-4107.fits ]] ; then - mkdir -p astrometry/data - (cd astrometry/data ; - curl --remote-name \ - http://broiler.astrometry.net/~dstn/4100/index-41[07-19].fits ) -fi - -if [[ ! -f astrometry/data/index-4208.fits ]] ; then - mkdir -p astrometry/data - (cd astrometry/data ; - curl --remote-name \ - http://broiler.astrometry.net/~dstn/4200/index-42[08-19].fits ) -fi - -# Upgrade pip itself before installing other python packages. -pip install -U pip -pip install -r "${THIS_DIR}/../requirements.txt" - -exit - -- cd cfitsio -- ./configure -- make -- make fpack -- make funpack -- sudo make install - - - - - - -exit - -dist: trusty -sudo: required -language: python -services: - - mongodb -python: - - "3.6" -cache: - - pip -env: - - POCS=$TRAVIS_BUILD_DIR PANDIR=/var/panoptes -before_install: - - sudo mkdir /var/panoptes && sudo chmod 777 /var/panoptes - - mkdir $PANDIR/logs - - ln -s $TRAVIS_BUILD_DIR /var/panoptes/POCS - - pip install coveralls - - pip install -U pip - - cd $HOME - - wget http://heasarc.gsfc.nasa.gov/FTP/software/fitsio/c/cfitsio_latest.tar.gz - - tar zxf cfitsio_latest.tar.gz - - cd cfitsio - - ./configure - - make - - make fpack - - make funpack - - sudo make install - - sudo mkdir -p /var/panoptes/astrometry/data - - sudo chmod -R 777 /var/panoptes/astrometry/ -addons: - apt: - packages: - - gphoto2 - - libcairo2-dev - - libnetpbm10-dev - - netpbm - - libpng12-dev - - libjpeg-dev - - python-numpy - - python-pyfits - - python-dev - - zlib1g-dev - - libbz2-dev - - swig - - cfitsio-dev -install: - - wget http://astrometry.net/downloads/astrometry.net-0.72.tar.gz - - tar zxvf astrometry.net-0.72.tar.gz - - cd astrometry.net-0.72 && make && make py && make install INSTALL_DIR=/var/panoptes/astrometry - - echo 'add_path /var/panoptes/astrometry/data' | sudo tee --append /var/panoptes/astrometry/etc/astrometry.cfg - - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; - - bash miniconda.sh -b -p $HOME/miniconda - - export PATH="$HOME/miniconda/bin:$HOME/cfitsio/bin:/var/panoptes/astrometry/bin:$PATH" - - hash -r - - - conda config --set always_yes yes --set changeps1 no - - conda update -q conda - - conda info -a # Useful for debugging any issues with conda - - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION - - source activate test-environment - - - cd $TRAVIS_BUILD_DIR - - pip install -r requirements.txt - - python setup.py install -script: - - coverage run setup.py test - - coverage combine .coverage* -after_success: - - if [[ $TRAVIS_PYTHON_VERSION == 3.6* ]]; then - bash <(curl -s https://codecov.io/bash); - fi From 0cb369115fb76545f02137efa638a750355b8de1 Mon Sep 17 00:00:00 2001 From: jamessynge Date: Fri, 24 Nov 2017 15:15:51 -0500 Subject: [PATCH 07/15] Move install script and input files into POCS/scripts/install. --- scripts/{ => install}/apt-packages-list.txt | 0 scripts/{ => install}/install-dependencies.sh | 0 requirements.txt => scripts/install/requirements.txt | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename scripts/{ => install}/apt-packages-list.txt (100%) rename scripts/{ => install}/install-dependencies.sh (100%) rename requirements.txt => scripts/install/requirements.txt (100%) diff --git a/scripts/apt-packages-list.txt b/scripts/install/apt-packages-list.txt similarity index 100% rename from scripts/apt-packages-list.txt rename to scripts/install/apt-packages-list.txt diff --git a/scripts/install-dependencies.sh b/scripts/install/install-dependencies.sh similarity index 100% rename from scripts/install-dependencies.sh rename to scripts/install/install-dependencies.sh diff --git a/requirements.txt b/scripts/install/requirements.txt similarity index 100% rename from requirements.txt rename to scripts/install/requirements.txt From 1e688c2604e0dba0b733dbf34f5255dfbe029951 Mon Sep 17 00:00:00 2001 From: jamessynge Date: Fri, 24 Nov 2017 16:09:15 -0500 Subject: [PATCH 08/15] Fix path to requirements.txt. Add libcfitsio-bin to apt packages. --- scripts/install/apt-packages-list.txt | 4 ++++ scripts/install/install-dependencies.sh | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/install/apt-packages-list.txt b/scripts/install/apt-packages-list.txt index 92b6d9f09..bc5a873a0 100644 --- a/scripts/install/apt-packages-list.txt +++ b/scripts/install/apt-packages-list.txt @@ -28,6 +28,10 @@ libcairo2-dev # exiftool is used in a couple of places to extract info from the .cr2 file exiftool +# We use cfitsio command line tools (e.g. fpack) from POCS, in addition to the +# uses of the cfitsio libraries by astrometry above. +libcfitsio-bin + # Linux monitoring sensors provides access to hardward sensors in the computer, # such as CPU or GPU temperature. May require bios or other changes to make use # of these. diff --git a/scripts/install/install-dependencies.sh b/scripts/install/install-dependencies.sh index 0d857145f..6fa21c3cf 100755 --- a/scripts/install/install-dependencies.sh +++ b/scripts/install/install-dependencies.sh @@ -384,7 +384,7 @@ if [[ "${DO_PIP_REQUIREMENTS}" -eq 1 ]] ; then # TODO(wtgee): Consider whether to inline the needed text files into this # file, and add git clone of the PANOPTES repos, so that the user can do # an install with just about two commands (wget script, then run script). - pip install -r "${THIS_DIR}/../requirements.txt" + pip install -r "${THIS_DIR}/requirements.txt" fi cd "${PANDIR}" From 8d6062a772fa845ff7cb4a80871ba65b7d7e18dc Mon Sep 17 00:00:00 2001 From: jamessynge Date: Fri, 24 Nov 2017 16:13:38 -0500 Subject: [PATCH 09/15] And fix the requirements.txt path for travis too. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7aa4863e0..c95a277c3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -59,7 +59,7 @@ install: - source activate test-environment - cd $TRAVIS_BUILD_DIR - - pip install -r requirements.txt + - pip install -r scripts/install/requirements.txt - python setup.py install script: - coverage run setup.py test From a2cbb07e1c8f0660987bf43e2c8c22cade4ae451 Mon Sep 17 00:00:00 2001 From: jamessynge Date: Fri, 24 Nov 2017 17:47:21 -0500 Subject: [PATCH 10/15] Move requirements.txt back to root of repo. Emit message reminding the user that PATH needs to be updated in current shell. --- scripts/install/requirements.txt => requirements.txt | 0 scripts/install/install-dependencies.sh | 9 ++++++++- 2 files changed, 8 insertions(+), 1 deletion(-) rename scripts/install/requirements.txt => requirements.txt (100%) diff --git a/scripts/install/requirements.txt b/requirements.txt similarity index 100% rename from scripts/install/requirements.txt rename to requirements.txt diff --git a/scripts/install/install-dependencies.sh b/scripts/install/install-dependencies.sh index 6fa21c3cf..a47771034 100755 --- a/scripts/install/install-dependencies.sh +++ b/scripts/install/install-dependencies.sh @@ -384,11 +384,18 @@ if [[ "${DO_PIP_REQUIREMENTS}" -eq 1 ]] ; then # TODO(wtgee): Consider whether to inline the needed text files into this # file, and add git clone of the PANOPTES repos, so that the user can do # an install with just about two commands (wget script, then run script). - pip install -r "${THIS_DIR}/requirements.txt" + pip install -r "${POCS}/requirements.txt" fi cd "${PANDIR}" rmdir --ignore-fail-on-non-empty "${PANDIR}/tmp" +set +x +echo +echo_bar +echo +echo "Remember to update PATH in your shell before running tests." +echo + exit From 8873b1b4add17ec2c728d238362b50036bd7e603 Mon Sep 17 00:00:00 2001 From: jamessynge Date: Fri, 24 Nov 2017 19:30:17 -0500 Subject: [PATCH 11/15] Write paths to .profile instead of .bashrc. Create a conda environment for all the python packages, etc. Install python packages before building astrometry.net, as it uses numpy. --- scripts/install/apt-packages-list.txt | 50 ++++++++---------- scripts/install/install-dependencies.sh | 70 ++++++++++++++++++------- 2 files changed, 74 insertions(+), 46 deletions(-) diff --git a/scripts/install/apt-packages-list.txt b/scripts/install/apt-packages-list.txt index bc5a873a0..f41734e9f 100644 --- a/scripts/install/apt-packages-list.txt +++ b/scripts/install/apt-packages-list.txt @@ -2,9 +2,15 @@ # Lines starting with # are comments as are blank lines, # and # starts trailing comments. -# dcraw decodes Canon raw images, and gphoto2 takes pictures with -# Canon DSLRs, among others. +# dcraw decodes Canon raw images dcraw + +# gphoto2 enables remote control of cameras, including the Canon DSLRs that +# Panoptes uses. This includes changing many settings and taking pictures. +# We are currently using the published linux package, but that can be quite +# old relative to the latest stable version. gphoto2-updater is an install +# script that we might want to use if we need a newer version. For more info, +# see: https://github.com/gonzalo/gphoto2-updater gphoto2 # Packages needed according to http://astrometry.net/doc/build.html @@ -32,40 +38,28 @@ exiftool # uses of the cfitsio libraries by astrometry above. libcfitsio-bin -# Linux monitoring sensors provides access to hardward sensors in the computer, -# such as CPU or GPU temperature. May require bios or other changes to make use -# of these. -# TODO(jamessynge): Discuss these with Wilfred as an additional way to get -# temperature data. -# lm-sensors - -# These are used if you want to setup SSH access into the computer. -# openssh-client -# openssh-server +# These are used if you want to setup SSH access into the computer (pretty +# darn likely). +openssh-client +openssh-server # Graphviz is used for rendering the state machine of POCS. -# TODO(wtgee): Are there any other uses? graphviz libgraphviz-dev # Linux tool used during building and installing software (e.g. astrometry). pkg-config -# Cross platform python library for getting system info. -# TODO(wtgee): Is this used by any tools we use? It isn't directly used by -# POCS et al. -# python-psutil - -# These additional packages may or may not be needed, I simply installed them on -# the NUC for PAN006 and need to determine if they are needed. Commenting them -# out for now until a reason for their presence is established. - -# iw # Wireless networking config & info. +# Improves interaction with pocs_shell (via readline). +libncurses5-dev -# libgeos-3.* -# libgeos-c1v5 -# libgeos-dev -# libncurses5-dev +# These support the Python package "Shapely", used in PIAA (called from POCS). +libgeos-3.* +libgeos-c1v5 +libgeos-dev -# xbase-clients +# byobu and tmux provide a means to run multiple interactive shells in a way +# that survives disconnection. +byobu +tmux diff --git a/scripts/install/install-dependencies.sh b/scripts/install/install-dependencies.sh index a47771034..49ffc4089 100755 --- a/scripts/install/install-dependencies.sh +++ b/scripts/install/install-dependencies.sh @@ -4,11 +4,12 @@ THIS_DIR="$(dirname $(readlink -f "${0}"))" THIS_PROGRAM="$(basename "${0}")" if [[ -z "${PANDIR}" || -z "${POCS}" || -z "${PAWS}" || -z "${PANLOG}" || - -z "${PANUSER}" || ! -d "${PANDIR}" ]] ; then + -z "${PANUSER}" ]] ; then echo "Please set the Panoptes environment variables, then re-run this script." exit 1 fi +mkdir -p "${PANDIR}" cd "${PANDIR}" # TODO(jamessynge): Add flags to control behavior, such as skipping apt-get. @@ -29,18 +30,44 @@ function echo_bar() { printf "%${COLUMNS:-80}s\n" | tr ' ' '#' } -# Append $1 to PATH and write command to do the same to .bashrc. +# Append $1 to .profile +function add_to_profile() { + local -r the_line="${1}" + local -r the_file="${HOME}/.profile" + if [[ ! -f "${the_file}" ]] ; then + touch "${the_file}" + fi + if [[ -z "$(fgrep -- "${the_line}" "${the_file}")" ]] ; then + echo >>"${the_file}" " +# Added by PANOPTES install-dependencies.sh +${the_line}" + echo "Appended to ${the_file}: ${the_line}" + else + echo "Already in ${the_file}: ${the_line}" + fi +} + +# Append $1 to PATH and write command to do the same to .profile. function add_to_PATH() { local -r the_dir="$(readlink -f "${1}")" - if [[ -z "$(egrep -- "PATH=.*${the_dir}" ~/.bashrc)" ]] ; then - echo >>"${HOME}/.bashrc" " + add_to_profile "PATH=\"${the_dir}:\${PATH}\"" + PATH="${the_dir}:${PATH}" + + return 0 + + local -r the_file="${HOME}/.profile" + if [[ ! -f "${the_file}" ]] ; then + touch "${the_file}" + fi + if [[ -z "$(egrep -- "PATH=.*${the_dir}" "${the_file}")" ]] ; then + echo >>"${the_file}" " # Added by PANOPTES install-dependencies.sh PATH=\"${the_dir}:\${PATH}\"" PATH="${the_dir}:${PATH}" echo - echo "Added ${the_dir} to PATH in .bashrc" + echo "Added ${the_dir} to PATH in ${the_file}" else - echo ".bashrc already adds ${the_dir} to PATH" + echo "${the_file} already adds ${the_dir} to PATH" fi } @@ -108,7 +135,7 @@ function install_conda() { local -r the_script="${PANDIR}/tmp/miniconda.sh" echo_bar echo - echo "Installing miniconda" + echo "Installing miniconda. License at: https://conda.io/docs/license.html" wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \ -O "${the_script}" bash "${the_script}" -b -p "${PANDIR}/miniconda" @@ -128,6 +155,12 @@ function install_conda_if_missing() { echo "Updating conda installation." conda update conda fi + # Make sure we use the correct Anaconda environment. + if [[ -z "$(conda info --envs | grep panoptes-env)" ]] ; then + conda create -y -n panoptes-env python=3 + fi + add_to_profile "source activate panoptes-env" + source activate panoptes-env } # Fetches and configures the latest version of cfitsio; this allows us to @@ -361,6 +394,18 @@ if [[ "${DO_CONDA}" -eq 1 ]] ; then fi +if [[ "${DO_PIP_REQUIREMENTS}" -eq 1 ]] ; then + # Upgrade pip itself before installing other python packages. + pip install -U pip + # TODO(jamessynge): Move this script and needed text files into an install + # directory. + # TODO(wtgee): Consider whether to inline the needed text files into this + # file, and add git clone of the PANOPTES repos, so that the user can do + # an install with just about two commands (wget script, then run script). + pip install -r "${POCS}/requirements.txt" +fi + + if [[ "${DO_CFITSIO}" -eq 1 ]] ; then install_latest_cfitsio fi @@ -376,17 +421,6 @@ if [[ "${DO_ASTROMETRY_INDICES}" -eq 1 ]] ; then fi -if [[ "${DO_PIP_REQUIREMENTS}" -eq 1 ]] ; then - # Upgrade pip itself before installing other python packages. - pip install -U pip - # TODO(jamessynge): Move this script and needed text files into an install - # directory. - # TODO(wtgee): Consider whether to inline the needed text files into this - # file, and add git clone of the PANOPTES repos, so that the user can do - # an install with just about two commands (wget script, then run script). - pip install -r "${POCS}/requirements.txt" -fi - cd "${PANDIR}" rmdir --ignore-fail-on-non-empty "${PANDIR}/tmp" From d223d2921507e1956ab9fd856057fd3bf683a039 Mon Sep 17 00:00:00 2001 From: jamessynge Date: Fri, 24 Nov 2017 21:38:45 -0500 Subject: [PATCH 12/15] Modify data.py to install both sets of index files. Create panoptes-env in conda. --- pocs/utils/data.py | 15 +++-- scripts/install/install-dependencies.sh | 84 ++++++++++++------------- 2 files changed, 50 insertions(+), 49 deletions(-) diff --git a/pocs/utils/data.py b/pocs/utils/data.py index dc11e21b6..b0b2487b1 100644 --- a/pocs/utils/data.py +++ b/pocs/utils/data.py @@ -12,18 +12,22 @@ def download_all_files(data_folder=None): if data_folder is None: data_folder = "{}/astrometry/data".format(os.getenv('PANDIR')) - for i in range(4210, 4219): - fn = 'index-{}.fits'.format(i) - dest = "{}/{}".format(data_folder, fn) - + def download_one_file(fn): + dest = "{}/{}".format(data_folder, os.path.basename(fn)) if not os.path.exists(dest): - url = "http://data.astrometry.net/4200/{}".format(fn) + url = "http://data.astrometry.net/{}".format(fn) df = data.download_file(url) try: shutil.move(df, dest) except OSError as e: print("Problem saving. (Maybe permissions?): {}".format(e)) + for i in range(4112, 4119): + download_one_file('4100/index-{}.fits'.format(i)) + + for i in range(4212, 4219): + download_one_file('4200/index-{}.fits'.format(i)) + if __name__ == '__main__': parser = argparse.ArgumentParser( @@ -38,3 +42,4 @@ def download_all_files(data_folder=None): print("{} does not exist.".format(args.folder)) download_all_files(data_folder=args.folder) + diff --git a/scripts/install/install-dependencies.sh b/scripts/install/install-dependencies.sh index 49ffc4089..4cb97f6fb 100755 --- a/scripts/install/install-dependencies.sh +++ b/scripts/install/install-dependencies.sh @@ -16,13 +16,14 @@ cd "${PANDIR}" ASTROMETRY_VERSION="0.72" INSTALL_PREFIX="/usr/local" + DO_APT_GET=1 DO_MONGODB=1 DO_CONDA=1 -DO_CFITSIO=0 +DO_REBUILD_CONDA_ENV=0 +DO_CFITSIO=0 # Disabled in favor of installing with apt-get. DO_ASTROMETRY=1 DO_ASTROMETRY_INDICES=1 -DO_ASTROMETRY_FULL_INDICES=0 DO_PIP_REQUIREMENTS=1 function echo_bar() { @@ -149,18 +150,7 @@ function install_conda_if_missing() { CONDA_BIN="$(which conda)" # Assuming is a python3 version if found. if [[ -z "${CONDA_BIN}" ]] ; then install_conda - else - echo_bar - echo - echo "Updating conda installation." - conda update conda - fi - # Make sure we use the correct Anaconda environment. - if [[ -z "$(conda info --envs | grep panoptes-env)" ]] ; then - conda create -y -n panoptes-env python=3 fi - add_to_profile "source activate panoptes-env" - source activate panoptes-env } # Fetches and configures the latest version of cfitsio; this allows us to @@ -270,25 +260,7 @@ function install_astrometry() { function install_astrometry_indices() { mkdir -p "${PANDIR}/astrometry/data" - cd "${PANDIR}/astrometry/data" - # See http://astrometry.net/doc/readme.html#getting-index-files - # for info on the index files you might need. - if [[ "${DO_ASTROMETRY_FULL_INDICES}" -eq 1 ]] ; then - if [[ ! -f index-4107.fits ]] ; then - curl --remote-name \ - http://broiler.astrometry.net/~dstn/4100/index-41[07-19].fits - fi - if [[ ! -f astrometry/data/index-4208.fits ]] ; then - curl --remote-name \ - http://broiler.astrometry.net/~dstn/4200/index-42[08-19].fits - fi - else - # Install wide field indices. If we change to using longer focal length - if [[ ! -f index-4116.fits ]] ; then - curl --remote-name \ - http://broiler.astrometry.net/~dstn/4100/index-411[6-9].fits - fi - fi + python "${POCS}/pocs/utils/data.py" } #------------------------------------------------------------------------------- @@ -306,9 +278,9 @@ function show_help() { echo "--no-mongodb don't install and start mongodb server" echo "--no-cfitsio don't install the latest version of cfitsio" echo "--no-conda don't install the latest version of Anaconda" + echo "--rebuild-conda-env rebuild the panoptes-env" echo "--no-astrometry don't install astrometry.net software" echo "--no-astrometry-indices don't install astrometry.net indices" - echo "--astrometry-full-indices install full indices, for narrow and wide fields" echo "--no-pip-requirements don't install python packages" } @@ -351,22 +323,20 @@ while test ${#} -gt 0; do DO_CONDA=0 shift ;; - --no-astrometry) - DO_ASTROMETRY=0 + --rebuild-conda-env) + DO_REBUILD_CONDA_ENV=1 shift ;; - --no-astrometry-ind*) - DO_ASTROMETRY_INDICES=0 - DO_ASTROMETRY_FULL_INDICES=0 + --no-pip-requirements) + DO_PIP_REQUIREMENTS=0 shift ;; - --astrometry-full-ind*) - DO_ASTROMETRY_FULL_INDICES=1 - DO_ASTROMETRY_INDICES=1 + --no-astrometry) + DO_ASTROMETRY=0 shift ;; - --no-pip-requirements) - DO_PIP_REQUIREMENTS=0 + --no-astrometry-ind*) + DO_ASTROMETRY_INDICES=0 shift ;; *) @@ -394,6 +364,30 @@ if [[ "${DO_CONDA}" -eq 1 ]] ; then fi +# Make sure we use the correct Anaconda environment. +DO_CREATE_CONDA_ENV=0 +if [[ -z "$(conda info --envs | grep panoptes-env)" ]] ; then + DO_CREATE_CONDA_ENV=1 +elif [[ "${DO_REBUILD_CONDA_ENV}" -eq 1 ]] ; then + source activate root + conda remove --all --yes --quiet -n panoptes-env + DO_CREATE_CONDA_ENV=1 +fi +if [[ "${DO_CREATE_CONDA_ENV}" -eq 1 ]] ; then + conda create --yes -n panoptes-env python=3 +fi + +add_to_profile "source activate panoptes-env" +source activate panoptes-env + +if [[ "${DO_CONDA}" -eq 1 || "${DO_CREATE_CONDA_ENV}" -eq 1 || \ + "${DO_PIP_REQUIREMENTS}" -eq 1 ]] ; then + echo_bar + echo + echo "Updating conda installation." + conda update --quiet --all --yes +fi + if [[ "${DO_PIP_REQUIREMENTS}" -eq 1 ]] ; then # Upgrade pip itself before installing other python packages. pip install -U pip @@ -427,8 +421,10 @@ rmdir --ignore-fail-on-non-empty "${PANDIR}/tmp" set +x echo echo_bar +echo_bar echo -echo "Remember to update PATH in your shell before running tests." +echo "Remember to update PATH in your shell before running tests" +echo "and to run \"source activate panoptes-env\"" echo exit From b472eea7194108b606b4d1f8797cbee556824e11 Mon Sep 17 00:00:00 2001 From: jamessynge Date: Fri, 24 Nov 2017 22:02:53 -0500 Subject: [PATCH 13/15] Fix path in travis.yml. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c95a277c3..7aa4863e0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -59,7 +59,7 @@ install: - source activate test-environment - cd $TRAVIS_BUILD_DIR - - pip install -r scripts/install/requirements.txt + - pip install -r requirements.txt - python setup.py install script: - coverage run setup.py test From 384b27d1a98a709451ed98a03149a49e92d5c14c Mon Sep 17 00:00:00 2001 From: jamessynge Date: Fri, 24 Nov 2017 22:24:05 -0500 Subject: [PATCH 14/15] Remove trailing blank line. --- pocs/utils/data.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pocs/utils/data.py b/pocs/utils/data.py index b0b2487b1..30c32c1bb 100644 --- a/pocs/utils/data.py +++ b/pocs/utils/data.py @@ -42,4 +42,3 @@ def download_one_file(fn): print("{} does not exist.".format(args.folder)) download_all_files(data_folder=args.folder) - From 5d34a22b68f3b4c7d8bd204c24689d31657c3350 Mon Sep 17 00:00:00 2001 From: jamessynge Date: Sat, 25 Nov 2017 17:50:58 -0500 Subject: [PATCH 15/15] Fetch more of the 4100 series indices, needed for some tests that use images that have narrow fields, unlike regular Panoptes images. --- pocs/utils/data.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pocs/utils/data.py b/pocs/utils/data.py index 30c32c1bb..b00f6a909 100644 --- a/pocs/utils/data.py +++ b/pocs/utils/data.py @@ -6,7 +6,7 @@ from astropy.utils import data -def download_all_files(data_folder=None): +def download_all_files(data_folder=None, wide_field=True, narrow_field=False): download_IERS_A() if data_folder is None: @@ -22,11 +22,13 @@ def download_one_file(fn): except OSError as e: print("Problem saving. (Maybe permissions?): {}".format(e)) - for i in range(4112, 4119): - download_one_file('4100/index-{}.fits'.format(i)) + if wide_field: + for i in range(4110, 4119): + download_one_file('4100/index-{}.fits'.format(i)) - for i in range(4212, 4219): - download_one_file('4200/index-{}.fits'.format(i)) + if narrow_field: + for i in range(4210, 4219): + download_one_file('4200/index-{}.fits'.format(i)) if __name__ == '__main__':