diff --git a/requirements.txt b/requirements.txt index 186ea552d..f8ead20aa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,26 +1,14 @@ -astroplan -astropy >= 3.0.0 -ccdproc -codecov -coveralls -dateparser +# These are packages that are NOT available via our +# choice of conda repositories; there are less well +# known repositories that do offer them, but that is +# more of a security risk. + +# Thin Python wrapper for ffmpeg. We *could* replace with our own wrapper. ffmpy -gcloud -google-cloud-storage -matplotlib >= 2.0.0 + +# Mock object support for Python testing. mocket -numpy >= 1.12.1 -pycodestyle == 2.3.1 -pymongo >= 3.2.2 -pyserial >= 3.1.1 -pytest >= 3.4.0 -python_dateutil >= 2.5.3 -PyYAML >= 3.11 -pyzmq >= 15.3.0 -readline -requests -scikit_image >= 0.12.3 -scipy >= 0.17.1 + +# State machine library for Python. transitions >= 0.4.0 -tweepy -wcsaxes + diff --git a/scripts/install/apt-packages-list.txt b/scripts/install/apt-packages-list.txt index 556a02e9a..df9d12926 100644 --- a/scripts/install/apt-packages-list.txt +++ b/scripts/install/apt-packages-list.txt @@ -2,7 +2,8 @@ # Lines starting with # are comments as are blank lines, # and # starts trailing comments. -# dcraw decodes Canon raw images +# dcraw decodes Canon raw images. See also libraw.org for an alternate source +# of the underlying algorithms. Neither has an up-to-date conda package. dcraw # gphoto2 enables remote control of cameras, including the Canon DSLRs that @@ -11,31 +12,34 @@ dcraw # old relative to the latest stable version. gphoto2-updater is an install # script that we might want to use if we need a newer version. For more info, # see: https://github.com/gonzalo/gphoto2-updater +# There is an old conda package. gphoto2 # Packages needed according to http://astrometry.net/doc/build.html -libbz2-dev -libbz2-dev -libcfitsio-dev -libjpeg-dev -libnetpbm10-dev -libpng12-dev -netpbm -python-dev -python-numpy -python-pyfits -swig -zlib1g-dev - -# Cairo is a graphics library, and matplotlib can use it as a backend -# for rendering. -libcairo2-dev - -# exiftool is used in a couple of places to extract info from the .cr2 file -exiftool +libbz2-dev # Not available in our conda repos. +libcfitsio-dev # Probably already satisfied in conda. +libjpeg-dev # Conda has libjpeg-turbo, apparently the same API but faster. +libnetpbm10-dev # Used for making pretty pictures of solutions. +libpng-dev # Used for making pretty pictures of solutions. ALREADY installed by conda. +netpbm # Used for making pretty pictures of solutions. +python-dev # Probably already satisfied in conda. +python-numpy # Probably already satisfied in conda. +python-pyfits # Not available in our conda repos. +swig # Available from conda. +zlib1g-dev # Available from conda as zlib. + +# Cairo is a 2D graphics library, and matplotlib can use it as a backend +# for rendering, and used by astrometry for pretty pictures of solutions. +# Stable, but not latest, version available from conda. +libcairo2-dev + +# exiftool is used in a couple of places to extract info from the .cr2 file. +# Consider using Python package exifread instead, which doesn't require exiftool +exiftool # We use cfitsio command line tools (e.g. fpack) from POCS, in addition to the # uses of the cfitsio libraries by astrometry above. +# 'conda install cfitsio' brings in fpack. libcfitsio-bin # These are used if you want to setup SSH access into the computer (pretty @@ -44,6 +48,8 @@ openssh-client openssh-server # Graphviz is used for rendering the state machine of POCS. +# Do we need both of these? Do we use the command-line tools in graphviz, +# or just the pygraphviz library? graphviz libgraphviz-dev @@ -51,9 +57,11 @@ libgraphviz-dev pkg-config # Improves interaction with pocs_shell (via readline). +# Available from conda as ncurses. libncurses5-dev # These support the Python package "Shapely", used in PIAA (called from POCS). +# Conda has geos 3.6. Not sure if this covers libgeos-c1v5 and libgeos-dev. libgeos-3.* libgeos-c1v5 libgeos-dev @@ -64,7 +72,14 @@ byobu tmux # Used for creating a timelapse. +# Available from conda. ffmpeg # Used for colorizing log files. grc + +# https://jupyter.org/ +# Lots of jupyter packages available from conda, not sure whether this includes +# jupyter-notebook features exactly as we need. +jupyter-notebook + diff --git a/scripts/install/conda-packages-list.txt b/scripts/install/conda-packages-list.txt index d820bb9bb..4e6ffc030 100644 --- a/scripts/install/conda-packages-list.txt +++ b/scripts/install/conda-packages-list.txt @@ -1,2 +1,38 @@ +# These are packages that we install with conda. They include both Python +# packages and non-Python libraries on which we depend (usually via a +# Python package depending on some native code). + +astroplan +astropy >= 3.0.0 +ccdproc +cfitsio # This is an experiment. Requires adjusting apt-packages-list. +codecov +coveralls +cython +dateparser +google-cloud-sdk +google-cloud-storage +matplotlib >= 2.0.0 +numpy >= 1.12.1 +pycodestyle == 2.3.1 +pymongo >= 3.2.2 +pyserial >= 3.1.1 +pytest >= 3.4.0 pandas +python +python-dateutil >= 2.5.3 +PyYAML >= 3.11 +pyzmq >= 15.3.0 +readline +requests +scikit-image >= 0.12.3 +scipy >= 0.17.1 + +# Jupyter-notebook requires Tornado versions between 4 and 5. See: +# https://stackoverflow.com/questions/49141525/install-jupyter-notebook-on-miniconda +# https://github.com/ipython/ipython/issues/8249 +# https://github.com/jupyter/help/issues/324 +tornado == 4.5.3 +tweepy +wcsaxes diff --git a/scripts/install/install-dependencies.sh b/scripts/install/install-dependencies.sh index d95f562de..080799f14 100755 --- a/scripts/install/install-dependencies.sh +++ b/scripts/install/install-dependencies.sh @@ -1,5 +1,8 @@ #!/bin/bash -e +# Run with --help to see your options. With no options, does a complete install +# of dependencies, though attempts to reuse existing installs. + THIS_DIR="$(dirname $(readlink -f "${0}"))" THIS_PROGRAM="$(basename "${0}")" @@ -12,8 +15,6 @@ fi mkdir -p "${PANDIR}" cd "${PANDIR}" -# TODO(jamessynge): Add flags to control behavior, such as skipping apt-get. - ASTROMETRY_VERSION="0.72" INSTALL_PREFIX="/usr/local" @@ -27,40 +28,88 @@ DO_ASTROMETRY=1 DO_ASTROMETRY_INDICES=1 DO_PIP_REQUIREMENTS=1 +# Which bash file do we need to modify? The last found here is the one that +# bash executes for login shells, and so provides the environment for +# all processes under that. +THE_PROFILE="${HOME}/.profile" +if [[ -f "${HOME}/.bash_login" ]] ; then + THE_PROFILE="${HOME}/.bash_login" +fi +if [[ -f "${HOME}/.bash_profile" ]] ; then + THE_PROFILE="${HOME}/.bash_profile" +fi + function echo_bar() { - eval $(resize|grep COLUMNS=) + if [[ -n "$(which resize)" ]] ; then + eval $(resize|grep COLUMNS=) + elif [[ -n "$(which stty)" ]] ; then + COLUMNS="$(stty size | cut '-d ' -f2)" + fi printf "%${COLUMNS:-80}s\n" | tr ' ' '#' } -# Append $1 to .profile -function add_to_profile() { - local -r the_line="${1}" - local -r the_file="${HOME}/.profile" - if [[ ! -f "${the_file}" ]] ; then - touch "${the_file}" +function ensure_profile_exists() { + if [[ ! -f "${THE_PROFILE}" ]] ; then + touch "${THE_PROFILE}" fi - if [[ -z "$(fgrep -- "${the_line}" "${the_file}")" ]] ; then - echo >>"${the_file}" " -# Added by PANOPTES install-dependencies.sh -${the_line}" - echo "Appended to ${the_file}: ${the_line}" +} + +function profile_contains_text() { + local -r target_text="${1}" + if [[ -n "$(fgrep -- "${target_text}" "${THE_PROFILE}")" ]] ; then + return 0 else - echo "Already in ${the_file}: ${the_line}" + return 1 fi } +function add_to_profile_before_target() { + local -r new_text="${1}" + local -r target_text="${2}" + if profile_contains_text "${new_text}" ; then + echo "Already in ${THE_PROFILE}: ${new_text}" + return 0 + fi + ensure_profile_exists + # This backup is just for debugging (i.e. showing the before and after + # diff). + local -r the_backup="$(mktemp "${THE_PROFILE}.pre-edit.XXXXX")" + cp -p "${THE_PROFILE}" "${the_backup}" + if profile_contains_text "${target_text}" ; then + # Add just before the target text. + sed -i "/${target_text}/i \ +# Added by PANOPTES install-dependencies.sh\n\ +${new_text}\n" "${THE_PROFILE}" + else + # Append to the end of the file. + echo >>"${THE_PROFILE}" " +# Added by PANOPTES install-dependencies.sh +${new_text}" + fi + # Again, this diff is just for debugging. + echo "Modified ${THE_PROFILE}:" + echo + diff -u "${the_backup}" "${THE_PROFILE}" || /bin/true + echo + rm "${the_backup}" +} + +# Add $1 to .profile before where bashrc is invoked. +# This assumes a standard (default) .profile. +function add_to_profile_before_bashrc() { + add_to_profile_before_target "${1}" "# if running bash" +} + # Append $1 to PATH and write command to do the same to .profile. function add_to_PATH() { local -r the_dir="$(readlink -f "${1}")" - add_to_profile "PATH=\"${the_dir}:\${PATH}\"" + add_to_profile_before_bashrc "PATH=\"${the_dir}:\${PATH}\"" PATH="${the_dir}:${PATH}" return 0 local -r the_file="${HOME}/.profile" - if [[ ! -f "${the_file}" ]] ; then - touch "${the_file}" - fi + ensure_profile_exists if [[ -z "$(egrep -- "PATH=.*${the_dir}" "${the_file}")" ]] ; then echo >>"${the_file}" " # Added by PANOPTES install-dependencies.sh @@ -94,19 +143,47 @@ function install_apt_packages() { } function install_mongodb() { - # This is based on https://www.howtoforge.com/tutorial/install-mongodb-on-ubuntu-16.04/ + # This is based on https://www.howtoforge.com/tutorial/install-mongodb-on-ubuntu/ # Note this function does not configure mongodb itself, i.e. no users or # security settings. echo_bar + local MONGO_KEY="" + local MONGO_URL="http://repo.mongodb.org/apt/ubuntu" + local MONGO_VERSION="" + local MONGO_SOURCE_PATH="" + local LSB_RELEASE="" + if [[ -n "$(which lsb_release)" ]] ; then + LSB_RELEASE="$(lsb_release -sc)" + fi + if [[ "${LSB_RELEASE}" = "xenial" ]] ; then + MONGO_KEY=2930ADAE8CAF5059EE73BB4B58712A2291FA4AD5 + MONGO_VERSION=3.6 + elif [[ "${LSB_RELEASE}" = "bionic" ]] ; then + MONGO_KEY=9DA31620334BD75D9DCB49F368818C72E52529D4 + MONGO_VERSION=4.0 + else + echo "ERROR: don't know which version of MongoDB to install." + return 1 + fi + MONGO_URL+=" ${LSB_RELEASE}/mongodb-org/${MONGO_VERSION}" + MONGO_SOURCE_PATH="/etc/apt/sources.list.d/mongodb-org-${MONGO_VERSION}.list" + echo " -Installing mongodb, for which several commands require sudo, so -you may be prompted for you password. +Installing MongoDB ${MONGO_VERSION}, for which several commands require sudo, +so you may be prompted for you password. Starting by telling APT where to find +the MongoDB packages. +" + sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv ${MONGO_KEY} + echo "deb ${MONGO_URL} multiverse" | sudo tee "${MONGO_SOURCE_PATH}" + echo " +Updating the list of packages so APT finds the MongoDB packages, +then installing MongoDB. " - sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv EA312927 - echo "deb http://repo.mongodb.org/apt/ubuntu "$(lsb_release -sc)"/mongodb-org/3.2 multiverse" \ - | sudo tee /etc/apt/sources.list.d/mongodb-org-3.2.list sudo apt-get update sudo apt-get install -y mongodb-org + echo " +MongoDB is installed, now updating the config and starting mongod. +" echo "[Unit] Description=High-performance, schema-free document-oriented database After=network.target @@ -135,22 +212,45 @@ function maybe_install_mongodb() { # the set to install. function install_conda() { local -r the_script="${PANDIR}/tmp/miniconda.sh" + local -r the_destination="${PANDIR}/miniconda" + if [[ -d "${the_destination}" ]] ; then + echo_bar + echo + echo "Removing previous miniconda installation from ${the_destination}" + rm -rf "${the_destination}" + fi echo_bar echo echo "Installing miniconda. License at: https://conda.io/docs/license.html" wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \ -O "${the_script}" - bash "${the_script}" -b -p "${PANDIR}/miniconda" + bash "${the_script}" -b -p "${the_destination}" rm "${the_script}" - add_to_PATH miniconda/bin + + # As per the Anaconda 4.4 release notes, one is supposed to add the following + # to .bash_profile, .bashrc or whereever is appropriate: + # . $CONDA_LOCATION/etc/profile.d/conda.sh + # conda activate base + # Where CONDA_LOCATION is where Anaconda or miniconda was installed. + # We do the first step here, but obviously only we we are actually + # installing conda. The second step will be done unconditionally + # elsewhere in this file. + + . ${the_destination}/etc/profile.d/conda.sh + add_to_profile_before_bashrc \ + ". ${the_destination}/etc/profile.d/conda.sh" } function install_conda_if_missing() { # Intall latest version of Miniconda (Anaconda with # fewer packages; any that are needed can then be installed). - CONDA_BIN="$(which conda)" # Assuming is a python3 version if found. - if [[ -z "${CONDA_BIN}" ]] ; then + # Note that if conda is found, we assume that a Python 3 version is installed. + if [[ -z "$(which conda)" ]] ; then install_conda + else + echo_bar + echo + echo "Reusing existing conda:" "$(which conda)" fi } @@ -369,48 +469,86 @@ if [[ "${DO_CONDA}" -eq 1 ]] ; then install_conda_if_missing fi +# Add the astropy channel, i.e. an additional repository in which to +# look for packages. With conda 4.1.0 and later, by default the highest +# priority repository that contains a package is used as the source for +# that package, even if there is a newer version in a lower priority +# package. And by default the most recently added repository is treated +# as the highest priority repository. Here we use prepend to be clear +# that we want astropy to be highest priority. +if [[ -n "$(conda config --show channels | fgrep astropy)" ]] ; then + conda config --remove channels astropy +fi +conda config --prepend channels astropy + +# And put conda-forge at the back of the line. +if [[ -n "$(conda config --show channels | fgrep conda-forge)" ]] ; then + conda config --remove channels conda-forge +fi +conda config --append channels conda-forge + +# Use the base Anaconda environment until we're ready to +# work with the PANOPTES environment. +conda activate base # Make sure we use the correct Anaconda environment. DO_CREATE_CONDA_ENV=0 -if [[ -z "$(conda info --envs | grep panoptes-env)" ]] ; then +if [[ -z "$(conda env list | grep panoptes-env)" ]] ; then DO_CREATE_CONDA_ENV=1 elif [[ "${DO_REBUILD_CONDA_ENV}" -eq 1 ]] ; then - source activate root + echo_bar + echo + echo "Removing previous PANOPTES conda env" conda remove --all --yes --quiet -n panoptes-env DO_CREATE_CONDA_ENV=1 fi + +if [[ "${DO_CONDA}" -eq 1 || "${DO_CREATE_CONDA_ENV}" -eq 1 || \ + "${DO_PIP_REQUIREMENTS}" -eq 1 ]] ; then + echo_bar + echo + echo "Updating base conda installation." + conda update --quiet --yes -n base conda +fi + if [[ "${DO_CREATE_CONDA_ENV}" -eq 1 ]] ; then + echo_bar + echo + echo "Creating conda env for PANOPTES: panoptes-env" conda create --yes -n panoptes-env python=3 fi -add_to_profile "source activate panoptes-env" -source activate panoptes-env +add_to_profile_before_bashrc "conda activate panoptes-env" +conda activate panoptes-env if [[ "${DO_CONDA}" -eq 1 || "${DO_CREATE_CONDA_ENV}" -eq 1 || \ "${DO_PIP_REQUIREMENTS}" -eq 1 ]] ; then echo_bar echo - echo "Updating conda installation." - conda update --quiet --all --yes + echo "Updating packages in panoptes-env." + conda update -n panoptes-env --quiet --yes conda + conda update -n panoptes-env --quiet --yes --all fi - if [[ "${DO_INSTALL_CONDA_PACKAGES}" -eq 1 ]] ; then echo_bar echo - echo "Installing conda packages" + echo "Installing conda packages needed for PANOPTES" conda install --yes "--file=${THIS_DIR}/conda-packages-list.txt" fi +echo_bar +echo "PATH=$PATH" +echo_bar + +exit if [[ "${DO_PIP_REQUIREMENTS}" -eq 1 ]] ; then + echo_bar + echo + echo "Installing Python packages using pip" # Upgrade pip itself before installing other python packages. pip install -U pip - # TODO(jamessynge): Move this script and needed text files into an install - # directory. - # TODO(wtgee): Consider whether to inline the needed text files into this - # file, and add git clone of the PANOPTES repos, so that the user can do - # an install with just about two commands (wget script, then run script). pip install -r "${POCS}/requirements.txt" fi @@ -437,10 +575,11 @@ set +x echo echo_bar echo_bar -echo -echo "Remember to update PATH in your shell before running tests" -echo "and to run \"source activate panoptes-env\"" -echo +echo " +Your ${THE_PROFILE} has been modified. To pickup that change, +please logout completely, then log back in. Don't just open +a new terminal window. +" exit