Add scripts/install/auto-install.sh (#824)

Auto-install script for POCS, to be executed as follows:

```
export AUTO_INSTALL_RAW_URL="https://raw.githubusercontent.com/panoptes/POCS/develop/scripts/install/auto-install.sh"
wget -q -O - "${AUTO_INSTALL_RAW_URL}" | bash
```

If executed by a user other than the panoptes user, will present the user with instructions for creating the panoptes user, logging in as panoptes, then re-executing the command above as panoptes.

If executed by the panoptes user, will install git, clone POCS, then run the install-dependencies.sh script in POCS.

This commit includes changes to various dependency specifications (e.g. urllib3) to deal with issues that were detected while testing this, and to install-dependencies.sh, et al, for the same reason.

requirements.txt

@@ -22,4 +22,8 @@ scikit_image >= 0.12.3
 scipy >= 0.17.1
 transitions >= 0.4.0
 tweepy
+
+# requests 2.21 doesn't like urllib3 >=1.25
+urllib3 >= 1.24, < 1.25  
+
 wcsaxes

scripts/install/apt-packages-list.txt

@@ -78,3 +78,6 @@ coreutils
 # includes this file).
 git
 
+# Installing some POCS dependencies (esp. astroscrappy) require a C compiler,
+# for which we install gcc via the build-essentials package.
+build-essential

scripts/install/auto-install.sh

@@ -0,0 +1,316 @@
+#!/bin/bash
+# auto-install.sh installs git if it isn't installed, then clones POCS and
+# installs its dependencies.
+#
+# To fetch this script from github and execute it immediately,
+# run these commands:
+#
+# export POCS_GITHUB_USER="panoptes"
+# export POCS_BRANCH="develop"
+# BASE_RAW_POCS_URL="https://raw.githubusercontent.com/${POCS_GITHUB_USER}/POCS/${POCS_BRANCH}"
+# export AUTO_INSTALL_RAW_URL="${BASE_RAW_POCS_URL}/scripts/install/auto-install.sh"
+# wget -q -O - "${AUTO_INSTALL_RAW_URL}" | bash
+
+################################################################################
+# Functions COPIED from install-helper-functions.sh
+# Print the disk location of the first arg.
+function safe_which() {
+  type -p "${1}" || /bin/true
+}
+
+# Print a separator bar of # characters.
+function echo_bar() {
+  local terminal_width="${COLUMNS}"
+  if [ -z "${terminal_width}" ] && [ -n "${TERM}" ]
+  then
+    if [[ -n "$(safe_which tput)" ]]
+    then
+      terminal_width="$(tput cols)"
+    elif [[ -n "$(safe_which resize)" ]]
+    then
+      terminal_width="$(resize 2>/dev/null | grep COLUMNS= | cut -d= -f2)"
+    elif [[ -n "$(safe_which stty)" ]]
+    then
+      terminal_width="$(stty size 2>/dev/null | cut '-d ' -f2)"
+    fi
+  fi
+  printf "%${terminal_width:-80}s\n" | tr ' ' '#'
+}
+
+################################################################################
+
+function do_sudo() {
+  if [ "$(id -u -n)" == "root" ] ; then
+    echo "Running ${*}"
+    "$@"
+  else
+    echo <<ENDOFMESSAGE
+Running:
+    sudo ${*}
+You may be prompted for your password.
+ENDOFMESSAGE
+    (set -x ; sudo "$@")
+  fi
+}
+
+function clone_or_update() {
+  local -r REPO_DIR="${1}"
+  local -r REPO_URL="${2}"
+  local -r ORIGIN_NAME="${3}"
+  local -r BRANCH="${4}"
+
+  echo_bar
+
+  if [ ! -d "${REPO_DIR}/.git" ]
+  then
+    if [ -d "${REPO_DIR}" ]
+    then
+      echo 2> "
+The directory (${REPO_DIR}) already exists, but doesn't appear
+to be a valid git repository. Please remove it (or move it out
+of the way) and re-run this script.
+  "
+      exit 1
+    fi
+    echo "
+Cloning ${REPO_URL} into ${REPO_DIR}
+"
+    (set -x ; git clone --origin "${ORIGIN_NAME}" "${REPO_URL}" "${REPO_DIR}")
+    cd "${REPO_DIR}"
+    git checkout "${BRANCH}"
+  else
+    echo "
+Pulling the latest software into the worktree at ${REPO_DIR}.
+"
+    cd "${REPO_DIR}"
+    git fetch --all
+    git checkout "${BRANCH}"
+    git pull
+  fi
+}
+
+function maybe_print_example() {
+  if [[ -n "${AUTO_INSTALL_RAW_URL}" ]]
+  then
+    >&2 cat <<ENDOFMESSAGE
+For example:
+
+  wget -q -O - "${AUTO_INSTALL_RAW_URL}" | bash
+
+ENDOFMESSAGE
+  fi
+}
+
+function ensure_ownership() {
+  local -r path="${1}"
+  local -r user="${2}"
+  local -r group="${3}"
+  echo_bar
+  echo "
+Ensuring that ${path} is owned by user ${user}, and by group ${group}.
+"
+  time do_sudo chown --recursive "${user}:${group}" "${path}"
+  echo
+}
+
+function ensure_directory_ownership() {
+  local -r path="${1}"
+  local -r user="${2}"
+  local -r group="${3}"
+
+  echo_bar
+  echo "
+Ensuring that directory ${path} exists.
+"
+  if [ ! -d "${path}" ]
+  then
+    do_sudo mkdir -p "${path}"
+    echo
+  fi
+
+  ensure_ownership "${path}" "${user}" "${group}"
+}
+
+function already_have_a_repo() {
+  >&2 cat <<ENDOFMESSAGE
+
+The purpose of this script is to make sure git is installed, then to clone the
+POCS and PAWS git repositories (which contain the PANOPTES software for a
+PANOPTES telescope), install software that POCS and PAWS depend upon, and to
+run tests. However, there is already a $1 repository located at:
+
+    $2
+
+You may be able to install those dependencies by running this command:
+
+    ${POCS}/scripts/install/install-dependencies.sh
+
+ENDOFMESSAGE
+}
+
+
+
+# End of functions.
+################################################################################
+
+# Exit immediately if a command fails:
+set -e
+
+# COPIED from default-env-vars.sh
+[[ -z "${PANUSER}" ]] && export PANUSER="panoptes"      # Default user
+[[ -z "${PANDIR}" ]] && export PANDIR="/var/panoptes"   # Main Dir
+[[ -z "${PANLOG}" ]] && export PANLOG="${PANDIR}/logs"  # Log files
+[[ -z "${POCS}" ]] && export POCS="${PANDIR}/POCS"      # Main Observatory Control
+[[ -z "${PAWS}" ]] && export PAWS="${PANDIR}/PAWS"      # Web Interface
+
+
+# Do we need to create the user PANUSER?
+
+if ! id -u "${PANUSER}" 2>/dev/null 1>/dev/null
+then
+  >&2 cat <<ENDOFMESSAGE
+The user ${PANUSER} doesn't exist yet. Please create it by running:
+
+  sudo adduser --shell /bin/bash --add_extra_groups ${PANUSER}
+
+You will be prompted to enter a password for this new user. You may also be
+prompted to provide contact info as if this were a user on a shared computer
+(e.g. Full Name and Home Phone). Press Enter to leave these unset.
+
+After the command completes, we need to ensure that the user is a
+member of some key groups:
+
+  sudo usermod --append --groups adm,dialout,plugdev,sudo ${PANUSER}
+
+Next, login as ${PANUSER} and re-execute the command you used to
+run this script.
+ENDOFMESSAGE
+
+  maybe_print_example
+  exit 1
+fi
+
+# Do we need to login as the user PANUSER?
+
+if [[ "$(whoami)" != "${PANUSER}" ]]
+then
+  echo >&2 "
+This script should be executed by the user ${PANUSER}, not as $(whoami).
+Please login as ${PANUSER} and re-execute the command you used to
+run this script.
+"
+
+  maybe_print_example
+  exit 1
+fi
+
+# I (James Synge) have noticed that if $HOME/.cache/ doesn't exist, then at some
+# point during the install it gets created, but is owned by root:root, which
+# then messes up later steps that attempt to use it. So, we make sure here that
+# it exists with the correct ownership.
+
+PANGROUP="$(id -gn "${PANUSER}")"
+ensure_directory_ownership "${HOME}/.cache" "${PANUSER}" "${PANGROUP}"
+
+# Do the same with PANDIR (usually /var/panoptes).
+ensure_directory_ownership "${PANDIR}" "${PANUSER}" "${PANGROUP}"
+
+################################################################################
+# Is there already a POCS or PAWS repository? If so, the user should not need
+# this script.
+if [[ -d "${POCS}/.git" ]]
+then
+  already_have_a_repo POCS "${POCS}"
+  exit 1
+fi
+
+if [[ -d "${PAWS}/.git" ]]
+then
+  already_have_a_repo PAWS "${PAWS}"
+  exit 1
+fi
+
+# Install git if necessary.
+
+if [ ! -x "$(safe_which git)" ]
+then
+  echo_bar
+  echo "
+git is not installed. Updating package cache, then installing git.
+"
+  do_sudo apt-get update
+  echo
+  echo
+  do_sudo apt-get install -y git
+  echo
+fi
+
+# Clone the POCS repo from github.
+
+clone_or_update "${POCS}" https://github.com/panoptes/POCS.git upstream develop
+
+# If the user specified another repo via POCS_GITHUB_USER, use that as
+# the origin, and checkout the branch POCS_BRANCH.
+
+if [[ -n "${POCS_GITHUB_USER}" && "${POCS_GITHUB_USER}" != "panoptes" ]]
+then
+  cd "${POCS}"
+  git remote add -f origin "https://github.com/${POCS_GITHUB_USER}/POCS.git" || true
+  git checkout "origin/${POCS_BRANCH:-develop}"
+fi
+
+# Clone PAWS too, but only from the upstream.
+clone_or_update \
+    "${PAWS}" "https://github.com/panoptes/PAWS.git" upstream develop
+
+################################################################################
+echo
+echo_bar
+echo_bar
+echo "
+Executing ${POCS}/scripts/install/install-dependencies.sh, which will
+install the tools needed to run POCS.
+"
+
+${POCS}/scripts/install/install-dependencies.sh
+
+echo
+echo
+echo_bar
+echo_bar
+echo "
+
+Testing the installation of POCS. This can easily take 5 minutes on an Intel NUC
+or longer on a Raspberry Pi.
+
+"
+
+if [[ ! -e  "${POCS}/conf_files/pocs_local.yaml" ]]
+then
+  # TODO Consider allowing the *_local.yaml files to be located outside of the
+  # /var/panoptes tree so that it is feasible to wipe out /var/panoptes and
+  # reinstall without losing local customizations.
+  cp "${POCS}/conf_files/pocs.yaml" "${POCS}/conf_files/pocs_local.yaml"
+  echo "
+
+You will need to modify the file ${POCS}/conf_files/pocs_local.yaml
+to suit your needs (e.g. set the unit name, location, etc.).
+"
+fi
+
+# shellcheck source=/var/panoptes/set-panoptes-env.sh
+source $PANDIR/set-panoptes-env.sh
+cd $POCS
+python setup.py install
+pytest --test-databases=all --solve
+
+if cmp "${POCS}/conf_files/pocs.yaml" "${POCS}/conf_files/pocs_local.yaml"
+then
+  echo "
+
+Don't forget to modify the file ${POCS}/conf_files/pocs_local.yaml
+to suit your needs (e.g. set the unit name, location, etc.).
+"
+fi
+
+exit

scripts/install/install-apt-packages.sh

@@ -1,4 +1,4 @@
-#!/bin/bash -ex
+#!/bin/bash -e
 
 # Installs APT packages (e.g. debian/ubuntu packages installed for all users).
 # Requires being logged in as root or the ability to successfully execute sudo.
@@ -17,7 +17,7 @@ fi
 
 # Generate the basic apt-get install command, minus the list of packages.
 # We store into a shell array, i.e. an array of strings.
-declare -a apt_get_install=(apt-get install --no-install-recommends --yes)
+declare -a apt_get_install=(apt-get --quiet install --no-install-recommends --yes)
 # shellcheck disable=SC2119
 apt_proxy_url="$(get_apt_proxy_url)"
 if [ -n "${apt_proxy_url}" ] ; then
@@ -26,17 +26,19 @@ fi
 
 # Install all of the packages specified in the files in the args.
 function install_apt_packages() {
+  echo_bar
   echo
   echo_running_sudo "apt-get update"
   echo
-  my_sudo apt-get update
+  my_sudo apt-get --quiet update
 
   for APT_PKGS_FILE in "$@"
   do
+    echo_bar
+    echo
     # Remove all the comments from the package list and install the packages whose
     # names are left.
     APT_PKGS="$(cut '-d#' -f1 "${APT_PKGS_FILE}" | sort | uniq)"
-    echo
     echo_running_sudo "apt-get install for the files in ${APT_PKGS_FILE}"
     echo
     # A note on syntax: ${array_variable} expands to just the first element
@@ -46,6 +48,7 @@ function install_apt_packages() {
     # a single element (e.g. "a b") doesn't result in multiple 'words'.
     # shellcheck disable=SC2086
     my_sudo "${apt_get_install[@]}" ${APT_PKGS}
+    echo
   done
 }