From 5c8343496373df568bf270990e57339f625c175b Mon Sep 17 00:00:00 2001 From: James Synge Date: Tue, 21 May 2019 19:49:11 -0400 Subject: [PATCH] Add scripts/install/auto-install.sh (#824) Auto-install script for POCS, to be executed as follows: ``` export AUTO_INSTALL_RAW_URL="https://raw.githubusercontent.com/panoptes/POCS/develop/scripts/install/auto-install.sh" wget -q -O - "${AUTO_INSTALL_RAW_URL}" | bash ``` If executed by a user other than the panoptes user, will present the user with instructions for creating the panoptes user, logging in as panoptes, then re-executing the command above as panoptes. If executed by the panoptes user, will install git, clone POCS, then run the install-dependencies.sh script in POCS. This commit includes changes to various dependency specifications (e.g. urllib3) to deal with issues that were detected while testing this, and to install-dependencies.sh, et al, for the same reason. --- requirements.txt | 4 + scripts/install/apt-packages-list.txt | 3 + scripts/install/auto-install.sh | 316 ++++++++++++++++++++ scripts/install/install-apt-packages.sh | 11 +- scripts/install/install-dependencies.sh | 137 +++------ scripts/install/install-gcloud.sh | 64 ++++ scripts/install/install-helper-functions.sh | 135 ++++++++- 7 files changed, 565 insertions(+), 105 deletions(-) create mode 100755 scripts/install/auto-install.sh create mode 100755 scripts/install/install-gcloud.sh diff --git a/requirements.txt b/requirements.txt index f1eee5f0c..2cfd76692 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,4 +22,8 @@ scikit_image >= 0.12.3 scipy >= 0.17.1 transitions >= 0.4.0 tweepy + +# requests 2.21 doesn't like urllib3 >=1.25 +urllib3 >= 1.24, < 1.25 + wcsaxes diff --git a/scripts/install/apt-packages-list.txt b/scripts/install/apt-packages-list.txt index 08af9bd2b..763af7073 100644 --- a/scripts/install/apt-packages-list.txt +++ b/scripts/install/apt-packages-list.txt @@ -78,3 +78,6 @@ coreutils # includes this file). git +# Installing some POCS dependencies (esp. astroscrappy) require a C compiler, +# for which we install gcc via the build-essentials package. +build-essential diff --git a/scripts/install/auto-install.sh b/scripts/install/auto-install.sh new file mode 100755 index 000000000..2b621d927 --- /dev/null +++ b/scripts/install/auto-install.sh @@ -0,0 +1,316 @@ +#!/bin/bash +# auto-install.sh installs git if it isn't installed, then clones POCS and +# installs its dependencies. +# +# To fetch this script from github and execute it immediately, +# run these commands: +# +# export POCS_GITHUB_USER="panoptes" +# export POCS_BRANCH="develop" +# BASE_RAW_POCS_URL="https://raw.githubusercontent.com/${POCS_GITHUB_USER}/POCS/${POCS_BRANCH}" +# export AUTO_INSTALL_RAW_URL="${BASE_RAW_POCS_URL}/scripts/install/auto-install.sh" +# wget -q -O - "${AUTO_INSTALL_RAW_URL}" | bash + +################################################################################ +# Functions COPIED from install-helper-functions.sh +# Print the disk location of the first arg. +function safe_which() { + type -p "${1}" || /bin/true +} + +# Print a separator bar of # characters. +function echo_bar() { + local terminal_width="${COLUMNS}" + if [ -z "${terminal_width}" ] && [ -n "${TERM}" ] + then + if [[ -n "$(safe_which tput)" ]] + then + terminal_width="$(tput cols)" + elif [[ -n "$(safe_which resize)" ]] + then + terminal_width="$(resize 2>/dev/null | grep COLUMNS= | cut -d= -f2)" + elif [[ -n "$(safe_which stty)" ]] + then + terminal_width="$(stty size 2>/dev/null | cut '-d ' -f2)" + fi + fi + printf "%${terminal_width:-80}s\n" | tr ' ' '#' +} + +################################################################################ + +function do_sudo() { + if [ "$(id -u -n)" == "root" ] ; then + echo "Running ${*}" + "$@" + else + echo < " +The directory (${REPO_DIR}) already exists, but doesn't appear +to be a valid git repository. Please remove it (or move it out +of the way) and re-run this script. + " + exit 1 + fi + echo " +Cloning ${REPO_URL} into ${REPO_DIR} +" + (set -x ; git clone --origin "${ORIGIN_NAME}" "${REPO_URL}" "${REPO_DIR}") + cd "${REPO_DIR}" + git checkout "${BRANCH}" + else + echo " +Pulling the latest software into the worktree at ${REPO_DIR}. +" + cd "${REPO_DIR}" + git fetch --all + git checkout "${BRANCH}" + git pull + fi +} + +function maybe_print_example() { + if [[ -n "${AUTO_INSTALL_RAW_URL}" ]] + then + >&2 cat <&2 cat </dev/null 1>/dev/null +then + >&2 cat <&2 " +This script should be executed by the user ${PANUSER}, not as $(whoami). +Please login as ${PANUSER} and re-execute the command you used to +run this script. +" + + maybe_print_example + exit 1 +fi + +# I (James Synge) have noticed that if $HOME/.cache/ doesn't exist, then at some +# point during the install it gets created, but is owned by root:root, which +# then messes up later steps that attempt to use it. So, we make sure here that +# it exists with the correct ownership. + +PANGROUP="$(id -gn "${PANUSER}")" +ensure_directory_ownership "${HOME}/.cache" "${PANUSER}" "${PANGROUP}" + +# Do the same with PANDIR (usually /var/panoptes). +ensure_directory_ownership "${PANDIR}" "${PANUSER}" "${PANGROUP}" + +################################################################################ +# Is there already a POCS or PAWS repository? If so, the user should not need +# this script. +if [[ -d "${POCS}/.git" ]] +then + already_have_a_repo POCS "${POCS}" + exit 1 +fi + +if [[ -d "${PAWS}/.git" ]] +then + already_have_a_repo PAWS "${PAWS}" + exit 1 +fi + +# Install git if necessary. + +if [ ! -x "$(safe_which git)" ] +then + echo_bar + echo " +git is not installed. Updating package cache, then installing git. +" + do_sudo apt-get update + echo + echo + do_sudo apt-get install -y git + echo +fi + +# Clone the POCS repo from github. + +clone_or_update "${POCS}" https://github.com/panoptes/POCS.git upstream develop + +# If the user specified another repo via POCS_GITHUB_USER, use that as +# the origin, and checkout the branch POCS_BRANCH. + +if [[ -n "${POCS_GITHUB_USER}" && "${POCS_GITHUB_USER}" != "panoptes" ]] +then + cd "${POCS}" + git remote add -f origin "https://github.com/${POCS_GITHUB_USER}/POCS.git" || true + git checkout "origin/${POCS_BRANCH:-develop}" +fi + +# Clone PAWS too, but only from the upstream. +clone_or_update \ + "${PAWS}" "https://github.com/panoptes/PAWS.git" upstream develop + +################################################################################ +echo +echo_bar +echo_bar +echo " +Executing ${POCS}/scripts/install/install-dependencies.sh, which will +install the tools needed to run POCS. +" + +${POCS}/scripts/install/install-dependencies.sh + +echo +echo +echo_bar +echo_bar +echo " + +Testing the installation of POCS. This can easily take 5 minutes on an Intel NUC +or longer on a Raspberry Pi. + +" + +if [[ ! -e "${POCS}/conf_files/pocs_local.yaml" ]] +then + # TODO Consider allowing the *_local.yaml files to be located outside of the + # /var/panoptes tree so that it is feasible to wipe out /var/panoptes and + # reinstall without losing local customizations. + cp "${POCS}/conf_files/pocs.yaml" "${POCS}/conf_files/pocs_local.yaml" + echo " + +You will need to modify the file ${POCS}/conf_files/pocs_local.yaml +to suit your needs (e.g. set the unit name, location, etc.). +" +fi + +# shellcheck source=/var/panoptes/set-panoptes-env.sh +source $PANDIR/set-panoptes-env.sh +cd $POCS +python setup.py install +pytest --test-databases=all --solve + +if cmp "${POCS}/conf_files/pocs.yaml" "${POCS}/conf_files/pocs_local.yaml" +then + echo " + +Don't forget to modify the file ${POCS}/conf_files/pocs_local.yaml +to suit your needs (e.g. set the unit name, location, etc.). +" +fi + +exit diff --git a/scripts/install/install-apt-packages.sh b/scripts/install/install-apt-packages.sh index 623d3c104..afaf1a17b 100755 --- a/scripts/install/install-apt-packages.sh +++ b/scripts/install/install-apt-packages.sh @@ -1,4 +1,4 @@ -#!/bin/bash -ex +#!/bin/bash -e # Installs APT packages (e.g. debian/ubuntu packages installed for all users). # Requires being logged in as root or the ability to successfully execute sudo. @@ -17,7 +17,7 @@ fi # Generate the basic apt-get install command, minus the list of packages. # We store into a shell array, i.e. an array of strings. -declare -a apt_get_install=(apt-get install --no-install-recommends --yes) +declare -a apt_get_install=(apt-get --quiet install --no-install-recommends --yes) # shellcheck disable=SC2119 apt_proxy_url="$(get_apt_proxy_url)" if [ -n "${apt_proxy_url}" ] ; then @@ -26,17 +26,19 @@ fi # Install all of the packages specified in the files in the args. function install_apt_packages() { + echo_bar echo echo_running_sudo "apt-get update" echo - my_sudo apt-get update + my_sudo apt-get --quiet update for APT_PKGS_FILE in "$@" do + echo_bar + echo # Remove all the comments from the package list and install the packages whose # names are left. APT_PKGS="$(cut '-d#' -f1 "${APT_PKGS_FILE}" | sort | uniq)" - echo echo_running_sudo "apt-get install for the files in ${APT_PKGS_FILE}" echo # A note on syntax: ${array_variable} expands to just the first element @@ -46,6 +48,7 @@ function install_apt_packages() { # a single element (e.g. "a b") doesn't result in multiple 'words'. # shellcheck disable=SC2086 my_sudo "${apt_get_install[@]}" ${APT_PKGS} + echo done } diff --git a/scripts/install/install-dependencies.sh b/scripts/install/install-dependencies.sh index 7fc9c98a0..57041017d 100755 --- a/scripts/install/install-dependencies.sh +++ b/scripts/install/install-dependencies.sh @@ -105,6 +105,7 @@ export PANDIR POCS PAWS PANLOG PANUSER export DO_APT_GET=1 export DO_MONGODB=1 export DO_CONDA=1 +export DO_GCLOUD=1 export DO_REBUILD_CONDA_ENV=0 export DO_INSTALL_CONDA_PACKAGES=1 export DO_ASTROMETRY=1 @@ -132,6 +133,7 @@ options: --no-astrometry don't install astrometry.net software --no-astrometry-indices don't install astrometry.net indices --no-pip-requirements don't install python packages +--no-gcloud don't install gcloud " } @@ -188,6 +190,10 @@ while test ${#} -gt 0; do DO_ASTROMETRY_INDICES=0 shift ;; + --no-gcloud) + DO_GCLOUD=0 + shift + ;; *) echo "Unknown parameter: ${1}" echo @@ -200,47 +206,6 @@ done #------------------------------------------------------------------------------- # Misc helper functions. -# Backup the file whose path is the first arg, and print the path of the -# backup file. If the file doesn't exist, no path is output. -function backup_file() { - local -r the_original="${1}" - if [[ ! -e "${the_original}" ]] ; then - return - fi - if [[ ! -f "${the_original}" ]] ; then - echo 1>2 " -${the_original} is not a regular file, can't copy! -" - exit 1 - fi - local -r the_backup="$(mktemp "${the_original}.backup.XXXXXXX")" - cp -p "${the_original}" "${the_backup}" - echo "${the_backup}" -} - -function diff_backup_and_file_then_cleanup() { - local -r the_backup="${1}" - local -r the_file="${2}" - if [[ -z "${the_backup}" ]] ; then - echo_bar - echo - echo "Created ${the_file}:" - echo - cat "${the_file}" - echo - return - fi - if ! cmp "${the_backup}" "${the_file}" ; then - echo_bar - echo - echo "Modified ${the_file}:" - echo - diff -u "${the_backup}" "${the_file}" || /bin/true - echo - fi - rm -f "${the_backup}" -} - #------------------------------------------------------------------------------- # Functions for creating the file in which we record the PANOPTES environment # variables and shell setup commands, and for inserting a 'source ' @@ -264,19 +229,6 @@ function profile_contains_text() { fi } -# Add the text of the first arg to the PANOPTES environment setup. -function add_to_panoptes_env_setup() { - PANOPTES_ENV_SETUP+=("${1}") -} - -# Append $1 to PATH and write command to do the same to the -# PANOPTES environment setup. -function add_to_PATH() { - local -r the_dir="$(readlink -f "${1}")" - PATH="${the_dir}:${PATH}" - PANOPTES_ENV_SETUP+=("PATH=\"${the_dir}:\${PATH}\"") -} - # Create (or overwrite) the PANOPTES environment setup file, # and print a diff showing the changes, if there are any. function update_panoptes_env_file() { @@ -294,56 +246,47 @@ export PANUSER="${USER}" . "\${PANDIR}/miniconda/etc/profile.d/conda.sh" conda activate panoptes-env -# Add astrometry to the path. -PATH="${ASTROMETRY_DIR}/bin:\${PATH}" +if [[ -f "\${POCS}/scripts/install/install/install-helper-functions.sh" ]] +then + source "\${POCS}/scripts/install/install/install-helper-functions.sh" + # Add astrometry to the path. + prepend_to_PATH "${ASTROMETRY_DIR}/bin" + clean_PATH +else + # Add astrometry to the path. + PATH="${ASTROMETRY_DIR}/bin:\${PATH}" +fi END_OF_FILE - # We allow for other (optional) commands to be added by adding to - # the array PANOPTES_ENV_SETUP. - printf '%s\n' "${PANOPTES_ENV_SETUP[@]}" >>"${PANOPTES_ENV_SH}" + diff_backup_and_file_then_cleanup "${the_backup}" "${PANOPTES_ENV_SH}" } # Arrange for the PANOPTES environment setup file to be used # when the rc file of the user's shell is executed. function update_shell_rc_file() { - if [[ ! -f "${SHELL_RC}" ]] ; then - cat >"${SHELL_RC}" < "${SHELL_RC}" -# Added by PANOPTES install-dependencies.sh -source ${PANOPTES_ENV_SH} - -END_OF_FILE + ensure_shell_rc_exists + local -r the_backup="$(backup_file "${SHELL_RC}")" + local -r addition1="# Invoke file created by PANOPTES install-dependencies.sh" + local -r addition2=\ +"if [[ -e \"${PANOPTES_ENV_SH}\" ]] ; then source ${PANOPTES_ENV_SH}; fi" + # Prepend new lines, so that for non-interactive shells, we have the env + # set correctly; necessary because the default .bashrc exits almost + # immediately for non-interactive shells. + echo "${addition1}" > "${SHELL_RC}" + echo "${addition2}" >> "${SHELL_RC}" + # Copy existing SHELL_RC contents, minus our two additional lines. + ( (fgrep -v -- "PANOPTES install-dependencies.sh" "${the_backup}" || /bin/true) | \ + (fgrep -v -- "${PANOPTES_ENV_SH}" || /bin/true) ) >> "${SHELL_RC}" + # Append the lines again, so that for interactive shells the prompt is + # also configured correctly. + echo "${addition1}" >> "${SHELL_RC}" + echo "${addition2}" >> "${SHELL_RC}" diff_backup_and_file_then_cleanup "${the_backup}" "${SHELL_RC}" } #------------------------------------------------------------------------------- -# Given the path to a pkg-config file (.pc), extract the version number. -function extract_version_from_pkg_config() { - if [[ -f "${1}" ]] ; then - grep -E '^Version:' "${1}" | cut '-d:' -f2 - else - echo "" - fi -} - function install_mongodb() { # This is based on https://www.howtoforge.com/tutorial/install-mongodb-on-ubuntu/ # Note this function does not configure mongodb itself, i.e. no users or @@ -571,10 +514,16 @@ function maybe_install_conda() { conda update -n base --yes --quiet conda } +#------------------------------------------------------------------------------- +# Astrometry support. + function get_installed_astrometry_version() { local -r solve_field="${ASTROMETRY_DIR}/bin/solve-field" if [[ -x "${solve_field}" ]] ; then - "${solve_field}" --help|(grep -E '^Revision [0-9.]+,' || /bin/true)|cut -c10-|cut -d, -f1 + ("${solve_field}" --help \ + | (grep -E --only-matching '^Revision [0-9.]+' || /bin/true) \ + | cut -c10- \ + | cut -d, -f1) fi } @@ -790,6 +739,10 @@ if [[ "${DO_ASTROMETRY_INDICES}" -eq 1 ]] ; then (install_astrometry_indices) fi +if [[ "${DO_GCLOUD}" -eq 1 ]] ; then + "${THIS_DIR}/install-gcloud.sh" +fi + update_panoptes_env_file update_shell_rc_file diff --git a/scripts/install/install-gcloud.sh b/scripts/install/install-gcloud.sh new file mode 100755 index 000000000..1c7b059ba --- /dev/null +++ b/scripts/install/install-gcloud.sh @@ -0,0 +1,64 @@ +#!/bin/bash -e +# Install the Google Cloud SDK. This requires that we "teach" apt-get where +# to find the repository where the SDK is located. + +THIS_DIR="$(dirname "$(readlink -f "${0}")")" +THIS_PROGRAM="$(basename "${0}")" + +# shellcheck source=/var/panoptes/POCS/scripts/install/install-helper-functions.sh +source "${THIS_DIR}/install-helper-functions.sh" + +echo_bar + +if [ -n "$(safe_which gcloud)" ] +then + echo "Google Cloud SDK (gcloud command) is already installed" + exit +fi + +if [ -z "$(safe_which lsb_release)" ] +then + echo 2> " +Unable to determine the release. Is this OS really Debian or Ubuntu? +" + exit 1 +fi + +CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" +CLOUD_SDK_LIST="/etc/apt/sources.list.d/google-cloud-sdk.list" +echo " +Setting Google Cloud SDK repository location to: + + ${CLOUD_SDK_REPO} + +You may be prompted for your password. +" +echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | \ + my_sudo tee -a "${CLOUD_SDK_LIST}" | cat >/dev/null + +# If we re-run this script, the above leads us with multiple entries for the +# same version. Resolve this by removing duplicate lines. + +echo "Cleaning duplicate sources entries, leaving:" +uniq <(cat "${CLOUD_SDK_LIST}") | my_sudo tee -a "${CLOUD_SDK_LIST}" + +echo " +Importing the public key of that ${CLOUD_SDK_REPO}. +" +wget --quiet -O - https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - + +echo " +Updating the package list (i.e. fetching from the new repository). +" +my_sudo apt-get --quiet update + +echo " +Installing the Google Cloud SDK. +" +my_sudo apt-get --quiet install --no-install-recommends --yes google-cloud-sdk + + +echo_bar +echo " +You'll now need to run [gcloud init] to set your credentials. +" diff --git a/scripts/install/install-helper-functions.sh b/scripts/install/install-helper-functions.sh index 01671b801..4483a1393 100644 --- a/scripts/install/install-helper-functions.sh +++ b/scripts/install/install-helper-functions.sh @@ -7,11 +7,17 @@ # Print a separator bar of # characters. function echo_bar() { - local terminal_width - if [ -n "${TERM}" ] && [ -t 0 ] ; then - if [[ -n "$(which resize)" ]] ; then + local terminal_width="${COLUMNS}" + if [ -z "${terminal_width}" ] && [ -n "${TERM}" ] + then + if [[ -n "$(safe_which tput)" ]] + then + terminal_width="$(tput cols)" + elif [[ -n "$(which resize)" ]] + then terminal_width="$(resize 2>/dev/null | grep COLUMNS= | cut -d= -f2)" - elif [[ -n "$(which stty)" ]] ; then + elif [[ -n "$(which stty)" ]] + then terminal_width="$(stty size 2>/dev/null | cut '-d ' -f2)" fi fi @@ -20,9 +26,13 @@ function echo_bar() { function echo_running_sudo() { if [ "$(id -u -n)" == "root" ] ; then - echo "Running $1" + echo "Running: $1" else - echo "Running sudo $1; you may be prompted for your password." + echo <&2 "Unable to confirm that PATH has a sensible value! + +PATH: ${PATH} + +path_parts: ${path_parts[@]} + +new_path: ${new_path} + +in_new_path_parts: ${in_new_path_parts[@]} + +" + exit 1 + fi + export PATH="${new_path}" +} + +# Backup the file whose path is the first arg, and print the path of the +# backup file. If the file doesn't exist, no path is output. +function backup_file() { + local -r the_original="${1}" + if [[ ! -e "${the_original}" ]] ; then + return + fi + if [[ ! -f "${the_original}" ]] ; then + echo 1>2 " +${the_original} is not a regular file, can't copy! +" + exit 1 + fi + local -r the_backup="$(mktemp "${the_original}.backup.XXXXXXX")" + cp -p "${the_original}" "${the_backup}" + echo "${the_backup}" +} + +function diff_backup_and_file_then_cleanup() { + local -r the_backup="${1}" + local -r the_file="${2}" + if [[ -z "${the_backup}" ]] ; then + echo_bar + echo + echo "Created ${the_file}:" + echo + cat "${the_file}" + echo + return + fi + if ! cmp "${the_backup}" "${the_file}" ; then + echo_bar + echo + echo "Modified ${the_file}:" + echo + diff -u "${the_backup}" "${the_file}" || /bin/true + echo + fi + rm -f "${the_backup}" +} + +#------------------------------------------------------------------------------- +# Helpers for finding the IP gateway of this device. + # Match a line that looks like this: # 8.8.8.8 via 192.168.86.1 dev wlp3s0 src 192.168.86.36 uid 1000 # And extract either the "via" or the "src" address. @@ -71,9 +191,6 @@ function sbin_ip_route_addr() { "${cmd}" route get 8.8.8.8 | awk -F" ${pattern} " 'NR==1{split($2,a," ");print a[1]}' } -#------------------------------------------------------------------------------- -# Helpers for finding the IP gateway of this device. - # Get the address of the gateway stored in /proc/net/route. Available inside # a docker container or in Ubuntu 18.04. When executed inside a container, # the address is that of the host; when executed on the host, the address is