diff --git a/.dockerignore b/.dockerignore index bcb02874b..26790adf8 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,16 +1,18 @@ -!.git docs/* -.eggs .idea .venv venv -*.egg-info + +.git .github *.md !README*.md -*.log -*.pdf +logs/ +**/.eggs +**/*.pdf +**/*.log +**/*.egg-info **/*.pyc **/__pycache__ \ No newline at end of file diff --git a/.github/workflows/pythontest.yaml b/.github/workflows/pythontest.yaml index e257653d5..e38fcf964 100644 --- a/.github/workflows/pythontest.yaml +++ b/.github/workflows/pythontest.yaml @@ -29,12 +29,12 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 - - name: Fetch all history for all tags and branches - run: git fetch --prune --unshallow - - name: Build pocs image + - name: Build panoptes-pocs image run: | - docker build -t pocs:testing -f docker/latest.Dockerfile . - - name: Test with pytest in pocs container + # Make sure git goes to the build context. + sed -i s'/^\.git$/\!\.git/' .dockerignore + docker build -t panoptes-pocs:develop -f docker/develop.Dockerfile . + - name: Test with pytest in panoptes-pocs container run: | mkdir -p coverage_dir && chmod 777 coverage_dir ci_env=`bash <(curl -s https://codecov.io/env)` @@ -42,9 +42,9 @@ jobs: $ci_env \ -e REPORT_FILE="/tmp/coverage/coverage.xml" \ --network "host" \ - -v $PWD:/var/panoptes/logs \ + -v $PWD/coverage_dir:/var/panoptes/logs \ -v $PWD/coverage_dir:/tmp/coverage \ - pocs:testing \ + panoptes-pocs:develop \ scripts/testing/run-tests.sh - name: Upload coverage report to codecov.io uses: codecov/codecov-action@v1 @@ -58,4 +58,4 @@ jobs: if: always() with: name: log-files - path: panoptes-testing.log + path: coverage_dir/panoptes-testing.log diff --git a/.gitignore b/.gitignore index bcbbc904e..2830c8d2b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,14 +1,14 @@ # Temporary and binary files *~ -*.py[cod] *.so *.cfg !.isort.cfg !setup.cfg *.orig -*.log *.pot -__pycache__/* +**/*.py[cod] +**/*.log +**/__pycache__/* .cache/* .*.swp */.ipynb_checkpoints/* diff --git a/.travis.yml b/.travis.yml index 1f010372e..f061e9f88 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,18 +1,19 @@ dist: xenial sudo: required language: python +addons: + apt: + packages: + - docker-ce python: - "3.6" services: - docker before_install: -- docker pull gcr.io/panoptes-exp/pocs:latest -- ci_env=`bash <(curl -s https://codecov.io/env)` +- sed -i s'/^\.git$/\!\.git/' .dockerignore +- docker build -t panoptes-pocs:develop -f ${TRAVIS_BUILD_DIR}/docker/develop.Dockerfile ${TRAVIS_BUILD_DIR} install: true script: - docker run -it - $ci_env - -e LOCAL_USER_ID=0 - -v $TRAVIS_BUILD_DIR:/var/panoptes/POCS - gcr.io/panoptes-exp/pocs:latest + panoptes-pocs:develop scripts/testing/run-tests.sh diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a048c3f94..4dba3c78a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,17 +6,31 @@ All notable changes to this project will be documented in this file. The format is based on `Keep a Changelog `__, and this project adheres to `Semantic Versioning `__. +[0.7.5dev] +---------- + +Changed +~~~~~~~ + +* Docker image updates (#972) + * Updated `install-pocs.sh` script. + * ``latest`` installs the ``panoptes-pocs`` module from pip + * ``develop`` installs via ``pip install -e[google.testing]`` and is used for running the CI tests. + * ``developer-env`` installs locally but with all options, i.e. ``pip install -e[google,testing,plotting,developer]``. Also builds ``jupyterlab`` and other developer tools. Starts a ``jupyterlab`` instance by default. + * Use new ``arduino-cli`` installer. + * Add ``bin/panoptes-develop`` and ``bin/wait-for-it.sh`` to installed scripts. + * Add ``docker/setup-local-environment.sh``, a convenience script for building local images. [0.7.4] - 2020-05-31 ---------- -Note that we skipped `0.7.2` and `0.7.3`. +Note that we skipped ``0.7.2`` and ``0.7.3``. Bug fixes ~~~~~~~~~ -* Package name is `panoptes-pocs` for namespace consistency. (#971) +* Package name is ``panoptes-pocs`` for namespace consistency. (#971) * README changed to rst. (#971) diff --git a/bin/panoptes-develop b/bin/panoptes-develop new file mode 100755 index 000000000..a4cc90264 --- /dev/null +++ b/bin/panoptes-develop @@ -0,0 +1,38 @@ +#!/usr/bin/bash +set -e + +PARAMS="$*" + +export PANDIR=${PANDIR:-/var/panoptes} +export IMAGE="${IMAGE:-panoptes-pocs}" +export TAG="${TAG:-developer-env}" +export CONTAINER_NAME="${CONTAINER_NAME:-pocs-developer-env}" + +cd "${PANDIR}" + +CMD="docker-compose \ + --project-directory ${PANDIR} \ + --env-file ${PANDIR}/env \ + -f POCS/docker/docker-compose-developer-env.yaml \ + -p panoptes" + +## Add the deamon option by default. +if [[ "$PARAMS" == "up" ]]; then + PARAMS="up -d" +fi + +# We use a docker container for docker-compose, so we need to pass the env vars to +# that container so it can properly place them in the docker-compose file. +export DOCKER_RUN_OPTIONS="${DOCKER_RUN_OPTIONS:--e IMAGE=${IMAGE} -e TAG=${TAG} -e CONTAINER_NAME=${CONTAINER_NAME}}" + +# Run the docker-compose command with user params. +eval "DOCKER_RUN_OPTIONS=\"${DOCKER_RUN_OPTIONS}\" ${CMD} ${PARAMS}" + +# If we just started the environment, try to open the browser for the user. +if [[ "$PARAMS" == "up -d" ]]; then + # Prompt for password + "${PANDIR}/POCS/bin/wait-for-it.sh" \ + localhost:8888 \ + -- \ + docker exec -it -u panoptes "${CONTAINER_NAME}" jupyter notebook list | grep http | cut -d ' ' -f 1 | xargs xdg-open +fi diff --git a/bin/wait-for-it.sh b/bin/wait-for-it.sh new file mode 100755 index 000000000..761bc5cee --- /dev/null +++ b/bin/wait-for-it.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash +# https://github.com/vishnubob/wait-for-it +# Use this script to test if a given TCP host/port are available + +WAITFORIT_cmdname=${0##*/} + +echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } + +usage() +{ + cat << USAGE >&2 +Usage: + $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args] + -h HOST | --host=HOST Host or IP under test + -p PORT | --port=PORT TCP port under test + Alternatively, you specify the host and port as host:port + -s | --strict Only execute subcommand if the test succeeds + -q | --quiet Don't output any status messages + -t TIMEOUT | --timeout=TIMEOUT + Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit 1 +} + +wait_for() +{ + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + else + echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout" + fi + WAITFORIT_start_ts=$(date +%s) + while : + do + if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then + nc -z $WAITFORIT_HOST $WAITFORIT_PORT + WAITFORIT_result=$? + else + (echo > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1 + WAITFORIT_result=$? + fi + if [[ $WAITFORIT_result -eq 0 ]]; then + WAITFORIT_end_ts=$(date +%s) + echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds" + break + fi + sleep 1 + done + return $WAITFORIT_result +} + +wait_for_wrapper() +{ + # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 + if [[ $WAITFORIT_QUIET -eq 1 ]]; then + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + else + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + fi + WAITFORIT_PID=$! + trap "kill -INT -$WAITFORIT_PID" INT + wait $WAITFORIT_PID + WAITFORIT_RESULT=$? + if [[ $WAITFORIT_RESULT -ne 0 ]]; then + echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + fi + return $WAITFORIT_RESULT +} + +# process arguments +while [[ $# -gt 0 ]] +do + case "$1" in + *:* ) + WAITFORIT_hostport=(${1//:/ }) + WAITFORIT_HOST=${WAITFORIT_hostport[0]} + WAITFORIT_PORT=${WAITFORIT_hostport[1]} + shift 1 + ;; + --child) + WAITFORIT_CHILD=1 + shift 1 + ;; + -q | --quiet) + WAITFORIT_QUIET=1 + shift 1 + ;; + -s | --strict) + WAITFORIT_STRICT=1 + shift 1 + ;; + -h) + WAITFORIT_HOST="$2" + if [[ $WAITFORIT_HOST == "" ]]; then break; fi + shift 2 + ;; + --host=*) + WAITFORIT_HOST="${1#*=}" + shift 1 + ;; + -p) + WAITFORIT_PORT="$2" + if [[ $WAITFORIT_PORT == "" ]]; then break; fi + shift 2 + ;; + --port=*) + WAITFORIT_PORT="${1#*=}" + shift 1 + ;; + -t) + WAITFORIT_TIMEOUT="$2" + if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi + shift 2 + ;; + --timeout=*) + WAITFORIT_TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + WAITFORIT_CLI=("$@") + break + ;; + --help) + usage + ;; + *) + echoerr "Unknown argument: $1" + usage + ;; + esac +done + +if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then + echoerr "Error: you need to provide a host and port to test." + usage +fi + +WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15} +WAITFORIT_STRICT=${WAITFORIT_STRICT:-0} +WAITFORIT_CHILD=${WAITFORIT_CHILD:-0} +WAITFORIT_QUIET=${WAITFORIT_QUIET:-0} + +# check to see if timeout is from busybox? +WAITFORIT_TIMEOUT_PATH=$(type -p timeout) +WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH) +if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then + WAITFORIT_ISBUSY=1 + WAITFORIT_BUSYTIMEFLAG="-t" + +else + WAITFORIT_ISBUSY=0 + WAITFORIT_BUSYTIMEFLAG="" +fi + +if [[ $WAITFORIT_CHILD -gt 0 ]]; then + wait_for + WAITFORIT_RESULT=$? + exit $WAITFORIT_RESULT +else + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + wait_for_wrapper + WAITFORIT_RESULT=$? + else + wait_for + WAITFORIT_RESULT=$? + fi +fi + +if [[ $WAITFORIT_CLI != "" ]]; then + if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then + echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess" + exit $WAITFORIT_RESULT + fi + exec "${WAITFORIT_CLI[@]}" +else + exit $WAITFORIT_RESULT +fi diff --git a/docker/README.rst b/docker/README.rst new file mode 100644 index 000000000..5ea19edf9 --- /dev/null +++ b/docker/README.rst @@ -0,0 +1,96 @@ +Docker Images +============= + +POCS is available as a docker image hosted on Google Cloud Registry (GCR): + +Image name: ``gcr.io/panoptes-exp/panoptes-pocs`` + +Tags: ``latest``, ``develop``, and ``developer-env``. + +Setup +~~~~~ + +To build the images locally: + +.. code:: bash + + docker/setup-local-environment.sh + +To run the test suite locally: + +.. code:: bash + + scripts/testing/test-software.sh + +This will build all three images locally and is suitable for testing and development. + +Description +~~~~~~~~~~~ + +The ``panoptes-pocs`` image comes in three separate flavors, or tags, +that serve different purposes. + +latest +^^^^^^ + +The ``latest`` image is the "production" version of ``panoptes-pocs``. + +PANOPTES units should be running this flavor. + +When running the install script, this will be the default install option unless the "developer" is selected. + +develop +^^^^^^^ + +The ``develop`` image is used for running the automated tests. These are +run automatically on both GitHub and Travis for all code pushes but can +also be run locally while doing development. + +developer-env +^^^^^^^^^^^^^ + +The ``developer-env`` image is meant to be be used by developers or anyone wishing to +explore the code. The image should be built locally using the ``docker/setup-local-environment.sh`` +script (or, ideally, just use the ``install-pocs`` script). + +The ``bin/panoptes-develop up`` can then be used to start a docker container +instance that will launch ``jupyter-lab`` from ``$PANDIR`` automatically. + +There are a few ways to get the development version. + +1) If you have ``git`` and are comfortable using the command line: + +.. code-block:: bash + + cd $PANDIR + + # Get the repository. + git clone https://github.com/panoptes/panoptes-pocs.git + cd panoptes-pocs + + # Run environment. + bin/panoptes-develop up + +2) If you would like to build your own local docker image: + +.. code-block:: bash + + cd $PANDIR/panoptes-pocs + # First build the 'latest' image locally. + docker build -t panoptes-pocs:latest -f docker/latest.Dockerfile . + + # Then build the develop image locally. + docker build \ + --build-arg base_image=panoptes-pocs:latest \ + -t panoptes-pocs:develop \ + -f docker/develop.Dockerfile . + + # Wait for build to finish... + + # Run with new image. + IMAGE=panoptes-pocs bin/panoptes-develop up + +3) If you are using a new system: + + TODO: Document this section. + diff --git a/docker/build-image.sh b/docker/build-image.sh index ee8510ab2..e559eced3 100755 --- a/docker/build-image.sh +++ b/docker/build-image.sh @@ -6,7 +6,7 @@ TAG="${1:-develop}" cd "${SOURCE_DIR}" -echo "Building gcr.io/panoptes-exp/pocs:${TAG}" +echo "Building gcr.io/panoptes-exp/panoptes-pocs:${TAG}" gcloud builds submit \ --timeout="1h" \ --substitutions="_TAG=${TAG}" \ diff --git a/docker/cloudbuild.yaml b/docker/cloudbuild.yaml index 882d41ebf..0372db9ac 100644 --- a/docker/cloudbuild.yaml +++ b/docker/cloudbuild.yaml @@ -1,19 +1,22 @@ steps: -- name: 'docker' - id: 'amd64-build' - args: - - 'build' - - '--build-arg image_url=gcr.io/panoptes-exp/panoptes-utils:${_TAG}' - - '-f=docker/${_TAG}.Dockerfile' - - '--tag=gcr.io/${PROJECT_ID}/panoptes-pocs:${_TAG}' - - '.' + - name: 'gcr.io/cloud-builders/docker' + id: 'amd64-build' + args: + - 'build' + - '--build-arg' + - 'IMAGE_URL=gcr.io/panoptes-exp/panoptes-utils:${_TAG}' + - '-f' + - 'docker/${_TAG}.Dockerfile' + - '--tag' + - 'gcr.io/${PROJECT_ID}/panoptes-pocs:${_TAG}' + - '.' -- name: 'docker' - id: 'amd64-push' - args: - - 'push' - - 'gcr.io/${PROJECT_ID}/panoptes-pocs:${_TAG}' - waitFor: ['amd64-build'] + - name: 'gcr.io/cloud-builders/docker' + id: 'amd64-push' + args: + - 'push' + - 'gcr.io/${PROJECT_ID}/panoptes-pocs:${_TAG}' + waitFor: ['amd64-build'] images: - 'gcr.io/${PROJECT_ID}/panoptes-pocs:${_TAG}' diff --git a/docker/develop.Dockerfile b/docker/develop.Dockerfile new file mode 100644 index 000000000..57b2e3bc1 --- /dev/null +++ b/docker/develop.Dockerfile @@ -0,0 +1,38 @@ +ARG IMAGE_URL=gcr.io/panoptes-exp/panoptes-pocs:latest +FROM ${IMAGE_URL} + +LABEL description="Installs the local folder in develop mode (i.e. pip install .e). \ +Used for running the tests and as a base for the for developer-env image." +LABEL maintainers="developers@projectpanoptes.org" +LABEL repo="github.com/panoptes/POCS" + +ARG pan_dir=/var/panoptes +ARG pocs_dir="${pan_dir}/POCS" + +ENV DEBIAN_FRONTEND=noninteractive +ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 +ENV SHELL /bin/zsh + +ENV PANUSER=panoptes +ENV PANDIR $pan_dir +ENV POCS $pocs_dir +ENV SOLVE_FIELD /usr/bin/solve-field + +# panoptes-utils +USER ${PANUSER} +COPY --chown=panoptes:panoptes . "${PANDIR}/POCS/" +RUN cd "${PANDIR}/POCS" && \ + pip install -e ".[testing,google]" + +# Cleanup apt. +USER root +RUN apt-get autoremove --purge -y && \ + apt-get -y clean && \ + rm -rf /var/lib/apt/lists/* && \ + chown -R "${PANUSER}:${PANUSER}" "${PANDIR}" && \ + chmod -R 777 /astrometry + +WORKDIR ${POCS} + +# Entrypoint runs gosu with panoptes user. +CMD ["/bin/zsh"] diff --git a/docker/developer-env.Dockerfile b/docker/developer-env.Dockerfile new file mode 100644 index 000000000..0a692d6ed --- /dev/null +++ b/docker/developer-env.Dockerfile @@ -0,0 +1,67 @@ +ARG BASE_IMAGE=panoptes-pocs:develop +FROM ${BASE_IMAGE} + +LABEL description="Installs the local folder in develop mode (i.e. pip install .e). \ +and installs a number of developer tools. Runs jupyter lab instance. This assumes the \ +`panoptes-pocs:develop` has already been built locally." +LABEL maintainers="developers@projectpanoptes.org" +LABEL repo="github.com/panoptes/POCS" + +ARG panuser=panoptes +ARG userid=1000 +ARG pan_dir=/var/panoptes +ARG pocs_dir="${pan_dir}/POCS" + +ENV DEBIAN_FRONTEND=noninteractive +ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 +ENV SHELL /bin/zsh + +ENV USERID $userid +ENV PANDIR $pan_dir +ENV PANLOG "$pan_dir/logs" +ENV PANUSER $panuser +ENV POCS $pocs_dir +ENV PATH "/home/${PANUSER}/.local/bin:$PATH" + +RUN apt-get update && \ + # Node for jupyterlab. + curl -sL https://deb.nodesource.com/setup_12.x | bash - && \ + # Make a developer's life easier. + apt-get install -y --no-install-recommends \ + wget curl bzip2 ca-certificates nano neovim \ + gcc git pkg-config ncdu sudo nodejs && \ + echo "$PANUSER ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers + + +USER $PANUSER +# Can't seem to get around the hard-coding +COPY --chown=panoptes:panoptes . ${PANDIR}/POCS/ +RUN cd ${PANDIR}/POCS && \ + # Install everything! + pip install -e ".[google,developer,plotting,testing]" && \ + # Set some jupyterlab defaults. + mkdir -p /home/panoptes/.jupyter && \ + jupyter-lab --generate-config && \ + # Jupyterlab extensions. + echo "c.JupyterApp.answer_yesBool = True" >> \ + "/home/panoptes/.jupyter/jupyter_notebook_config.py" && \ + echo "c.JupyterApp.open_browserBool = False" >> \ + "/home/panoptes/.jupyter/jupyter_notebook_config.py" && \ + echo "c.JupyterAppy.notebook_dir = '${PANDIR}'" >> \ + "/home/panoptes/.jupyter/jupyter_notebook_config.py" && \ + jupyter labextension install @pyviz/jupyterlab_pyviz \ + jupyterlab-drawio \ + @aquirdturtle/collapsible_headings \ + @telamonian/theme-darcula + +USER root + +# Cleanup apt. +RUN apt-get autoremove --purge -y && \ + apt-get -y clean && \ + rm -rf /var/lib/apt/lists/* + +WORKDIR ${PANDIR} + +# Start a jupyterlab instance. +CMD ["/home/panoptes/.local/bin/jupyter-lab"] diff --git a/docker/docker-compose-developer-env.yaml b/docker/docker-compose-developer-env.yaml new file mode 100644 index 000000000..1416a4528 --- /dev/null +++ b/docker/docker-compose-developer-env.yaml @@ -0,0 +1,19 @@ +version: '3.7' +services: + develop-env: + image: "${IMAGE:-panoptes-pocs}:${TAG:-developer-env}" + init: true + container_name: "${CONTAINER_NAME:-pocs-developer-env}" + privileged: true + network_mode: host + env_file: $PANDIR/env + volumes: + - pandir:/var/panoptes +volumes: + pandir: + driver: local + driver_opts: + type: none + device: /var/panoptes + o: bind + diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 6acf52339..5cb77c273 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -1,7 +1,7 @@ version: '3.7' services: peas-shell: - image: gcr.io/panoptes-exp/pocs:latest + image: gcr.io/panoptes-exp/panoptes-pocs:latest init: true container_name: peas-shell hostname: peas-shell @@ -19,7 +19,7 @@ services: - "-f" - "/dev/null" pocs-shell: - image: gcr.io/panoptes-exp/pocs:latest + image: gcr.io/panoptes-exp/panoptes-pocs:latest init: true container_name: pocs-shell hostname: pocs-shell diff --git a/docker/latest.Dockerfile b/docker/latest.Dockerfile index 202d82b3b..7416b4776 100644 --- a/docker/latest.Dockerfile +++ b/docker/latest.Dockerfile @@ -1,43 +1,35 @@ -ARG image_url=gcr.io/panoptes-exp/panoptes-utils:testing +ARG IMAGE_URL=gcr.io/panoptes-exp/panoptes-utils:latest +FROM ${IMAGE_URL} AS pocs-base -FROM $image_url AS pocs-base -LABEL maintainer="developers@projectpanoptes.org" +LABEL description="Installs the panoptes-pocs module from pip. \ +Used as a production image, i.e. for running on PANOPTES units." +LABEL maintainers="developers@projectpanoptes.org" +LABEL repo="github.com/panoptes/POCS" ARG pandir=/var/panoptes -ARG arduino_url="https://downloads.arduino.cc/arduino-cli/arduino-cli_latest_Linux_64bit.tar.gz" +ARG arduino_url="https://raw.githubusercontent.com/arduino/arduino-cli/master/install.sh" +ARG gphoto2_url="https://raw.githubusercontent.com/gonzalo/gphoto2-updater/master/gphoto2-updater.sh" ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 ENV SHELL /bin/zsh ENV PANDIR $pandir ENV POCS ${PANDIR}/POCS -ENV USER panoptes +ENV SOLVE_FIELD /usr/bin/solve-field RUN apt-get update \ && apt-get install --no-install-recommends --yes \ gcc libncurses5-dev udev \ # GPhoto2 - && wget https://raw.githubusercontent.com/gonzalo/gphoto2-updater/master/gphoto2-updater.sh \ + && wget $gphoto2_url \ && chmod +x gphoto2-updater.sh \ && /bin/bash gphoto2-updater.sh --stable \ && rm gphoto2-updater.sh \ # arduino-cli - && wget -q $arduino_url -O arduino-cli.tar.gz \ - # Untar and capture output name (NOTE: assumes only one file). - && tar xvfz arduino-cli.tar.gz \ - && mv arduino-cli /usr/local/bin/arduino-cli \ - && chmod +x /usr/local/bin/arduino-cli - -COPY ./requirements.txt /tmp/requirements.txt -# First deal with pip and PyYAML - see https://github.com/pypa/pip/issues/5247 -RUN pip install --no-cache-dir --no-deps --ignore-installed pip PyYAML && \ - pip install --no-cache-dir -r /tmp/requirements.txt - -# Install module -COPY . ${POCS}/ -RUN cd ${POCS} && pip install -e ".[google]" + && curl -fsSL $arduino_url | BINDIR="/usr/local/bin" sh \ + # Install the module. + && pip install "panoptes-pocs[google]" # Cleanup apt. -USER root RUN apt-get autoremove --purge -y \ autoconf \ automake \ diff --git a/docker/setup-local-environment.sh b/docker/setup-local-environment.sh new file mode 100755 index 000000000..385adf24b --- /dev/null +++ b/docker/setup-local-environment.sh @@ -0,0 +1,45 @@ +#!/bin/bash -e + +POCS=${POCS:-/var/panoptes/POCS} +TAG="${1:-develop}" + +cd "${POCS}" + +echo "Building local panoptes-pocs:latest" +docker build \ + --quiet \ + -t "panoptes-pocs:latest" \ + -f "${POCS}/docker/latest.Dockerfile" \ + "${POCS}" + +# In the local develop we need to pass git to the docker build context. +sed -i s'/^\.git$/\!\.git/' .dockerignore + +echo "Building local panoptes-pocs:develop" +docker build \ + --quiet \ + --build-arg IMAGE_URL="panoptes-pocs:latest" \ + -t "panoptes-pocs:develop" \ + -f "${POCS}/docker/develop.Dockerfile" \ + "${POCS}" + +echo "Building local panoptes-pocs:developer-env" +docker build \ + --quiet \ + --build-arg IMAGE_URL="panoptes-pocs:develop" \ + -t "panoptes-pocs:developer-env" \ + -f "${POCS}/docker/developer-env.Dockerfile" \ + "${POCS}" + +# Revert our .dockerignore changes. +sed -i s'/^!\.git$/\.git/' .dockerignore + +cat < - Authors + Docker Guider + Contributing Guide Changelog Module Reference - Contributing Guide + Authors + License Indices and tables diff --git a/scripts/install/install-pocs.sh b/scripts/install/install-pocs.sh index 17bee90d8..d08174b27 100755 --- a/scripts/install/install-pocs.sh +++ b/scripts/install/install-pocs.sh @@ -14,44 +14,66 @@ usage() { # or # $ wget -O - https://install.projectpanoptes.org | bash # -# The script will insure that Docker is installed, download the -# latest Docker images (see list below) and clone a copy of the -# relevant PANOPTES repositories. +# The script will do the following: +# +# * Create the needed directory structure. +# * Ensure that docker and docker-compose are installed. +# * Fetch and/or build the docker images needed to run. +# * If in "developer" mode, clone user's fork and set panoptes upstream. +# * Write the environment variables to $PANDIR/env # # Docker Images: # # ${DOCKER_BASE}/panoptes-utils # ${DOCKER_BASE}/pocs # -# The script will ask for a github user name. If you are a developer -# you can enter your github username to work from your fork. Otherwise -# the default user (panoptes) is okay for running the unit. # -# The script has been tested with a fresh install of Ubuntu 19.04 +# The script will ask if it should be installed in "developer" mode or not. +# +# The regular install is for running units and will not create local (to the +# host system) copies of the files. +# +# The "developer" mode will ask for a github username and will clone and +# fetch the repos. The `docker/setup-local-enviornment.sh` script will then +# be run to build the docker images locally. +# +# If not in "developer" mode, the docker images will be pulled from GCR. +# +# The script has been tested with a fresh install of Ubuntu 20.04 # but may work on other linux systems. ############################################################# - $ $(basename $0) [--user panoptes] [--pandir /var/panoptes] + $ $(basename $0) [--developer] [--user panoptes] [--pandir /var/panoptes] Options: - USER The PANUSER environment variable, defaults to current user (i.e. USER=`$USER`). - PANDIR Default install directory, defaults to /var/panoptes. Saved as PANDIR + DEVELOPER Install POCS in developer mode, default False. + + If in DEVELOPER mode, the following options are also available: + USER The PANUSER environment variable, defaults to current user (i.e. PANUSER=$USER). + PANDIR Default install directory, defaults to PANDIR=${PANDIR}. Saved as PANDIR environment variable. " } -DOCKER_BASE="gcr.io/panoptes-exp" -if [ -z "${PANUSER}" ]; then - export PANUSER=$USER -fi -if [ -z "${PANDIR}" ]; then - export PANDIR='/var/panoptes' -fi +DEVELOPER=${DEVELOPER:-false} +PANUSER=${PANUSER:-$USER} +PANDIR=${PANDIR:-/var/panoptes} +LOGFILE="${PANDIR}/install-pocs.log" +OS="$(uname -s)" +ARCH="$(uname -m)" + +DOCKER_COMPOSE_VERSION="${DOCKER_COMPOSE_VERSION:-1.26.0}" +DOCKER_COMPOSE_INSTALL="https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-${OS}-${ARCH}" +DOCKER_BASE=${DOCKER_BASE:-"gcr.io/panoptes-exp"} while [[ $# -gt 0 ]] do key="$1" -case $key in +case ${key} in + --developer) + DEVELOPER=true + shift # past bool argument + ;; -u|--user) PANUSER="$2" shift # past argument @@ -70,6 +92,14 @@ case $key in esac done +if "${DEVELOPER}"; then + while [[ -z "${GITHUB_USER}" ]]; do + read -p "Github User [NOTE: you must have a fork created already]: " GITHUB_USER + done +fi + +echo "DEVELOPER=${DEVELOPER} PANDIR=${PANDIR} PANUSER=${PANUSER} GITHUB_USER=${GITHUB_USER}" + function command_exists { # https://gist.github.com/gubatron/1eb077a1c5fcf510e8e5 # this should be a very portable way of checking if something is on the path @@ -77,89 +107,100 @@ function command_exists { type "$1" &> /dev/null } -do_install() { - clear - - OS="$(uname -s)" - case "${OS}" in - Linux*) machine=Linux;; - Darwin*) machine=Mac;; - *) machine="UNKNOWN:${unameOut}" - esac - echo ${machine} - - # Install directory - read -p "PANOPTES base directory [${PANDIR:-/var/panoptes}]: " PANDIR - PANDIR=${PANDIR:-/var/panoptes} - - LOGFILE="${PANDIR}/logs/install-pocs.log" - - echo "Installing PANOPTES software." - echo "USER: ${PANUSER}" - echo "OS: ${OS}" - echo "Base dir: ${PANDIR}" - echo "Logfile: ${LOGFILE}" - - # Directories +function make_directories { if [[ ! -d "${PANDIR}" ]]; then - echo "Creating directories in ${PANDIR}" - # Make directories + # Make directories and make PANUSER the owner. sudo mkdir -p "${PANDIR}" - sudo chown -R "${PANUSER}":"${PANUSER}" "${PANDIR}" - - mkdir -p "${PANDIR}/logs" - mkdir -p "${PANDIR}/images" - mkdir -p "${PANDIR}/conf_files" - mkdir -p "${PANDIR}/.key" else echo "WARNING ${PANDIR} already exists. You can exit and specify an alternate directory with --pandir or continue." select yn in "Yes" "No"; do - case $yn in + case ${yn} in Yes ) echo "Proceeding with existing directory"; break;; No ) echo "Exiting"; exit 1;; esac done fi - # apt: git, wget - echo "Installing system dependencies" + sudo mkdir -p "${PANDIR}/logs" + sudo mkdir -p "${PANDIR}/images" + sudo mkdir -p "${PANDIR}/config_files" + sudo mkdir -p "${PANDIR}/.key" + sudo chown -R "${PANUSER}":"${PANUSER}" "${PANDIR}" +} + +function setup_env_vars { + ENV_FILE="${PANDIR}/env" + echo "Writing environment variables to ${ENV_FILE}" + if -f "${ENV_FILE}"; then + echo "\n**** Added by install-pocs script ****\n" >> "${ENV_FILE}" + fi + + cat >> "${ENV_FILE}" <> "${LOGFILE}" 2>&1 - sudo apt-get --yes install wget curl git openssh-server ack jq httpie byobu >> "${LOGFILE}" 2>&1 + # TODO(wtgee) figure out why we needed openssh-server on the host. + sudo apt-get --yes install \ + wget curl git openssh-server ack jq httpie byobu \ + >> "${LOGFILE}" 2>&1 elif [[ "${OS}" = "Darwin" ]]; then sudo brew update | sudo tee -a "${LOGFILE}" - sudo brew install wget curl git jq httpie | sudo tee -a "${LOGFILE}" + sudo brew install \ + wget curl git jq httpie \ + | sudo tee -a "${LOGFILE}" fi - echo "Cloning PANOPTES source code." - echo "Github user for PANOPTES repos (POCS, panoptes-utils)." + # Add an SSH key if one doesn't exist. + if [[ ! -f "${HOME}/.ssh/id_rsa" ]]; then + echo "Adding ssh key" + ssh-keygen -t rsa -N "" -f "${HOME}/.ssh/id_rsa"; + fi +} - # Default user - read -p "Github User [if you are a developer, enter your name or press Enter for 'panoptes']: " github_user - github_user=${github_user:-panoptes} - echo "Using repositories from user '${github_user}'." +function get_repos { + PUBLIC_GITHUB_URL="https://github.com/panoptes" - GIT_BRANCH="develop" + if "${DEVELOPER}"; then + echo "Using repositories from user: ${GITHUB_USER}" + declare -a repos=("POCS" "panoptes-utils" "panoptes-tutorials") + GITHUB_URL="git@github.com:${GITHUB_USER}" + else + declare -a repos=("POCS" "panoptes-utils") + GITHUB_URL="${PUBLIC_GITHUB_URL}" + fi - cd "${PANDIR}" - declare -a repos=("POCS" "panoptes-utils") for repo in "${repos[@]}"; do if [[ ! -d "${PANDIR}/${repo}" ]]; then + cd "${PANDIR}" echo "Cloning ${repo}" # Just redirect the errors because otherwise looks like it hangs. - git clone "https://github.com/${github_user}/${repo}.git" >> "${LOGFILE}" 2>&1 + # TODO handle errors if repo doesn't exist (e.g. bad github name). + git clone "https://github.com/${GITHUB_USER}/${repo}.git" >> "${LOGFILE}" 2>&1 + + # Set panoptes as upstream + cd "${repo}" + git remote add upstream "${PUBLIC_GITHUB_URL}/${repo}" else - # TODO Do an update here. - echo "" + # TODO Figure out how to do updates. + echo "${repo} already exists in ${PANDIR}. No auto-update for now, skipping repo." fi done +} +function get_docker { # Get Docker if ! command_exists docker; then echo "Installing Docker" if [[ "${OS}" = "Linux" ]]; then - /bin/bash -c "$(wget -qO- https://get.docker.com)" &>> ${PANDIR}/logs/install-pocs.log + /bin/bash -c "$(wget -qO- https://get.docker.com)" &>> "${LOGFILE}" echo "Adding ${PANUSER} to docker group" sudo usermod -aG docker "${PANUSER}" >> "${LOGFILE}" 2>&1 @@ -168,29 +209,60 @@ do_install() { echo "Adding ${PANUSER} to docker group" sudo dscl -aG docker "${PANUSER}" fi - else - echo "WARNING: Docker images not installed/downloaded." fi if ! command_exists docker-compose; then echo "Installing docker-compose" # Docker compose as container - https://docs.docker.com/compose/install/#install-compose - sudo wget -q https://github.com/docker/compose/releases/download/1.25.4/docker-compose-`uname -s`-`uname -m` -O /usr/local/bin/docker-compose + sudo wget -q "${DOCKER_COMPOSE_INSTALL}" -O /usr/local/bin/docker-compose sudo chmod a+x /usr/local/bin/docker-compose + fi +} + +function get_or_build_images { + if ${DEVELOPER}; then + echo "Building local PANOPTES docker images." + + cd "${POCS}" + ./docker/setup-local-environment.sh + else + echo "Pulling PANOPTES docker images from Google Cloud Registry (GCR)." - docker pull docker/compose + docker pull "${DOCKER_BASE}/panoptes-pocs:latest" + docker pull "${DOCKER_BASE}/panoptes-utils:latest" + docker pull "${DOCKER_BASE}/aag-weather:latest" fi +} - echo "Pulling PANOPTES docker images" - docker pull "${DOCKER_BASE}/panoptes-utils:latest" - docker pull "${DOCKER_BASE}/aag-weather:latest" - docker pull "${DOCKER_BASE}/pocs:latest" +function do_install { + clear - # Add an SSH key if one doesn't exists - if [[ ! -f "${HOME}/.ssh/id_rsa" ]]; then - echo "Looks like you don't have an SSH key set up yet, adding one now." - ssh-keygen -t rsa -N "" -f "${HOME}/.ssh/id_rsa"; + if ${DEVELOPER}; then + echo "" + echo "**** Developer Mode ****" + echo "" fi + echo "Installing PANOPTES software." + echo "PANUSER: ${PANUSER}" + echo "PANDIR: ${PANDIR}" + echo "OS: ${OS}" + echo "Logfile: ${LOGFILE}" + + exit 0; + + echo "Creating directories in ${PANDIR}" + make_directories + + echo "Installing system dependencies" + system_deps + + echo "Installing docker and docker-compose" + get_docker + + echo "Cloning PANOPTES source code" + get_repos + + get_or_build_images echo "Please reboot your machine before using POCS." @@ -201,6 +273,4 @@ do_install() { } -# wrapped up in a function so that we have some protection against only getting -# half the file during "curl | sh" - copied from get.docker.com do_install diff --git a/scripts/testing/test-software.sh b/scripts/testing/test-software.sh index 36f1b6891..99131f077 100755 --- a/scripts/testing/test-software.sh +++ b/scripts/testing/test-software.sh @@ -6,14 +6,14 @@ cat <=0.3.1' -google = - gcloud - google-cloud-storage + responses + [options.entry_points] # Add here console scripts like: