From ff20135322916e33eb504d1862151a6fbac59cfc Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Wed, 3 Jun 2020 09:46:19 -1000 Subject: [PATCH 01/22] Development environment (#972) * * Image name changed from `pocs` -> `panoptes-pocs` * Better install options for extra features. * * Developer option Docker moving from panoptes-utils to pocs. * * Remove TODO about docker install. That should be handled by the script. * * Update test files. * * Cleaning up docker images: * `latest` installs the `panoptes-pocs` module from pip * `develop` installs via `pip install -e[google.testing]` and is used for running the CI tests. `developer-env` installs locally but with all options, i.e. `pip install -e[google,testing,plotting,developer]`. Also builds jupyterlab and other developer tools. Starts a jupyterlab instance by default. * * Changelog updates. * * `develop` image installs the module as the root user. * * Docker build ignore git directory unless buliding locally (needed by pyscaffold and we are buliding a local develop image anyway). * Use new `arduino-cli` installer. * Add `bin/panoptes-develop` and `bin/wait-for-it.sh` to installed scripts. * Adding a convenience script for building local images. * Docker image updates * * Pass git folder to build context for testing images. * * $PANUSER owns $PANDIR. * * Ignore log files.. * * Fix log files on GHA. I don't think this should be needed. * * PANUSER owns PANDIR in develop. * Adding the SOLVE_FIELD env var. * * Give astrometry data all readable. * * Docker docs * * Updated install script. * * Update changelog --- .dockerignore | 12 +- .github/workflows/pythontest.yaml | 16 +- .gitignore | 6 +- .travis.yml | 13 +- CHANGELOG.rst | 18 +- bin/panoptes-develop | 38 ++++ bin/wait-for-it.sh | 179 +++++++++++++++++ docker/README.rst | 96 ++++++++++ docker/build-image.sh | 2 +- docker/cloudbuild.yaml | 31 +-- docker/develop.Dockerfile | 38 ++++ docker/developer-env.Dockerfile | 67 +++++++ docker/docker-compose-developer-env.yaml | 19 ++ docker/docker-compose.yaml | 4 +- docker/latest.Dockerfile | 34 ++-- docker/setup-local-environment.sh | 45 +++++ docs/docker.rst | 2 + docs/index.rst | 7 +- scripts/install/install-pocs.sh | 234 +++++++++++++++-------- scripts/testing/test-software.sh | 8 +- setup.cfg | 31 ++- 21 files changed, 741 insertions(+), 159 deletions(-) create mode 100755 bin/panoptes-develop create mode 100755 bin/wait-for-it.sh create mode 100644 docker/README.rst create mode 100644 docker/develop.Dockerfile create mode 100644 docker/developer-env.Dockerfile create mode 100644 docker/docker-compose-developer-env.yaml create mode 100755 docker/setup-local-environment.sh create mode 100644 docs/docker.rst diff --git a/.dockerignore b/.dockerignore index bcb02874b..26790adf8 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,16 +1,18 @@ -!.git docs/* -.eggs .idea .venv venv -*.egg-info + +.git .github *.md !README*.md -*.log -*.pdf +logs/ +**/.eggs +**/*.pdf +**/*.log +**/*.egg-info **/*.pyc **/__pycache__ \ No newline at end of file diff --git a/.github/workflows/pythontest.yaml b/.github/workflows/pythontest.yaml index e257653d5..e38fcf964 100644 --- a/.github/workflows/pythontest.yaml +++ b/.github/workflows/pythontest.yaml @@ -29,12 +29,12 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 - - name: Fetch all history for all tags and branches - run: git fetch --prune --unshallow - - name: Build pocs image + - name: Build panoptes-pocs image run: | - docker build -t pocs:testing -f docker/latest.Dockerfile . - - name: Test with pytest in pocs container + # Make sure git goes to the build context. + sed -i s'/^\.git$/\!\.git/' .dockerignore + docker build -t panoptes-pocs:develop -f docker/develop.Dockerfile . + - name: Test with pytest in panoptes-pocs container run: | mkdir -p coverage_dir && chmod 777 coverage_dir ci_env=`bash <(curl -s https://codecov.io/env)` @@ -42,9 +42,9 @@ jobs: $ci_env \ -e REPORT_FILE="/tmp/coverage/coverage.xml" \ --network "host" \ - -v $PWD:/var/panoptes/logs \ + -v $PWD/coverage_dir:/var/panoptes/logs \ -v $PWD/coverage_dir:/tmp/coverage \ - pocs:testing \ + panoptes-pocs:develop \ scripts/testing/run-tests.sh - name: Upload coverage report to codecov.io uses: codecov/codecov-action@v1 @@ -58,4 +58,4 @@ jobs: if: always() with: name: log-files - path: panoptes-testing.log + path: coverage_dir/panoptes-testing.log diff --git a/.gitignore b/.gitignore index bcbbc904e..2830c8d2b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,14 +1,14 @@ # Temporary and binary files *~ -*.py[cod] *.so *.cfg !.isort.cfg !setup.cfg *.orig -*.log *.pot -__pycache__/* +**/*.py[cod] +**/*.log +**/__pycache__/* .cache/* .*.swp */.ipynb_checkpoints/* diff --git a/.travis.yml b/.travis.yml index 1f010372e..f061e9f88 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,18 +1,19 @@ dist: xenial sudo: required language: python +addons: + apt: + packages: + - docker-ce python: - "3.6" services: - docker before_install: -- docker pull gcr.io/panoptes-exp/pocs:latest -- ci_env=`bash <(curl -s https://codecov.io/env)` +- sed -i s'/^\.git$/\!\.git/' .dockerignore +- docker build -t panoptes-pocs:develop -f ${TRAVIS_BUILD_DIR}/docker/develop.Dockerfile ${TRAVIS_BUILD_DIR} install: true script: - docker run -it - $ci_env - -e LOCAL_USER_ID=0 - -v $TRAVIS_BUILD_DIR:/var/panoptes/POCS - gcr.io/panoptes-exp/pocs:latest + panoptes-pocs:develop scripts/testing/run-tests.sh diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a048c3f94..4dba3c78a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,17 +6,31 @@ All notable changes to this project will be documented in this file. The format is based on `Keep a Changelog `__, and this project adheres to `Semantic Versioning `__. +[0.7.5dev] +---------- + +Changed +~~~~~~~ + +* Docker image updates (#972) + * Updated `install-pocs.sh` script. + * ``latest`` installs the ``panoptes-pocs`` module from pip + * ``develop`` installs via ``pip install -e[google.testing]`` and is used for running the CI tests. + * ``developer-env`` installs locally but with all options, i.e. ``pip install -e[google,testing,plotting,developer]``. Also builds ``jupyterlab`` and other developer tools. Starts a ``jupyterlab`` instance by default. + * Use new ``arduino-cli`` installer. + * Add ``bin/panoptes-develop`` and ``bin/wait-for-it.sh`` to installed scripts. + * Add ``docker/setup-local-environment.sh``, a convenience script for building local images. [0.7.4] - 2020-05-31 ---------- -Note that we skipped `0.7.2` and `0.7.3`. +Note that we skipped ``0.7.2`` and ``0.7.3``. Bug fixes ~~~~~~~~~ -* Package name is `panoptes-pocs` for namespace consistency. (#971) +* Package name is ``panoptes-pocs`` for namespace consistency. (#971) * README changed to rst. (#971) diff --git a/bin/panoptes-develop b/bin/panoptes-develop new file mode 100755 index 000000000..a4cc90264 --- /dev/null +++ b/bin/panoptes-develop @@ -0,0 +1,38 @@ +#!/usr/bin/bash +set -e + +PARAMS="$*" + +export PANDIR=${PANDIR:-/var/panoptes} +export IMAGE="${IMAGE:-panoptes-pocs}" +export TAG="${TAG:-developer-env}" +export CONTAINER_NAME="${CONTAINER_NAME:-pocs-developer-env}" + +cd "${PANDIR}" + +CMD="docker-compose \ + --project-directory ${PANDIR} \ + --env-file ${PANDIR}/env \ + -f POCS/docker/docker-compose-developer-env.yaml \ + -p panoptes" + +## Add the deamon option by default. +if [[ "$PARAMS" == "up" ]]; then + PARAMS="up -d" +fi + +# We use a docker container for docker-compose, so we need to pass the env vars to +# that container so it can properly place them in the docker-compose file. +export DOCKER_RUN_OPTIONS="${DOCKER_RUN_OPTIONS:--e IMAGE=${IMAGE} -e TAG=${TAG} -e CONTAINER_NAME=${CONTAINER_NAME}}" + +# Run the docker-compose command with user params. +eval "DOCKER_RUN_OPTIONS=\"${DOCKER_RUN_OPTIONS}\" ${CMD} ${PARAMS}" + +# If we just started the environment, try to open the browser for the user. +if [[ "$PARAMS" == "up -d" ]]; then + # Prompt for password + "${PANDIR}/POCS/bin/wait-for-it.sh" \ + localhost:8888 \ + -- \ + docker exec -it -u panoptes "${CONTAINER_NAME}" jupyter notebook list | grep http | cut -d ' ' -f 1 | xargs xdg-open +fi diff --git a/bin/wait-for-it.sh b/bin/wait-for-it.sh new file mode 100755 index 000000000..761bc5cee --- /dev/null +++ b/bin/wait-for-it.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash +# https://github.com/vishnubob/wait-for-it +# Use this script to test if a given TCP host/port are available + +WAITFORIT_cmdname=${0##*/} + +echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } + +usage() +{ + cat << USAGE >&2 +Usage: + $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args] + -h HOST | --host=HOST Host or IP under test + -p PORT | --port=PORT TCP port under test + Alternatively, you specify the host and port as host:port + -s | --strict Only execute subcommand if the test succeeds + -q | --quiet Don't output any status messages + -t TIMEOUT | --timeout=TIMEOUT + Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit 1 +} + +wait_for() +{ + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + else + echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout" + fi + WAITFORIT_start_ts=$(date +%s) + while : + do + if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then + nc -z $WAITFORIT_HOST $WAITFORIT_PORT + WAITFORIT_result=$? + else + (echo > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1 + WAITFORIT_result=$? + fi + if [[ $WAITFORIT_result -eq 0 ]]; then + WAITFORIT_end_ts=$(date +%s) + echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds" + break + fi + sleep 1 + done + return $WAITFORIT_result +} + +wait_for_wrapper() +{ + # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 + if [[ $WAITFORIT_QUIET -eq 1 ]]; then + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + else + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + fi + WAITFORIT_PID=$! + trap "kill -INT -$WAITFORIT_PID" INT + wait $WAITFORIT_PID + WAITFORIT_RESULT=$? + if [[ $WAITFORIT_RESULT -ne 0 ]]; then + echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + fi + return $WAITFORIT_RESULT +} + +# process arguments +while [[ $# -gt 0 ]] +do + case "$1" in + *:* ) + WAITFORIT_hostport=(${1//:/ }) + WAITFORIT_HOST=${WAITFORIT_hostport[0]} + WAITFORIT_PORT=${WAITFORIT_hostport[1]} + shift 1 + ;; + --child) + WAITFORIT_CHILD=1 + shift 1 + ;; + -q | --quiet) + WAITFORIT_QUIET=1 + shift 1 + ;; + -s | --strict) + WAITFORIT_STRICT=1 + shift 1 + ;; + -h) + WAITFORIT_HOST="$2" + if [[ $WAITFORIT_HOST == "" ]]; then break; fi + shift 2 + ;; + --host=*) + WAITFORIT_HOST="${1#*=}" + shift 1 + ;; + -p) + WAITFORIT_PORT="$2" + if [[ $WAITFORIT_PORT == "" ]]; then break; fi + shift 2 + ;; + --port=*) + WAITFORIT_PORT="${1#*=}" + shift 1 + ;; + -t) + WAITFORIT_TIMEOUT="$2" + if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi + shift 2 + ;; + --timeout=*) + WAITFORIT_TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + WAITFORIT_CLI=("$@") + break + ;; + --help) + usage + ;; + *) + echoerr "Unknown argument: $1" + usage + ;; + esac +done + +if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then + echoerr "Error: you need to provide a host and port to test." + usage +fi + +WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15} +WAITFORIT_STRICT=${WAITFORIT_STRICT:-0} +WAITFORIT_CHILD=${WAITFORIT_CHILD:-0} +WAITFORIT_QUIET=${WAITFORIT_QUIET:-0} + +# check to see if timeout is from busybox? +WAITFORIT_TIMEOUT_PATH=$(type -p timeout) +WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH) +if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then + WAITFORIT_ISBUSY=1 + WAITFORIT_BUSYTIMEFLAG="-t" + +else + WAITFORIT_ISBUSY=0 + WAITFORIT_BUSYTIMEFLAG="" +fi + +if [[ $WAITFORIT_CHILD -gt 0 ]]; then + wait_for + WAITFORIT_RESULT=$? + exit $WAITFORIT_RESULT +else + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + wait_for_wrapper + WAITFORIT_RESULT=$? + else + wait_for + WAITFORIT_RESULT=$? + fi +fi + +if [[ $WAITFORIT_CLI != "" ]]; then + if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then + echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess" + exit $WAITFORIT_RESULT + fi + exec "${WAITFORIT_CLI[@]}" +else + exit $WAITFORIT_RESULT +fi diff --git a/docker/README.rst b/docker/README.rst new file mode 100644 index 000000000..5ea19edf9 --- /dev/null +++ b/docker/README.rst @@ -0,0 +1,96 @@ +Docker Images +============= + +POCS is available as a docker image hosted on Google Cloud Registry (GCR): + +Image name: ``gcr.io/panoptes-exp/panoptes-pocs`` + +Tags: ``latest``, ``develop``, and ``developer-env``. + +Setup +~~~~~ + +To build the images locally: + +.. code:: bash + + docker/setup-local-environment.sh + +To run the test suite locally: + +.. code:: bash + + scripts/testing/test-software.sh + +This will build all three images locally and is suitable for testing and development. + +Description +~~~~~~~~~~~ + +The ``panoptes-pocs`` image comes in three separate flavors, or tags, +that serve different purposes. + +latest +^^^^^^ + +The ``latest`` image is the "production" version of ``panoptes-pocs``. + +PANOPTES units should be running this flavor. + +When running the install script, this will be the default install option unless the "developer" is selected. + +develop +^^^^^^^ + +The ``develop`` image is used for running the automated tests. These are +run automatically on both GitHub and Travis for all code pushes but can +also be run locally while doing development. + +developer-env +^^^^^^^^^^^^^ + +The ``developer-env`` image is meant to be be used by developers or anyone wishing to +explore the code. The image should be built locally using the ``docker/setup-local-environment.sh`` +script (or, ideally, just use the ``install-pocs`` script). + +The ``bin/panoptes-develop up`` can then be used to start a docker container +instance that will launch ``jupyter-lab`` from ``$PANDIR`` automatically. + +There are a few ways to get the development version. + +1) If you have ``git`` and are comfortable using the command line: + +.. code-block:: bash + + cd $PANDIR + + # Get the repository. + git clone https://github.com/panoptes/panoptes-pocs.git + cd panoptes-pocs + + # Run environment. + bin/panoptes-develop up + +2) If you would like to build your own local docker image: + +.. code-block:: bash + + cd $PANDIR/panoptes-pocs + # First build the 'latest' image locally. + docker build -t panoptes-pocs:latest -f docker/latest.Dockerfile . + + # Then build the develop image locally. + docker build \ + --build-arg base_image=panoptes-pocs:latest \ + -t panoptes-pocs:develop \ + -f docker/develop.Dockerfile . + + # Wait for build to finish... + + # Run with new image. + IMAGE=panoptes-pocs bin/panoptes-develop up + +3) If you are using a new system: + + TODO: Document this section. + diff --git a/docker/build-image.sh b/docker/build-image.sh index ee8510ab2..e559eced3 100755 --- a/docker/build-image.sh +++ b/docker/build-image.sh @@ -6,7 +6,7 @@ TAG="${1:-develop}" cd "${SOURCE_DIR}" -echo "Building gcr.io/panoptes-exp/pocs:${TAG}" +echo "Building gcr.io/panoptes-exp/panoptes-pocs:${TAG}" gcloud builds submit \ --timeout="1h" \ --substitutions="_TAG=${TAG}" \ diff --git a/docker/cloudbuild.yaml b/docker/cloudbuild.yaml index 882d41ebf..0372db9ac 100644 --- a/docker/cloudbuild.yaml +++ b/docker/cloudbuild.yaml @@ -1,19 +1,22 @@ steps: -- name: 'docker' - id: 'amd64-build' - args: - - 'build' - - '--build-arg image_url=gcr.io/panoptes-exp/panoptes-utils:${_TAG}' - - '-f=docker/${_TAG}.Dockerfile' - - '--tag=gcr.io/${PROJECT_ID}/panoptes-pocs:${_TAG}' - - '.' + - name: 'gcr.io/cloud-builders/docker' + id: 'amd64-build' + args: + - 'build' + - '--build-arg' + - 'IMAGE_URL=gcr.io/panoptes-exp/panoptes-utils:${_TAG}' + - '-f' + - 'docker/${_TAG}.Dockerfile' + - '--tag' + - 'gcr.io/${PROJECT_ID}/panoptes-pocs:${_TAG}' + - '.' -- name: 'docker' - id: 'amd64-push' - args: - - 'push' - - 'gcr.io/${PROJECT_ID}/panoptes-pocs:${_TAG}' - waitFor: ['amd64-build'] + - name: 'gcr.io/cloud-builders/docker' + id: 'amd64-push' + args: + - 'push' + - 'gcr.io/${PROJECT_ID}/panoptes-pocs:${_TAG}' + waitFor: ['amd64-build'] images: - 'gcr.io/${PROJECT_ID}/panoptes-pocs:${_TAG}' diff --git a/docker/develop.Dockerfile b/docker/develop.Dockerfile new file mode 100644 index 000000000..57b2e3bc1 --- /dev/null +++ b/docker/develop.Dockerfile @@ -0,0 +1,38 @@ +ARG IMAGE_URL=gcr.io/panoptes-exp/panoptes-pocs:latest +FROM ${IMAGE_URL} + +LABEL description="Installs the local folder in develop mode (i.e. pip install .e). \ +Used for running the tests and as a base for the for developer-env image." +LABEL maintainers="developers@projectpanoptes.org" +LABEL repo="github.com/panoptes/POCS" + +ARG pan_dir=/var/panoptes +ARG pocs_dir="${pan_dir}/POCS" + +ENV DEBIAN_FRONTEND=noninteractive +ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 +ENV SHELL /bin/zsh + +ENV PANUSER=panoptes +ENV PANDIR $pan_dir +ENV POCS $pocs_dir +ENV SOLVE_FIELD /usr/bin/solve-field + +# panoptes-utils +USER ${PANUSER} +COPY --chown=panoptes:panoptes . "${PANDIR}/POCS/" +RUN cd "${PANDIR}/POCS" && \ + pip install -e ".[testing,google]" + +# Cleanup apt. +USER root +RUN apt-get autoremove --purge -y && \ + apt-get -y clean && \ + rm -rf /var/lib/apt/lists/* && \ + chown -R "${PANUSER}:${PANUSER}" "${PANDIR}" && \ + chmod -R 777 /astrometry + +WORKDIR ${POCS} + +# Entrypoint runs gosu with panoptes user. +CMD ["/bin/zsh"] diff --git a/docker/developer-env.Dockerfile b/docker/developer-env.Dockerfile new file mode 100644 index 000000000..0a692d6ed --- /dev/null +++ b/docker/developer-env.Dockerfile @@ -0,0 +1,67 @@ +ARG BASE_IMAGE=panoptes-pocs:develop +FROM ${BASE_IMAGE} + +LABEL description="Installs the local folder in develop mode (i.e. pip install .e). \ +and installs a number of developer tools. Runs jupyter lab instance. This assumes the \ +`panoptes-pocs:develop` has already been built locally." +LABEL maintainers="developers@projectpanoptes.org" +LABEL repo="github.com/panoptes/POCS" + +ARG panuser=panoptes +ARG userid=1000 +ARG pan_dir=/var/panoptes +ARG pocs_dir="${pan_dir}/POCS" + +ENV DEBIAN_FRONTEND=noninteractive +ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 +ENV SHELL /bin/zsh + +ENV USERID $userid +ENV PANDIR $pan_dir +ENV PANLOG "$pan_dir/logs" +ENV PANUSER $panuser +ENV POCS $pocs_dir +ENV PATH "/home/${PANUSER}/.local/bin:$PATH" + +RUN apt-get update && \ + # Node for jupyterlab. + curl -sL https://deb.nodesource.com/setup_12.x | bash - && \ + # Make a developer's life easier. + apt-get install -y --no-install-recommends \ + wget curl bzip2 ca-certificates nano neovim \ + gcc git pkg-config ncdu sudo nodejs && \ + echo "$PANUSER ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers + + +USER $PANUSER +# Can't seem to get around the hard-coding +COPY --chown=panoptes:panoptes . ${PANDIR}/POCS/ +RUN cd ${PANDIR}/POCS && \ + # Install everything! + pip install -e ".[google,developer,plotting,testing]" && \ + # Set some jupyterlab defaults. + mkdir -p /home/panoptes/.jupyter && \ + jupyter-lab --generate-config && \ + # Jupyterlab extensions. + echo "c.JupyterApp.answer_yesBool = True" >> \ + "/home/panoptes/.jupyter/jupyter_notebook_config.py" && \ + echo "c.JupyterApp.open_browserBool = False" >> \ + "/home/panoptes/.jupyter/jupyter_notebook_config.py" && \ + echo "c.JupyterAppy.notebook_dir = '${PANDIR}'" >> \ + "/home/panoptes/.jupyter/jupyter_notebook_config.py" && \ + jupyter labextension install @pyviz/jupyterlab_pyviz \ + jupyterlab-drawio \ + @aquirdturtle/collapsible_headings \ + @telamonian/theme-darcula + +USER root + +# Cleanup apt. +RUN apt-get autoremove --purge -y && \ + apt-get -y clean && \ + rm -rf /var/lib/apt/lists/* + +WORKDIR ${PANDIR} + +# Start a jupyterlab instance. +CMD ["/home/panoptes/.local/bin/jupyter-lab"] diff --git a/docker/docker-compose-developer-env.yaml b/docker/docker-compose-developer-env.yaml new file mode 100644 index 000000000..1416a4528 --- /dev/null +++ b/docker/docker-compose-developer-env.yaml @@ -0,0 +1,19 @@ +version: '3.7' +services: + develop-env: + image: "${IMAGE:-panoptes-pocs}:${TAG:-developer-env}" + init: true + container_name: "${CONTAINER_NAME:-pocs-developer-env}" + privileged: true + network_mode: host + env_file: $PANDIR/env + volumes: + - pandir:/var/panoptes +volumes: + pandir: + driver: local + driver_opts: + type: none + device: /var/panoptes + o: bind + diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 6acf52339..5cb77c273 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -1,7 +1,7 @@ version: '3.7' services: peas-shell: - image: gcr.io/panoptes-exp/pocs:latest + image: gcr.io/panoptes-exp/panoptes-pocs:latest init: true container_name: peas-shell hostname: peas-shell @@ -19,7 +19,7 @@ services: - "-f" - "/dev/null" pocs-shell: - image: gcr.io/panoptes-exp/pocs:latest + image: gcr.io/panoptes-exp/panoptes-pocs:latest init: true container_name: pocs-shell hostname: pocs-shell diff --git a/docker/latest.Dockerfile b/docker/latest.Dockerfile index 202d82b3b..7416b4776 100644 --- a/docker/latest.Dockerfile +++ b/docker/latest.Dockerfile @@ -1,43 +1,35 @@ -ARG image_url=gcr.io/panoptes-exp/panoptes-utils:testing +ARG IMAGE_URL=gcr.io/panoptes-exp/panoptes-utils:latest +FROM ${IMAGE_URL} AS pocs-base -FROM $image_url AS pocs-base -LABEL maintainer="developers@projectpanoptes.org" +LABEL description="Installs the panoptes-pocs module from pip. \ +Used as a production image, i.e. for running on PANOPTES units." +LABEL maintainers="developers@projectpanoptes.org" +LABEL repo="github.com/panoptes/POCS" ARG pandir=/var/panoptes -ARG arduino_url="https://downloads.arduino.cc/arduino-cli/arduino-cli_latest_Linux_64bit.tar.gz" +ARG arduino_url="https://raw.githubusercontent.com/arduino/arduino-cli/master/install.sh" +ARG gphoto2_url="https://raw.githubusercontent.com/gonzalo/gphoto2-updater/master/gphoto2-updater.sh" ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 ENV SHELL /bin/zsh ENV PANDIR $pandir ENV POCS ${PANDIR}/POCS -ENV USER panoptes +ENV SOLVE_FIELD /usr/bin/solve-field RUN apt-get update \ && apt-get install --no-install-recommends --yes \ gcc libncurses5-dev udev \ # GPhoto2 - && wget https://raw.githubusercontent.com/gonzalo/gphoto2-updater/master/gphoto2-updater.sh \ + && wget $gphoto2_url \ && chmod +x gphoto2-updater.sh \ && /bin/bash gphoto2-updater.sh --stable \ && rm gphoto2-updater.sh \ # arduino-cli - && wget -q $arduino_url -O arduino-cli.tar.gz \ - # Untar and capture output name (NOTE: assumes only one file). - && tar xvfz arduino-cli.tar.gz \ - && mv arduino-cli /usr/local/bin/arduino-cli \ - && chmod +x /usr/local/bin/arduino-cli - -COPY ./requirements.txt /tmp/requirements.txt -# First deal with pip and PyYAML - see https://github.com/pypa/pip/issues/5247 -RUN pip install --no-cache-dir --no-deps --ignore-installed pip PyYAML && \ - pip install --no-cache-dir -r /tmp/requirements.txt - -# Install module -COPY . ${POCS}/ -RUN cd ${POCS} && pip install -e ".[google]" + && curl -fsSL $arduino_url | BINDIR="/usr/local/bin" sh \ + # Install the module. + && pip install "panoptes-pocs[google]" # Cleanup apt. -USER root RUN apt-get autoremove --purge -y \ autoconf \ automake \ diff --git a/docker/setup-local-environment.sh b/docker/setup-local-environment.sh new file mode 100755 index 000000000..385adf24b --- /dev/null +++ b/docker/setup-local-environment.sh @@ -0,0 +1,45 @@ +#!/bin/bash -e + +POCS=${POCS:-/var/panoptes/POCS} +TAG="${1:-develop}" + +cd "${POCS}" + +echo "Building local panoptes-pocs:latest" +docker build \ + --quiet \ + -t "panoptes-pocs:latest" \ + -f "${POCS}/docker/latest.Dockerfile" \ + "${POCS}" + +# In the local develop we need to pass git to the docker build context. +sed -i s'/^\.git$/\!\.git/' .dockerignore + +echo "Building local panoptes-pocs:develop" +docker build \ + --quiet \ + --build-arg IMAGE_URL="panoptes-pocs:latest" \ + -t "panoptes-pocs:develop" \ + -f "${POCS}/docker/develop.Dockerfile" \ + "${POCS}" + +echo "Building local panoptes-pocs:developer-env" +docker build \ + --quiet \ + --build-arg IMAGE_URL="panoptes-pocs:develop" \ + -t "panoptes-pocs:developer-env" \ + -f "${POCS}/docker/developer-env.Dockerfile" \ + "${POCS}" + +# Revert our .dockerignore changes. +sed -i s'/^!\.git$/\.git/' .dockerignore + +cat < - Authors + Docker Guider + Contributing Guide Changelog Module Reference - Contributing Guide + Authors + License Indices and tables diff --git a/scripts/install/install-pocs.sh b/scripts/install/install-pocs.sh index 17bee90d8..d08174b27 100755 --- a/scripts/install/install-pocs.sh +++ b/scripts/install/install-pocs.sh @@ -14,44 +14,66 @@ usage() { # or # $ wget -O - https://install.projectpanoptes.org | bash # -# The script will insure that Docker is installed, download the -# latest Docker images (see list below) and clone a copy of the -# relevant PANOPTES repositories. +# The script will do the following: +# +# * Create the needed directory structure. +# * Ensure that docker and docker-compose are installed. +# * Fetch and/or build the docker images needed to run. +# * If in "developer" mode, clone user's fork and set panoptes upstream. +# * Write the environment variables to $PANDIR/env # # Docker Images: # # ${DOCKER_BASE}/panoptes-utils # ${DOCKER_BASE}/pocs # -# The script will ask for a github user name. If you are a developer -# you can enter your github username to work from your fork. Otherwise -# the default user (panoptes) is okay for running the unit. # -# The script has been tested with a fresh install of Ubuntu 19.04 +# The script will ask if it should be installed in "developer" mode or not. +# +# The regular install is for running units and will not create local (to the +# host system) copies of the files. +# +# The "developer" mode will ask for a github username and will clone and +# fetch the repos. The `docker/setup-local-enviornment.sh` script will then +# be run to build the docker images locally. +# +# If not in "developer" mode, the docker images will be pulled from GCR. +# +# The script has been tested with a fresh install of Ubuntu 20.04 # but may work on other linux systems. ############################################################# - $ $(basename $0) [--user panoptes] [--pandir /var/panoptes] + $ $(basename $0) [--developer] [--user panoptes] [--pandir /var/panoptes] Options: - USER The PANUSER environment variable, defaults to current user (i.e. USER=`$USER`). - PANDIR Default install directory, defaults to /var/panoptes. Saved as PANDIR + DEVELOPER Install POCS in developer mode, default False. + + If in DEVELOPER mode, the following options are also available: + USER The PANUSER environment variable, defaults to current user (i.e. PANUSER=$USER). + PANDIR Default install directory, defaults to PANDIR=${PANDIR}. Saved as PANDIR environment variable. " } -DOCKER_BASE="gcr.io/panoptes-exp" -if [ -z "${PANUSER}" ]; then - export PANUSER=$USER -fi -if [ -z "${PANDIR}" ]; then - export PANDIR='/var/panoptes' -fi +DEVELOPER=${DEVELOPER:-false} +PANUSER=${PANUSER:-$USER} +PANDIR=${PANDIR:-/var/panoptes} +LOGFILE="${PANDIR}/install-pocs.log" +OS="$(uname -s)" +ARCH="$(uname -m)" + +DOCKER_COMPOSE_VERSION="${DOCKER_COMPOSE_VERSION:-1.26.0}" +DOCKER_COMPOSE_INSTALL="https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-${OS}-${ARCH}" +DOCKER_BASE=${DOCKER_BASE:-"gcr.io/panoptes-exp"} while [[ $# -gt 0 ]] do key="$1" -case $key in +case ${key} in + --developer) + DEVELOPER=true + shift # past bool argument + ;; -u|--user) PANUSER="$2" shift # past argument @@ -70,6 +92,14 @@ case $key in esac done +if "${DEVELOPER}"; then + while [[ -z "${GITHUB_USER}" ]]; do + read -p "Github User [NOTE: you must have a fork created already]: " GITHUB_USER + done +fi + +echo "DEVELOPER=${DEVELOPER} PANDIR=${PANDIR} PANUSER=${PANUSER} GITHUB_USER=${GITHUB_USER}" + function command_exists { # https://gist.github.com/gubatron/1eb077a1c5fcf510e8e5 # this should be a very portable way of checking if something is on the path @@ -77,89 +107,100 @@ function command_exists { type "$1" &> /dev/null } -do_install() { - clear - - OS="$(uname -s)" - case "${OS}" in - Linux*) machine=Linux;; - Darwin*) machine=Mac;; - *) machine="UNKNOWN:${unameOut}" - esac - echo ${machine} - - # Install directory - read -p "PANOPTES base directory [${PANDIR:-/var/panoptes}]: " PANDIR - PANDIR=${PANDIR:-/var/panoptes} - - LOGFILE="${PANDIR}/logs/install-pocs.log" - - echo "Installing PANOPTES software." - echo "USER: ${PANUSER}" - echo "OS: ${OS}" - echo "Base dir: ${PANDIR}" - echo "Logfile: ${LOGFILE}" - - # Directories +function make_directories { if [[ ! -d "${PANDIR}" ]]; then - echo "Creating directories in ${PANDIR}" - # Make directories + # Make directories and make PANUSER the owner. sudo mkdir -p "${PANDIR}" - sudo chown -R "${PANUSER}":"${PANUSER}" "${PANDIR}" - - mkdir -p "${PANDIR}/logs" - mkdir -p "${PANDIR}/images" - mkdir -p "${PANDIR}/conf_files" - mkdir -p "${PANDIR}/.key" else echo "WARNING ${PANDIR} already exists. You can exit and specify an alternate directory with --pandir or continue." select yn in "Yes" "No"; do - case $yn in + case ${yn} in Yes ) echo "Proceeding with existing directory"; break;; No ) echo "Exiting"; exit 1;; esac done fi - # apt: git, wget - echo "Installing system dependencies" + sudo mkdir -p "${PANDIR}/logs" + sudo mkdir -p "${PANDIR}/images" + sudo mkdir -p "${PANDIR}/config_files" + sudo mkdir -p "${PANDIR}/.key" + sudo chown -R "${PANUSER}":"${PANUSER}" "${PANDIR}" +} + +function setup_env_vars { + ENV_FILE="${PANDIR}/env" + echo "Writing environment variables to ${ENV_FILE}" + if -f "${ENV_FILE}"; then + echo "\n**** Added by install-pocs script ****\n" >> "${ENV_FILE}" + fi + + cat >> "${ENV_FILE}" <> "${LOGFILE}" 2>&1 - sudo apt-get --yes install wget curl git openssh-server ack jq httpie byobu >> "${LOGFILE}" 2>&1 + # TODO(wtgee) figure out why we needed openssh-server on the host. + sudo apt-get --yes install \ + wget curl git openssh-server ack jq httpie byobu \ + >> "${LOGFILE}" 2>&1 elif [[ "${OS}" = "Darwin" ]]; then sudo brew update | sudo tee -a "${LOGFILE}" - sudo brew install wget curl git jq httpie | sudo tee -a "${LOGFILE}" + sudo brew install \ + wget curl git jq httpie \ + | sudo tee -a "${LOGFILE}" fi - echo "Cloning PANOPTES source code." - echo "Github user for PANOPTES repos (POCS, panoptes-utils)." + # Add an SSH key if one doesn't exist. + if [[ ! -f "${HOME}/.ssh/id_rsa" ]]; then + echo "Adding ssh key" + ssh-keygen -t rsa -N "" -f "${HOME}/.ssh/id_rsa"; + fi +} - # Default user - read -p "Github User [if you are a developer, enter your name or press Enter for 'panoptes']: " github_user - github_user=${github_user:-panoptes} - echo "Using repositories from user '${github_user}'." +function get_repos { + PUBLIC_GITHUB_URL="https://github.com/panoptes" - GIT_BRANCH="develop" + if "${DEVELOPER}"; then + echo "Using repositories from user: ${GITHUB_USER}" + declare -a repos=("POCS" "panoptes-utils" "panoptes-tutorials") + GITHUB_URL="git@github.com:${GITHUB_USER}" + else + declare -a repos=("POCS" "panoptes-utils") + GITHUB_URL="${PUBLIC_GITHUB_URL}" + fi - cd "${PANDIR}" - declare -a repos=("POCS" "panoptes-utils") for repo in "${repos[@]}"; do if [[ ! -d "${PANDIR}/${repo}" ]]; then + cd "${PANDIR}" echo "Cloning ${repo}" # Just redirect the errors because otherwise looks like it hangs. - git clone "https://github.com/${github_user}/${repo}.git" >> "${LOGFILE}" 2>&1 + # TODO handle errors if repo doesn't exist (e.g. bad github name). + git clone "https://github.com/${GITHUB_USER}/${repo}.git" >> "${LOGFILE}" 2>&1 + + # Set panoptes as upstream + cd "${repo}" + git remote add upstream "${PUBLIC_GITHUB_URL}/${repo}" else - # TODO Do an update here. - echo "" + # TODO Figure out how to do updates. + echo "${repo} already exists in ${PANDIR}. No auto-update for now, skipping repo." fi done +} +function get_docker { # Get Docker if ! command_exists docker; then echo "Installing Docker" if [[ "${OS}" = "Linux" ]]; then - /bin/bash -c "$(wget -qO- https://get.docker.com)" &>> ${PANDIR}/logs/install-pocs.log + /bin/bash -c "$(wget -qO- https://get.docker.com)" &>> "${LOGFILE}" echo "Adding ${PANUSER} to docker group" sudo usermod -aG docker "${PANUSER}" >> "${LOGFILE}" 2>&1 @@ -168,29 +209,60 @@ do_install() { echo "Adding ${PANUSER} to docker group" sudo dscl -aG docker "${PANUSER}" fi - else - echo "WARNING: Docker images not installed/downloaded." fi if ! command_exists docker-compose; then echo "Installing docker-compose" # Docker compose as container - https://docs.docker.com/compose/install/#install-compose - sudo wget -q https://github.com/docker/compose/releases/download/1.25.4/docker-compose-`uname -s`-`uname -m` -O /usr/local/bin/docker-compose + sudo wget -q "${DOCKER_COMPOSE_INSTALL}" -O /usr/local/bin/docker-compose sudo chmod a+x /usr/local/bin/docker-compose + fi +} + +function get_or_build_images { + if ${DEVELOPER}; then + echo "Building local PANOPTES docker images." + + cd "${POCS}" + ./docker/setup-local-environment.sh + else + echo "Pulling PANOPTES docker images from Google Cloud Registry (GCR)." - docker pull docker/compose + docker pull "${DOCKER_BASE}/panoptes-pocs:latest" + docker pull "${DOCKER_BASE}/panoptes-utils:latest" + docker pull "${DOCKER_BASE}/aag-weather:latest" fi +} - echo "Pulling PANOPTES docker images" - docker pull "${DOCKER_BASE}/panoptes-utils:latest" - docker pull "${DOCKER_BASE}/aag-weather:latest" - docker pull "${DOCKER_BASE}/pocs:latest" +function do_install { + clear - # Add an SSH key if one doesn't exists - if [[ ! -f "${HOME}/.ssh/id_rsa" ]]; then - echo "Looks like you don't have an SSH key set up yet, adding one now." - ssh-keygen -t rsa -N "" -f "${HOME}/.ssh/id_rsa"; + if ${DEVELOPER}; then + echo "" + echo "**** Developer Mode ****" + echo "" fi + echo "Installing PANOPTES software." + echo "PANUSER: ${PANUSER}" + echo "PANDIR: ${PANDIR}" + echo "OS: ${OS}" + echo "Logfile: ${LOGFILE}" + + exit 0; + + echo "Creating directories in ${PANDIR}" + make_directories + + echo "Installing system dependencies" + system_deps + + echo "Installing docker and docker-compose" + get_docker + + echo "Cloning PANOPTES source code" + get_repos + + get_or_build_images echo "Please reboot your machine before using POCS." @@ -201,6 +273,4 @@ do_install() { } -# wrapped up in a function so that we have some protection against only getting -# half the file during "curl | sh" - copied from get.docker.com do_install diff --git a/scripts/testing/test-software.sh b/scripts/testing/test-software.sh index 36f1b6891..99131f077 100755 --- a/scripts/testing/test-software.sh +++ b/scripts/testing/test-software.sh @@ -6,14 +6,14 @@ cat <=0.3.1' -google = - gcloud - google-cloud-storage + responses + [options.entry_points] # Add here console scripts like: From 5c67c190d26ae2d616b832fce218cd8d9d0e9a21 Mon Sep 17 00:00:00 2001 From: Joshua Liberman <65671429+jlibermann@users.noreply.github.com> Date: Fri, 5 Jun 2020 01:07:27 -0400 Subject: [PATCH 02/22] Fixes the install-pocs script (#973) * Fixes the install-pocs script Thanks @jlibermann! --- scripts/install/install-pocs.sh | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) mode change 100755 => 100644 scripts/install/install-pocs.sh diff --git a/scripts/install/install-pocs.sh b/scripts/install/install-pocs.sh old mode 100755 new mode 100644 index d08174b27..33db0f000 --- a/scripts/install/install-pocs.sh +++ b/scripts/install/install-pocs.sh @@ -20,7 +20,7 @@ usage() { # * Ensure that docker and docker-compose are installed. # * Fetch and/or build the docker images needed to run. # * If in "developer" mode, clone user's fork and set panoptes upstream. -# * Write the environment variables to $PANDIR/env +# * Write the environment variables to ${PANDIR}/env # # Docker Images: # @@ -61,11 +61,13 @@ PANDIR=${PANDIR:-/var/panoptes} LOGFILE="${PANDIR}/install-pocs.log" OS="$(uname -s)" ARCH="$(uname -m)" +ENV_FILE="${PANDIR}/env" DOCKER_COMPOSE_VERSION="${DOCKER_COMPOSE_VERSION:-1.26.0}" DOCKER_COMPOSE_INSTALL="https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-${OS}-${ARCH}" DOCKER_BASE=${DOCKER_BASE:-"gcr.io/panoptes-exp"} + while [[ $# -gt 0 ]] do key="$1" @@ -92,6 +94,16 @@ case ${key} in esac done +if ! ${DEVELOPER}; then + echo -n "Are you installing POCS as a developer? (for PANOPTES units, select No)" + select yn in "Yes" "No"; do + case ${yn} in + Yes ) echo "Enabling developer mode. Note that you will need your GitHub username to proceed"; DEVELOPER=true; break;; + No ) echo "Installing POCS in production mode"; break;; + esac + done +fi + if "${DEVELOPER}"; then while [[ -z "${GITHUB_USER}" ]]; do read -p "Github User [NOTE: you must have a fork created already]: " GITHUB_USER @@ -129,10 +141,8 @@ function make_directories { } function setup_env_vars { - ENV_FILE="${PANDIR}/env" - echo "Writing environment variables to ${ENV_FILE}" - if -f "${ENV_FILE}"; then + if [[ -f "${ENV_FILE}" ]]; then echo "\n**** Added by install-pocs script ****\n" >> "${ENV_FILE}" fi @@ -248,11 +258,15 @@ function do_install { echo "OS: ${OS}" echo "Logfile: ${LOGFILE}" - exit 0; + echo "Creating directories in ${PANDIR}" make_directories + echo "Setting up environment variables in ${ENV_FILE}" + setup_env_vars + + echo "Installing system dependencies" system_deps @@ -271,6 +285,7 @@ function do_install { sudo reboot fi + exit 0; } do_install From 727049b1d2eb716022ab3676c3e4e27c53f3759e Mon Sep 17 00:00:00 2001 From: Joshua Liberman <65671429+jlibermann@users.noreply.github.com> Date: Mon, 8 Jun 2020 23:37:58 -0400 Subject: [PATCH 03/22] Adds a line that exports environment variables to the bashrc (#975) * Adds a line that exports environment variables to the bashrc * cleans up line spacing * Update scripts/install/install-pocs.sh Thanks @jlibermann --- scripts/install/install-pocs.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/install/install-pocs.sh b/scripts/install/install-pocs.sh index 33db0f000..5ad6a7755 100644 --- a/scripts/install/install-pocs.sh +++ b/scripts/install/install-pocs.sh @@ -152,6 +152,8 @@ export PANDIR=${PANDIR} export POCS=${PANDIR}/POCS export PANLOG=${PANDIR}/logs EOF + + [ ! -f /var/panoptes/env ] && echo '. /var/panoptes/env' >> ~/.bashrc } function system_deps { From 216407bbf8b12a0d021e0688f8c9a7442cfeb0f6 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Sun, 14 Jun 2020 16:02:26 -1000 Subject: [PATCH 04/22] More cleanup to make dev env work. (#974) * More cleanup to make dev env work. * Minor cleanups. * * Bumping panoptes-utils version. * * Updates to tests and sitecustomize for coverage to match panoptes-utils. * * Python to 3.8 * Markdown readme for github repo. * Install script writes to bash and zsh if rc files are present. * * Test fixes. * * Remove the dynamic_config_server fixture. * * Fix build and test to run properly. * * Use standard port and host for config server (but still a different path). * * Removing `config_port` items. * * Removing `config_port` items. * * Fix test assertion. * Scheduler * The `checks_file` config value works the same at init. * Scheduler * test fixes. * Developer Environment * Automatically start up the config server. * Fix mount simulator params. * * Use docker-compose and `panoptes-develop` to run tests. * * Use docker-compose and `panoptes-develop` to run tests. * * Removing `test-software` references. * * Fixing tests. * * Just run all test for now. * * Add reset for conf in mount tests. * * Add reset for conf in pocs tests. * Moving some config settings directly to file rather than setting in conftest. * Change back directories images. * Config server in developer env. * Changelog updates. * Fix observatory tests. * Fix observatory tests. * Logging on set config. * Logging on set config. * * POCS testing observatory uses weather and power simulator. * Config server works at module scope. * * POCS testing observatory uses weather and power simulator. * Config server works at module scope. * * Python to 3.8 in setup.cfg. * Set `simulator` config item at start of `POCS` init method if `simulators` (note plural) is passed. * Simplification of the `run` method and the various predicates used to control it. Now just use the computed `keep_running`. * Adding some action flags to the `pocs.yaml` file. * POCS instance cannot `initialize` unless it's `observatory.can_observe`. * Cleanup of `PanBase` for db creation. * Moving some logging items to `trace` level. * Documentation updates. * * Test cleanup. * Fixing tests and clean up to machine running. * Fixing tests and clean up to machine running. * Test updates. * Test updates. * Test updates. * Adding some coverage for the hardware file. * Adding some coverage for custom state file. * Adding some coverage for custom state file. * * Ignore doctest import errors. * * Ignore doctest import errors. * * Ignore doctest import errors. * * Ignore doctest import errors. * * Ignore doctest import errors. * * Ignore doctest import errors. * * Ignore doctest import errors. * * Ignore doctest import errors. * * Move pocs end to end test to end. ;) * Code style cleanup * * Move pocs end to end test to end. ;) * Code style cleanup * * Make sure night time for full test. * * Remove `POCS.check_environment` * Changelog --- .dockerignore | 9 +- codecov.yml => .github/codecov.yml | 1 - .github/workflows/pythontest.yaml | 4 +- CHANGELOG.rst | 47 +++- CONTRIBUTING.rst | 10 +- README.md | 82 +++++++ bin/panoptes-develop | 38 ++- conf_files/pocs.yaml | 132 ++++++----- conftest.py | 190 +++++---------- docker/README.rst | 4 +- docker/develop.Dockerfile | 2 +- docker/docker-compose-developer-env.yaml | 16 ++ docker/docker-compose-testing.yaml | 25 ++ docker/latest.Dockerfile | 2 +- docker/setup-local-environment.sh | 41 +++- README.rst => docs/README.rst | 32 ++- docs/index.rst | 2 +- scripts/install/install-pocs.sh | 6 +- scripts/testing/run-tests.sh | 24 +- scripts/testing/test-software.sh | 31 --- setup.cfg | 27 +-- .../sitecustomize.py => sitecustomize.py | 1 + src/panoptes/pocs/base.py | 34 ++- src/panoptes/pocs/camera/__init__.py | 102 ++++---- src/panoptes/pocs/camera/camera.py | 32 ++- src/panoptes/pocs/core.py | 151 ++++++------ src/panoptes/pocs/dome/__init__.py | 18 +- src/panoptes/pocs/hardware.py | 46 +++- src/panoptes/pocs/mount/__init__.py | 28 +-- src/panoptes/pocs/mount/mount.py | 2 +- src/panoptes/pocs/mount/simulator.py | 26 +- src/panoptes/pocs/observatory.py | 64 ++--- src/panoptes/pocs/scheduler/__init__.py | 28 +-- src/panoptes/pocs/scheduler/field.py | 23 +- src/panoptes/pocs/scheduler/scheduler.py | 50 ++-- src/panoptes/pocs/state/machine.py | 157 ++++++------ .../pocs/state/states/default/pointing.py | 1 - src/panoptes/pocs/tests/bisque/test_mount.py | 4 +- src/panoptes/pocs/tests/bisque/test_run.py | 8 +- .../pocs/tests/test_astrohaven_dome.py | 8 +- src/panoptes/pocs/tests/test_base.py | 27 +-- .../pocs/tests/test_base_scheduler.py | 66 ++---- src/panoptes/pocs/tests/test_camera.py | 91 ++++--- src/panoptes/pocs/tests/test_constraints.py | 10 +- .../pocs/tests/test_dispatch_scheduler.py | 31 +-- .../pocs/tests/test_dome_simulator.py | 6 +- src/panoptes/pocs/tests/test_filterwheel.py | 39 ++- src/panoptes/pocs/tests/test_focuser.py | 7 +- src/panoptes/pocs/tests/test_images.py | 46 ++-- src/panoptes/pocs/tests/test_ioptron.py | 15 +- src/panoptes/pocs/tests/test_mount.py | 80 ++++--- .../pocs/tests/test_mount_simulator.py | 28 +-- src/panoptes/pocs/tests/test_observation.py | 74 +++--- src/panoptes/pocs/tests/test_observatory.py | 134 ++++++----- src/panoptes/pocs/tests/test_pocs.py | 224 ++++++++++-------- src/panoptes/pocs/tests/test_scheduler.py | 43 +++- src/panoptes/pocs/tests/test_state_machine.py | 8 +- src/panoptes/pocs/utils/location.py | 25 +- src/panoptes/pocs/utils/logger.py | 3 +- tests/pocs_testing.yaml | 8 +- 60 files changed, 1295 insertions(+), 1178 deletions(-) rename codecov.yml => .github/codecov.yml (96%) create mode 100644 README.md create mode 100644 docker/docker-compose-testing.yaml rename README.rst => docs/README.rst (75%) delete mode 100755 scripts/testing/test-software.sh rename scripts/coverage/sitecustomize.py => sitecustomize.py (99%) diff --git a/.dockerignore b/.dockerignore index 26790adf8..d04f9f888 100644 --- a/.dockerignore +++ b/.dockerignore @@ -10,9 +10,16 @@ venv !README*.md logs/ +**/.ipynb_checkpoints +**/.pytest_cache **/.eggs **/*.pdf **/*.log **/*.egg-info **/*.pyc -**/__pycache__ \ No newline at end of file +**/__pycache__ +notebooks +examples +docs +build +coverage.xml \ No newline at end of file diff --git a/codecov.yml b/.github/codecov.yml similarity index 96% rename from codecov.yml rename to .github/codecov.yml index 3360aac18..00297b4ae 100644 --- a/codecov.yml +++ b/.github/codecov.yml @@ -7,7 +7,6 @@ ignore: - "pocs/camera/libfliconstants.py" - "pocs/camera/zwo.py" - "pocs/camera/libasi.py" - - "pocs/tests/*" - "pocs/tests/bisque/*" - "pocs/dome/bisque.py" - "pocs/mount/bisque.py" diff --git a/.github/workflows/pythontest.yaml b/.github/workflows/pythontest.yaml index e38fcf964..7fc6d7630 100644 --- a/.github/workflows/pythontest.yaml +++ b/.github/workflows/pythontest.yaml @@ -6,7 +6,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7] + python-version: [3.8] steps: - name: Checkout code uses: actions/checkout@v2 @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7] + python-version: [3.8] steps: - name: Checkout code uses: actions/checkout@v2 diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4dba3c78a..e5e022ef9 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -12,14 +12,47 @@ adheres to `Semantic Versioning `__. Changed ~~~~~~~ +* Python moved to 3.8. (#974) +* `panoptes-utils` to `0.2.20`. (#974) +* Install script. (#974) + + * Env var file is sourced for zshrc and bashrc. + +* Development Environment (#974) + + * Many cleanups to environment and launch. See docs. + * Config server started along with development environment. + * Docker image updates (#972) - * Updated `install-pocs.sh` script. - * ``latest`` installs the ``panoptes-pocs`` module from pip - * ``develop`` installs via ``pip install -e[google.testing]`` and is used for running the CI tests. - * ``developer-env`` installs locally but with all options, i.e. ``pip install -e[google,testing,plotting,developer]``. Also builds ``jupyterlab`` and other developer tools. Starts a ``jupyterlab`` instance by default. - * Use new ``arduino-cli`` installer. - * Add ``bin/panoptes-develop`` and ``bin/wait-for-it.sh`` to installed scripts. - * Add ``docker/setup-local-environment.sh``, a convenience script for building local images. + + * Updated `install-pocs.sh` script. + * ``latest`` installs the ``panoptes-pocs`` module from pip + * ``develop`` installs via ``pip install -e[google.testing]`` and is used for running the CI tests. + * ``developer-env`` installs locally but with all options, i.e. ``pip install -e[google,testing,plotting,developer]``. Also builds ``jupyterlab`` and other developer tools. Starts a ``jupyterlab`` instance by default. + * Use new ``arduino-cli`` installer. + * Add ``bin/panoptes-develop`` and ``bin/wait-for-it.sh`` to installed scripts. + * Add ``docker/setup-local-environment.sh``, a convenience script for building local images. + +* Testing (#974) + + * Removing all the dynamic config server info, making things a lot simpler. + * `docker-compose` files for running tests. + * Misc documentation updates. + * Code coverage no longer ignores test. + * Testing is run via `panoptes-develop test`. + * Log files are rotated during each run. + +* POCS (#974) + + * POCS instance cannot `initialize` unless it's `observatory.can_observe`. + * Set `simulator` config item at start of `POCS` init method if `simulators` (note plural) is passed. + * Simplification of the `run` method and the various predicates used to control it. Now just use the computed `keep_running`. + * Adding some action flags to the `pocs.yaml` file. + * Remove `POCS.check_environment` class method. + +* Camera simulator cleanup. (#974) +* Scheduler (#974) + * The `fields_file` is read when scheduler is created. [0.7.4] - 2020-05-31 ---------- diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index e95bb6ca8..1cf8cafb5 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -182,12 +182,12 @@ In order to test your installation you should have followed all of the steps abo for getting your unit ready. To run the test suite, you will need to open a terminal and navigate to the ``$POCS`` directory. -.. code:: bash +.. code-block:: bash cd $POCS # Run the software testing - scripts/testing/test-software.sh + panoptes-develop test .. note:: @@ -197,7 +197,7 @@ and navigate to the ``$POCS`` directory. It is often helpful to view the log output in another terminal window while the test suite is running: -.. code:: bash +.. code-block:: bash # Follow the log file tail -F $PANDIR/logs/panoptes.log @@ -215,7 +215,7 @@ to github. This can be done either by running the entire test suite as above or by running an individual test related to the code you are changing. For instance, to test the code related to the cameras one can run: -.. code:: bash +.. code-block:: bash pytest -xv pocs/tests/test_camera.py @@ -256,7 +256,7 @@ and ``weather``. Optionally you can use ``all`` to test a fully connected unit. connected but does not test the safety conditions. It is assumed that hardware testing is always done with direct supervision. -.. code:: bash +.. code-block:: bash # Test an attached camera pytest --with-hardware=camera diff --git a/README.md b/README.md new file mode 100644 index 000000000..5fe691d5e --- /dev/null +++ b/README.md @@ -0,0 +1,82 @@ +Welcome to POCS documentation! +============================== + +

+PAN001 +

+
+ +[![GHA Status](https://img.shields.io/endpoint.svg?url=https%3A%2F%2Factions-badge.atrox.dev%2Fpanoptes%2FPOCS%2Fbadge%3Fref%3Ddevelop&style=flat)](https://actions-badge.atrox.dev/panoptes/POCS/goto?ref=develop) [![Travis Status](https://travis-ci.com/panoptes/POCS.svg?branch=develop)](https://travis-ci.com/panoptes/POCS) [![codecov](https://codecov.io/gh/panoptes/POCS/branch/develop/graph/badge.svg)](https://codecov.io/gh/panoptes/POCS) [![Documentation Status](https://readthedocs.org/projects/panoptes-pocs/badge/?version=latest)](https://panoptes-pocs.readthedocs.io/en/latest/?badge=latest) [![PyPI version](https://badge.fury.io/py/panoptes-pocs.svg)](https://badge.fury.io/py/panoptes-pocs) + +# Project PANOPTES + +[PANOPTES](https://www.projectpanoptes.org) is an open source citizen science project +designed to find [transiting exoplanets](https://spaceplace.nasa.gov/transits/en/) with +digital cameras. The goal of PANOPTES is to establish a global network of of robotic +cameras run by amateur astronomers and schools (or anyone!) in order to monitor, +as continuously as possible, a very large number of stars. For more general information +about the project, including the science case and resources for interested individuals, see the +[project overview](https://projectpanoptes.org/articles/). + +# POCS + +POCS (PANOPTES Observatory Control System) is the main software driver for a +PANOPTES unit, responsible for high-level control of the unit. + +For more information, see the full documentation at: https://pocs.readthedocs.io. + +[`panoptes-utils`](https://www.github.com/panoptes/panoptes-utils) is a related repository and POCS +relies on most of the tools within `panoptes-utils`. + +## Install + +### POCS Environment + +If you are running a PANOPTES unit then you will most likely want an entire PANOPTES environment. + +There is a bash shell script that will install an entire working POCS system on your computer. Some +folks even report that it works on a Mac. + +To test the script, open a terminal and enter: + +```bash +curl -L https://install.projectpanoptes.org | bash +``` + +Or using `wget`: + +```bash +wget -O - https://install.projectpanoptes.org | bash +``` + + +### POCS Module + +If you want just the POCS module, for instance if you want to override it in +your own OCS (see [Huntsman-POCS](https://github.com/AstroHuntsman/huntsman-pocs) +for an example), then install via `pip`: + +```bash +pip install panoptes-pocs +``` + +If you want the extra features, such as Google Cloud Platform connectivity, then +use the extras options: + +```bash +pip install "panoptes-pocs[google,testing]" +``` + +See the full documentation at: https://pocs.readthedocs.io + +### For helping develop POCS software + +See [Coding in PANOPTES](https://github.com/panoptes/POCS/wiki/Coding-in-PANOPTES) + +Links +----- + +- PANOPTES Homepage: https://www.projectpanoptes.org +- Forum: https://forum.projectpanoptes.org +- Documentation: https://pocs.readthedocs.io +- Source Code: https://github.com/panoptes/POCS diff --git a/bin/panoptes-develop b/bin/panoptes-develop index a4cc90264..b62212326 100755 --- a/bin/panoptes-develop +++ b/bin/panoptes-develop @@ -1,35 +1,47 @@ -#!/usr/bin/bash +#!/usr/bin/env bash set -e -PARAMS="$*" +SUBCMD=$1 export PANDIR=${PANDIR:-/var/panoptes} export IMAGE="${IMAGE:-panoptes-pocs}" export TAG="${TAG:-developer-env}" -export CONTAINER_NAME="${CONTAINER_NAME:-pocs-developer-env}" cd "${PANDIR}" -CMD="docker-compose \ - --project-directory ${PANDIR} \ - --env-file ${PANDIR}/env \ - -f POCS/docker/docker-compose-developer-env.yaml \ - -p panoptes" +## Add the deamon option by default. +if [[ "${SUBCMD}" == "up" ]]; then + SUBCMD="up -d" + export CONTAINER_NAME="pocs-developer-env" + export COMPOSE_FILE="${PANDIR}/POCS/docker/docker-compose-developer-env.yaml" +fi ## Add the deamon option by default. -if [[ "$PARAMS" == "up" ]]; then - PARAMS="up -d" +if [[ "${SUBCMD}" == "test" ]]; then + SUBCMD="up" + export CONTAINER_NAME="pocs-develop-testing" + export COMPOSE_FILE="${PANDIR}/POCS/docker/docker-compose-testing.yaml" fi +# Pass any other cli args to the containers as an env var named CLI_ARGS +CLI_ARGS=("${@:2}") + # We use a docker container for docker-compose, so we need to pass the env vars to # that container so it can properly place them in the docker-compose file. -export DOCKER_RUN_OPTIONS="${DOCKER_RUN_OPTIONS:--e IMAGE=${IMAGE} -e TAG=${TAG} -e CONTAINER_NAME=${CONTAINER_NAME}}" +export DOCKER_RUN_OPTIONS="${DOCKER_RUN_OPTIONS:--e IMAGE=${IMAGE} -e TAG=${TAG} -e CONTAINER_NAME=${CONTAINER_NAME} -e CLI_ARGS=\"${CLI_ARGS}\"}" # Run the docker-compose command with user params. -eval "DOCKER_RUN_OPTIONS=\"${DOCKER_RUN_OPTIONS}\" ${CMD} ${PARAMS}" +eval "DOCKER_RUN_OPTIONS=\"${DOCKER_RUN_OPTIONS}\" \ + docker-compose \ + --project-directory ${PANDIR} \ + --env-file ${PANDIR}/env \ + -f ${COMPOSE_FILE} \ + -p panoptes \ + ${SUBCMD} \ + ${PARAMS}" # If we just started the environment, try to open the browser for the user. -if [[ "$PARAMS" == "up -d" ]]; then +if [[ "${SUBCMD}" == "up -d" ]]; then # Prompt for password "${PANDIR}/POCS/bin/wait-for-it.sh" \ localhost:8888 \ diff --git a/conf_files/pocs.yaml b/conf_files/pocs.yaml index 5be6c3e56..5c85fec59 100644 --- a/conf_files/pocs.yaml +++ b/conf_files/pocs.yaml @@ -8,68 +8,71 @@ # files and communicate with the Google Cloud network. # # Leave the pan_id at `PAN000` for testing until you have been assigned -# an official id. Update pocs_local.yaml with offical name once received. +# an official id. Update pocs_local.yaml with official name once received. ################################################################################ name: Generic PANOPTES Unit pan_id: PAN000 location: - name: Mauna Loa Observatory - latitude: 19.54 deg - longitude: -155.58 deg - elevation: 3400.0 m - horizon: 30 deg # targets must be above this to be considered valid. - flat_horizon: -6 deg # Flats when sun between this and focus horizon. - focus_horizon: -12 deg # Dark enough to focus on stars. - observe_horizon: -18 deg # Sun below this limit to observe. - obsctructions: [] - timezone: US/Hawaii - gmt_offset: -600 # Offset in minutes from GMT during. - # standard time (not daylight saving). + name: Mauna Loa Observatory + latitude: 19.54 deg + longitude: -155.58 deg + elevation: 3400.0 m + horizon: 30 deg # targets must be above this to be considered valid. + flat_horizon: -6 deg # Flats when sun between this and focus horizon. + focus_horizon: -12 deg # Dark enough to focus on stars. + observe_horizon: -18 deg # Sun below this limit to observe. + obsctructions: [] + timezone: US/Hawaii + gmt_offset: -600 # Offset in minutes from GMT during. + directories: - base: /var/panoptes - images: images - data: data - resources: POCS/resources/ - targets: POCS/resources/targets - mounts: POCS/resources/mounts + base: /var/panoptes + images: images + data: data + resources: POCS/resources/ + targets: POCS/resources/targets + mounts: POCS/resources/mounts + db: - name: panoptes - type: file + name: panoptes + type: file + +wait_delay: 180 # time in seconds before checking safety/etc while waiting. +max_transition_attempts: 5 # number of transitions attempts. +status_check_interval: 60 # periodic status check. state_machine: simple_state_table scheduler: - type: dispatch - fields_file: simple.yaml - check_file: False + type: dispatch + fields_file: simple.yaml + check_file: False mount: - brand: ioptron - model: 30 - driver: ioptron - serial: - port: /dev/ttyUSB0 - timeout: 0. - baudrate: 9600 - non_sidereal_available: True - min_tracking_threshold: 100 # ms - max_tracking_threshold: 99999 # ms + brand: ioptron + model: 30 + driver: ioptron + serial: + port: /dev/ttyUSB0 + timeout: 0. + baudrate: 9600 + non_sidereal_available: True + min_tracking_threshold: 100 # ms + max_tracking_threshold: 99999 # ms pointing: - auto_correct: True - threshold: 100 # arcseconds ~ 10 pixels - exptime: 30 # seconds - max_iterations: 5 + auto_correct: True + threshold: 100 # arcseconds ~ 10 pixels + exptime: 30 # seconds + max_iterations: 5 cameras: - auto_detect: True - primary: 14d3bd - devices: - - - model: canon_gphoto2 - - - model: canon_gphoto2 + auto_detect: True + primary: 14d3bd + devices: + - model: canon_gphoto2 + - model: canon_gphoto2 ######################### Environmental Sensors ################################ # Configure the environmental sensors that are attached. @@ -82,13 +85,13 @@ cameras: # serial_port: /dev/ttyACM1 ################################################################################ environment: - auto_detect: False - camera_board: - serial_port: /dev/ttyACM0 - control_board: - serial_port: /dev/ttyACM1 - weather: - url: http://localhost:5000/latest.json + auto_detect: False + camera_board: + serial_port: /dev/ttyACM0 + control_board: + serial_port: /dev/ttyACM1 + weather: + url: http://localhost:5000/latest.json ########################## Observations ######################################## @@ -107,8 +110,8 @@ environment: # TODO: Add options for cleaning up old data (e.g. >30 days) ################################################################################ observations: - make_timelapse: True - keep_jpgs: True + make_timelapse: True + keep_jpgs: True ######################## Google Network ######################################## # By default all images are stored on googlecloud servers and we also @@ -121,8 +124,19 @@ observations: # service_account_key: Location of the JSON service account key. ################################################################################ panoptes_network: - image_storage: False - service_account_key: # Location of JSON account key - project_id: panoptes-survey - buckets: - images: panoptes-survey + image_storage: False + service_account_key: # Location of JSON account key + project_id: panoptes-survey + buckets: + images: panoptes-survey + +############################### pocs ################################## +# POCS status flags. The values below represent initial values but +# they can be switched at run-time if needed. +####################################################################### +pocs: + INITIALIZED: false + INTERRUPTED: false + KEEP_RUNNING: true + DO_STATES: true + RUN_ONCE: false \ No newline at end of file diff --git a/conftest.py b/conftest.py index 7ebaf58ac..73df248e6 100644 --- a/conftest.py +++ b/conftest.py @@ -2,21 +2,17 @@ import os import stat import pytest -from _pytest.logging import caplog as _caplog -import subprocess import time import tempfile import shutil - from contextlib import suppress -from multiprocessing import Process -from scalpl import Cut +from _pytest.logging import caplog as _caplog from panoptes.pocs import hardware from panoptes.utils.database import PanDB -from panoptes.utils.config import load_config +from panoptes.utils.config.client import get_config from panoptes.utils.config.client import set_config -from panoptes.utils.config.server import app as config_server_app +from panoptes.utils.config.server import config_server from panoptes.pocs.utils.logger import get_logger, PanLogger @@ -33,13 +29,17 @@ os.getenv('PANLOG', '/var/panoptes/logs'), 'panoptes-testing.log' ) +startup_message = ' STARTING NEW PYTEST RUN ' logger.add(log_file_path, - format=LOGGER_INFO.format, - colorize=True, enqueue=True, # multiprocessing + colorize=True, backtrace=True, diagnose=True, + catch=True, + # Start new log file for each testing run. + rotation=lambda msg, _: startup_message in msg, level='TRACE') +logger.log('testing', '*' * 25 + startup_message + '*' * 25) # Make the log file world readable. os.chmod(log_file_path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) @@ -58,16 +58,12 @@ def pytest_addoption(parser): nargs='+', default=[], help=f"A comma separated list of hardware to NOT test. List items can include: {hw_names}") - group.addoption( - "--solve", - action="store_true", - default=False, - help="If tests that require solving should be run") group.addoption( "--test-databases", nargs="+", default=['file'], - help=f"Test databases in the list. List items can include: {db_names}. Note that travis-ci will test all of " + help=f"Test databases in the list. List items can include: {db_names}. Note that " + f"travis-ci will test all of " f"them by default.") @@ -96,11 +92,11 @@ def pytest_collection_modifyitems(config, items): # User does not want to run tests that interact with hardware called name, # whether it is marked as with_name or without_name. if name in with_hardware: - print('Warning: {!r} in both --with-hardware and --without-hardware'.format(name)) + print(f'Warning: {name} in both --with-hardware and --without-hardware') with_hardware.remove(name) - skip = pytest.mark.skip(reason="--without-hardware={} specified".format(name)) - with_keyword = 'with_' + name - without_keyword = 'without_' + name + skip = pytest.mark.skip(reason=f"--without-hardware={name} specified") + with_keyword = f'with_{name}' + without_keyword = f'without_{name}' for item in items: if with_keyword in item.keywords or without_keyword in item.keywords: item.add_marker(skip) @@ -108,7 +104,7 @@ def pytest_collection_modifyitems(config, items): for name in hardware.get_all_names(without=with_hardware): # We don't have hardware called name, so find all tests that need that # hardware and mark it to be skipped. - skip = pytest.mark.skip(reason="Test needs --with-hardware={} option to run".format(name)) + skip = pytest.mark.skip(reason=f"Test needs --with-hardware={name} option to run") keyword = 'with_' + name for item in items: if keyword in item.keywords: @@ -147,137 +143,48 @@ def pytest_runtest_logreport(report): """Adds the failure info that pytest prints to stdout into the log.""" if report.skipped or report.outcome != 'failed': return - try: + with suppress(Exception): logger.log('testing', '') - logger.log('testing', f' TEST {report.nodeid} FAILED during {report.when} {report.longreprtext} ') + logger.log('testing', + f' TEST {report.nodeid} FAILED during {report.when} {report.longreprtext} ') if report.capstdout: - logger.log('testing', f'============ Captured stdout during {report.when} {report.capstdout} ============') + logger.log('testing', + f'============ Captured stdout during {report.when} {report.capstdout} ' + f'============') if report.capstderr: - logger.log('testing', f'============ Captured stdout during {report.when} {report.capstderr} ============') - except Exception: - pass - - -@pytest.fixture(scope='session') -def db_name(): - return 'panoptes_testing' - - -@pytest.fixture(scope='session') -def images_dir(tmpdir_factory): - directory = tmpdir_factory.mktemp('images') - return str(directory) - - -@pytest.fixture(scope='session') -def config_host(): - return 'localhost' - - -@pytest.fixture(scope='session') -def static_config_port(): - """Used for the session-scoped config_server where no config values - are expected to change during testing. - """ - return '6563' - - -@pytest.fixture(scope='module') -def config_port(): - """Used for the function-scoped config_server when it is required to change - config values during testing. See `dynamic_config_server` docs below. - """ - return '4861' + logger.log('testing', + f'============ Captured stdout during {report.when} {report.capstderr} ' + f'============') @pytest.fixture(scope='session') def config_path(): - return os.path.join(os.getenv('POCS'), 'tests', 'pocs_testing.yaml') + return os.path.expandvars('${POCS}/tests/pocs_testing.yaml') -@pytest.fixture(scope='session') -def config_server_args(config_path): - loaded_config = load_config(config_files=config_path, ignore_local=True) - return { - 'config_file': config_path, - 'auto_save': False, - 'ignore_local': True, - 'POCS': loaded_config, - 'POCS_cut': Cut(loaded_config) - } - - -def make_config_server(config_host, config_port, config_server_args, images_dir, db_name): - def start_config_server(): - # Load the config items into the app config. - for k, v in config_server_args.items(): - config_server_app.config[k] = v - - # Start the actual flask server. - config_server_app.run(host=config_host, port=config_port) +@pytest.fixture(scope='module', autouse=True) +def static_config_server(config_path, images_dir, db_name): + logger.log('testing', f'Starting static_config_server for testing session') - proc = Process(target=start_config_server) - proc.start() + proc = config_server( + config_path, + ignore_local=True, + auto_save=False + ) - logger.log('testing', f'config_server started with PID={proc.pid}') + logger.log('testing', f'static_config_server started with {proc.pid=}') # Give server time to start - time.sleep(1) - - # Adjust various config items for testing - unit_name = 'Generic PANOPTES Unit' - unit_id = 'PAN000' - logger.log('testing', f'Setting testing name and unit_id to {unit_id}') - set_config('name', unit_name, port=config_port) - set_config('pan_id', unit_id, port=config_port) - - logger.log('testing', f'Setting testing database to {db_name}') - set_config('db.name', db_name, port=config_port) - - fields_file = 'simulator.yaml' - logger.log('testing', f'Setting testing scheduler fields_file to {fields_file}') - set_config('scheduler.fields_file', fields_file, port=config_port) - - # TODO(wtgee): determine if we need separate directories for each module. - logger.log('testing', f'Setting temporary image directory for testing') - set_config('directories.images', images_dir, port=config_port) - - # Make everything a simulator - simulators = hardware.get_simulator_names(simulator=['all']) - logger.log('testing', f'Setting all hardware to use simulators: {simulators}') - set_config('simulator', simulators, port=config_port) - - return proc - - -@pytest.fixture(scope='session', autouse=True) -def static_config_server(config_host, static_config_port, config_server_args, images_dir, db_name): - logger.log('testing', f'Starting config_server for testing session') - proc = make_config_server(config_host, static_config_port, config_server_args, images_dir, db_name) - yield proc - pid = proc.pid - proc.terminate() - time.sleep(0.1) - logger.log('testing', f'Killed config_server started with PID={pid}') - + while get_config('name') is None: # pragma: no cover + logger.log('testing', f'Waiting for static_config_server {proc.pid=}, sleeping 1 second.') + time.sleep(1) -@pytest.fixture(scope='function') -def dynamic_config_server(config_host, config_port, config_server_args, images_dir, db_name): - """If a test requires changing the configuration we use a function-scoped testing - server. We only do this on tests that require it so we are not constantly starting and stopping - the config server unless necessary. To use this, each test that requires it must use the - `dynamic_config_server` and `config_port` fixtures and must pass the `config_port` to all - instances that are created (propagated through PanBase). - """ + set_config('directories.images', images_dir) - logger.log('testing', f'Starting config_server for testing function') - proc = make_config_server(config_host, config_port, config_server_args, images_dir, db_name) - - yield proc - pid = proc.pid + logger.log('testing', f'Startup config_server name=[{get_config("name")}]') + yield + logger.log('testing', f'Killing static_config_server started with PID={proc.pid}') proc.terminate() - time.sleep(0.1) - logger.log('testing', f'Killed config_server started with PID={pid}') @pytest.fixture @@ -289,6 +196,17 @@ def temp_file(tmp_path): f.unlink(missing_ok=True) +@pytest.fixture(scope='session') +def db_name(): + return 'panoptes_testing' + + +@pytest.fixture(scope='session') +def images_dir(tmpdir_factory): + directory = tmpdir_factory.mktemp('images') + return str(directory) + + @pytest.fixture(scope='function', params=_all_databases) def db_type(request, db_name): db_list = request.config.option.test_databases @@ -312,7 +230,7 @@ def memory_db(db_name): @pytest.fixture(scope='session') def data_dir(): - return '/var/panoptes/panoptes-utils/tests/data' + return os.path.expandvars('${POCS}/tests/data') @pytest.fixture(scope='function') diff --git a/docker/README.rst b/docker/README.rst index 5ea19edf9..b92b7b10b 100644 --- a/docker/README.rst +++ b/docker/README.rst @@ -16,11 +16,11 @@ To build the images locally: docker/setup-local-environment.sh -To run the test suite locally: +Then, to run the test suite locally: .. code:: bash - scripts/testing/test-software.sh + panoptes-develop test This will build all three images locally and is suitable for testing and development. diff --git a/docker/develop.Dockerfile b/docker/develop.Dockerfile index 57b2e3bc1..7afa710e0 100644 --- a/docker/develop.Dockerfile +++ b/docker/develop.Dockerfile @@ -22,7 +22,7 @@ ENV SOLVE_FIELD /usr/bin/solve-field USER ${PANUSER} COPY --chown=panoptes:panoptes . "${PANDIR}/POCS/" RUN cd "${PANDIR}/POCS" && \ - pip install -e ".[testing,google]" + pip install -U -e ".[testing,google]" # Cleanup apt. USER root diff --git a/docker/docker-compose-developer-env.yaml b/docker/docker-compose-developer-env.yaml index 1416a4528..9abaa45a8 100644 --- a/docker/docker-compose-developer-env.yaml +++ b/docker/docker-compose-developer-env.yaml @@ -1,5 +1,19 @@ version: '3.7' services: + config-server: + image: "panoptes-utils:develop" + init: true + container_name: config-server + privileged: true + network_mode: host + env_file: $PANDIR/env + restart: on-failure + volumes: + - pandir:/var/panoptes + command: + - "panoptes-config-server" + - "run" + - "/var/panoptes/POCS/conf_files/pocs.yaml" develop-env: image: "${IMAGE:-panoptes-pocs}:${TAG:-developer-env}" init: true @@ -7,6 +21,8 @@ services: privileged: true network_mode: host env_file: $PANDIR/env + depends_on: + - "config-server" volumes: - pandir:/var/panoptes volumes: diff --git a/docker/docker-compose-testing.yaml b/docker/docker-compose-testing.yaml new file mode 100644 index 000000000..98cbbc998 --- /dev/null +++ b/docker/docker-compose-testing.yaml @@ -0,0 +1,25 @@ +version: '3.7' +services: + develop-env: + image: "${IMAGE:-panoptes-pocs}:${TAG:-develop}" + init: true + container_name: "${CONTAINER_NAME:-panoptes-pocs:develop}" + privileged: true + network_mode: host + env_file: $PANDIR/env + tty: true + environment: + - CLI_ARGS="${CLI_ARGS}" + command: + - "${POCS}/scripts/testing/run-tests.sh" + volumes: + - pandir:/var/panoptes + - /var/panoptes/logs:/var/panoptes/logs + - /var/panoptes/POCS:/var/panoptes/POCS +volumes: + pandir: + driver: local + driver_opts: + type: none + device: /var/panoptes + o: bind diff --git a/docker/latest.Dockerfile b/docker/latest.Dockerfile index 7416b4776..3ae5e31ee 100644 --- a/docker/latest.Dockerfile +++ b/docker/latest.Dockerfile @@ -27,7 +27,7 @@ RUN apt-get update \ # arduino-cli && curl -fsSL $arduino_url | BINDIR="/usr/local/bin" sh \ # Install the module. - && pip install "panoptes-pocs[google]" + && pip install -U "panoptes-pocs[google]" # Cleanup apt. RUN apt-get autoremove --purge -y \ diff --git a/docker/setup-local-environment.sh b/docker/setup-local-environment.sh index 385adf24b..f88e901b8 100755 --- a/docker/setup-local-environment.sh +++ b/docker/setup-local-environment.sh @@ -1,23 +1,33 @@ -#!/bin/bash -e +#!/usr/bin/env bash +set -e -POCS=${POCS:-/var/panoptes/POCS} -TAG="${1:-develop}" +export PANDIR=${PANDIR:-/var/panoptes} +export POCS=${POCS:-/var/panoptes/POCS} + +echo "Setting up local environment." + +echo "Removing stale docker images to make space" +docker system prune --force + +echo "Building local panoptes-utils" +. "${PANDIR}/panoptes-utils/docker/setup-local-environment.sh" cd "${POCS}" -echo "Building local panoptes-pocs:latest" +# In the local develop we need to pass git to the docker build context. +sed -i s'/^\.git$/\!\.git/' .dockerignore + +echo "Building local panoptes-pocs:latest from panoptes-utils:develop" docker build \ - --quiet \ + --quiet --force-rm \ + --build-arg IMAGE_URL="panoptes-utils:develop" \ -t "panoptes-pocs:latest" \ -f "${POCS}/docker/latest.Dockerfile" \ "${POCS}" -# In the local develop we need to pass git to the docker build context. -sed -i s'/^\.git$/\!\.git/' .dockerignore - echo "Building local panoptes-pocs:develop" docker build \ - --quiet \ + --quiet --force-rm \ --build-arg IMAGE_URL="panoptes-pocs:latest" \ -t "panoptes-pocs:develop" \ -f "${POCS}/docker/develop.Dockerfile" \ @@ -25,7 +35,7 @@ docker build \ echo "Building local panoptes-pocs:developer-env" docker build \ - --quiet \ + --quiet --force-rm \ --build-arg IMAGE_URL="panoptes-pocs:develop" \ -t "panoptes-pocs:developer-env" \ -f "${POCS}/docker/developer-env.Dockerfile" \ @@ -34,12 +44,17 @@ docker build \ # Revert our .dockerignore changes. sed -i s'/^!\.git$/\.git/' .dockerignore +docker system prune --force + cat <`__ for advice. @@ -23,16 +23,28 @@ PANOPTES Observatory Control System Overview -------- -`PANOPTES `__ is an open source citizen science project -that is designed to find transiting exoplanets with digital cameras. The goal of -PANOPTES is to establish a global network of of robotic cameras run by amateur -astronomers schools in order to monitor, as continuously as possible, a very large -number of stars. For more general information about the project, including the -science case and resources for interested individuals, see the `about page `__. +Project PANOPTES +^^^^^^^^^^^^^^^^ + +`PANOPTES `_ is an open source citizen science project +designed to find `transiting exoplanets `_ with +digital cameras. The goal of PANOPTES is to establish a global network of of robotic +cameras run by amateur astronomers and schools (or anyone!) in order to monitor, +as continuously as possible, a very large number of stars. For more general information +about the project, including the science case and resources for interested individuals, see the +`project overview `_. + +POCS +^^^^ + +POCS (PANOPTES Observatory Control System) is the main software driver for a +PANOPTES unit, responsible for high-level control of the unit. + +For more information, see the full documentation at: https://pocs.readthedocs.io. -POCS (PANOPTES Observatory Control System) is the main software driver for the -PANOPTES unit, responsible for high-level control of the unit. This repository -also contains a number of scripts for running a full instance of POCS. +`panoptes-utils `_ is a related repository and POCS +relies on most of the tools within `panoptes-utils`. See https://panoptes-pocs.readthedocs.io for +more information. Getting Started --------------- diff --git a/docs/index.rst b/docs/index.rst index fe339c135..f459c12b9 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2,7 +2,7 @@ POCS ==== -.. include:: ../README.rst +.. include:: ./README.rst Contents ======== diff --git a/scripts/install/install-pocs.sh b/scripts/install/install-pocs.sh index 5ad6a7755..a22c5a729 100644 --- a/scripts/install/install-pocs.sh +++ b/scripts/install/install-pocs.sh @@ -153,7 +153,10 @@ export POCS=${PANDIR}/POCS export PANLOG=${PANDIR}/logs EOF - [ ! -f /var/panoptes/env ] && echo '. /var/panoptes/env' >> ~/.bashrc + if [[ ! -f /var/panoptes/env ]]; then + [[ -f "$HOME/.bashrc" ]] && echo '. /var/panoptes/env' >> ~/.bashrc + [[ -f "$HOME/.zshrc" ]] && echo '. /var/panoptes/env' >> ~/.zshrc + fi } function system_deps { @@ -268,7 +271,6 @@ function do_install { echo "Setting up environment variables in ${ENV_FILE}" setup_env_vars - echo "Installing system dependencies" system_deps diff --git a/scripts/testing/run-tests.sh b/scripts/testing/run-tests.sh index dcbf09ee0..c8cb25ae3 100755 --- a/scripts/testing/run-tests.sh +++ b/scripts/testing/run-tests.sh @@ -1,20 +1,24 @@ -#!/bin/bash -e +#!/usr/bin/env bash +set -e -REPORT_FILE=${REPORT_FILE:-coverage.xml} +cd "${POCS}" +PYTEST_CMD="$(command -v pytest)" -export PYTHONPATH="${PYTHONPATH}:/var/panoptes/POCS/scripts/testing/coverage" -export COVERAGE_PROCESS_START="/var/panoptes/POCS/setup.cfg" +echo "Run test params: ${PYTEST_CMD} $@" -coverage erase +REPORT_FILE=${REPORT_FILE:-coverage.xml} + +# This assumes we are always running in a docker container. +export COVERAGE_PROCESS_START="/var/panoptes/panoptes-pocs/setup.cfg" -# Run coverage over the pytest suite +# Run coverage over the pytest suite. echo "Starting tests" -coverage run "$(command -v pytest)" -x -vv -rfes -echo "Combining coverage" -coverage combine +coverage erase +coverage run "${PYTEST_CMD}" -echo "Making XML coverage report at ${REPORT_FILE}" +coverage combine coverage xml -o "${REPORT_FILE}" +coverage report --show-missing exit 0 diff --git a/scripts/testing/test-software.sh b/scripts/testing/test-software.sh deleted file mode 100755 index 99131f077..000000000 --- a/scripts/testing/test-software.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash - -clear; - -cat <=0.2.17 + panoptes-utils>=0.2.20 pyserial>=3.1.1 PyYaml pendulum @@ -60,7 +59,7 @@ install_requires = # The usage of test_requires is discouraged, see `Dependency Management` docs # tests_require = pytest; pytest-cov # Require a specific Python version, e.g. Python 2.7 or >= 3.4 -python_requires = >=3.7 +python_requires = >=3.8 [options.packages.find] where = src @@ -110,8 +109,14 @@ testing = [test] # py.test options when running `python setup.py test` addopts = - --cov panoptes.pocs --cov panoptes.peas --cov-report term-missing + --cov panoptes.pocs + --cov panoptes.peas + --cov-branch + --cov-report term-missing:skip-covered + --no-cov-on-fail --doctest-modules + --doctest-ignore-import-errors + --failed-first -x --verbose extras = True @@ -153,12 +158,6 @@ universal = 1 source_dir = docs build_dir = build/sphinx -[devpi:upload] -# Options for the devpi: PyPI server and packaging tool -# VCS export must be deactivated since we are using setuptools-scm -no-vcs = 1 -formats = bdist_wheel - [flake8] # Some sane defaults for the code style checker flake8 exclude = diff --git a/scripts/coverage/sitecustomize.py b/sitecustomize.py similarity index 99% rename from scripts/coverage/sitecustomize.py rename to sitecustomize.py index 1dc959e54..6c70e25d1 100644 --- a/scripts/coverage/sitecustomize.py +++ b/sitecustomize.py @@ -1,5 +1,6 @@ # Ensure coverage starts for all Python processes so that test coverage is calculated # properly when using subprocesses (see https://coverage.readthedocs.io/en/latest/subprocess.html) import coverage + print("Starting coverage from sitecustomize") coverage.process_startup() diff --git a/src/panoptes/pocs/base.py b/src/panoptes/pocs/base.py index ac5f56d6d..b6e2308c3 100644 --- a/src/panoptes/pocs/base.py +++ b/src/panoptes/pocs/base.py @@ -4,6 +4,7 @@ from panoptes.utils.database import PanDB from panoptes.utils.config import client from panoptes.pocs.utils.logger import get_logger +from panoptes.pocs import hardware class PanBase(object): @@ -19,24 +20,11 @@ def __init__(self, config_port='6563', *args, **kwargs): self.logger = get_logger() - simulators = self.get_config('simulator', default=[]) - if simulators: - self.logger.warning(f'Using simulators: {simulators}') + # If the user requests a db_type then update runtime config + db_type = kwargs.get('db_type', self.get_config('db.type', default='file')) + db_name = kwargs.get('db_name', self.get_config('db.name', default='panoptes')) - # Get passed DB or set up new connection - _db = kwargs.get('db', None) - if _db is None: - # If the user requests a db_type then update runtime config - db_type = kwargs.get('db_type', self.get_config('db.type', default='file')) - db_name = kwargs.get('db_name', self.get_config('db.name', default='panoptes')) - - _db = PanDB(db_type=db_type, db_name=db_name) - - self.db = _db - - @property - def config_port(self): - return self._config_port + self.db = PanDB(db_type=db_type, db_name=db_name) def get_config(self, *args, **kwargs): """Thin-wrapper around client based get_config that sets default port. @@ -49,7 +37,7 @@ def get_config(self, *args, **kwargs): """ config_value = None try: - config_value = client.get_config(port=self.config_port, *args, **kwargs) + config_value = client.get_config(port=self._config_port, *args, **kwargs) except ConnectionError as e: # pragma: no cover self.logger.critical(f'Cannot connect to config_server from {self.__class__}: {e!r}') @@ -67,8 +55,16 @@ def set_config(self, key, new_value, *args, **kwargs): **kwargs: Passed to set_config """ config_value = None + + if key == 'simulator' and new_value == 'all': + # Don't use hardware.get_simulator_names because it checks config. + new_value = hardware.ALL_NAMES + try: - config_value = client.set_config(key, new_value, port=self.config_port, *args, **kwargs) + self.logger.trace(f'Setting config {key=} {new_value=}') + config_value = client.set_config(key, new_value, port=self._config_port, *args, + **kwargs) + self.logger.trace(f'Config set {config_value=}') except ConnectionError as e: # pragma: no cover self.logger.critical(f'Cannot connect to config_server from {self.__class__}: {e!r}') diff --git a/src/panoptes/pocs/camera/__init__.py b/src/panoptes/pocs/camera/__init__.py index 80b68e18e..90ecce299 100644 --- a/src/panoptes/pocs/camera/__init__.py +++ b/src/panoptes/pocs/camera/__init__.py @@ -5,14 +5,16 @@ import random from astropy import units as u -from panoptes.pocs.camera.camera import AbstractCamera # pragma: no flakes -from panoptes.pocs.camera.camera import AbstractGPhotoCamera # pragma: no flakes +from panoptes.pocs.camera.camera import AbstractCamera # noqa +from panoptes.pocs.camera.camera import AbstractGPhotoCamera # noqa from panoptes.pocs.utils.logger import get_logger from panoptes.utils import error from panoptes.utils.config.client import get_config from panoptes.utils.library import load_module +logger = get_logger() + def list_connected_cameras(): """Detect connected cameras. @@ -43,17 +45,17 @@ def list_connected_cameras(): return ports -def create_cameras_from_config(config_port='6563', **kwargs): +def create_cameras_from_config(*args, **kwargs): """Create camera object(s) based on the config. Creates a camera for each camera item listed in the config. Ensures the appropriate camera module is loaded. Args: - config_port (str, optional): config_server port, default '6563'. + *args (list): Passed to `get_config`. **kwargs (dict): Can pass a `cameras` object that overrides the info in the configuration file. Can also pass `auto_detect`(bool) to try and - automatically discover the ports. + automatically discover the ports. Any other items as passed to `get_config`. Returns: OrderedDict: An ordered dictionary of created camera objects, with the @@ -65,9 +67,8 @@ def create_cameras_from_config(config_port='6563', **kwargs): auto_detect=True and no cameras are found. error.PanError: Description """ - logger = get_logger() - config = get_config(port=config_port) + config = get_config(*args, **kwargs) # Helper method to first check kwargs then config def kwargs_or_config(item, default=None): @@ -141,7 +142,7 @@ def kwargs_or_config(item, default=None): module = load_module(f'panoptes.pocs.camera.{model}') logger.debug(f'Camera module: {module}') # Create the camera object - cam = module.Camera(config_port=config_port, **device_config) + cam = module.Camera(**device_config) except error.NotFound: logger.error(f"Cannot find camera module with config: {device_config}") except Exception as e: @@ -171,15 +172,11 @@ def kwargs_or_config(item, default=None): return cameras -def create_camera_simulator(num_cameras=2, config_port='6563', **kwargs): +def create_camera_simulator(num_cameras=2): """Create simulator camera object(s). Args: num_cameras (int): The number of simulated cameras to create, default 2. - config_port (int): The port to use to connect to the config server, default 6563. - **kwargs (dict): Can pass a `cameras` object that overrides the info in - the configuration file. Can also pass `auto_detect`(bool) to try and - automatically discover the ports. Returns: OrderedDict: An ordered dictionary of created camera objects, with the @@ -189,70 +186,55 @@ def create_camera_simulator(num_cameras=2, config_port='6563', **kwargs): Raises: error.CameraNotFound: Raised if camera cannot be found at specified port or if auto_detect=True and no cameras are found. - error.PanError: Description """ - logger = get_logger() + if num_cameras == 0: + raise error.CameraNotFound(msg="No cameras available") cameras = OrderedDict() - # Create a minimal dummy camera config to get a simulated camera - camera_info = {'autodetect': False, - 'devices': [ - {'model': 'simulator'}, ]} - - logger.debug(f"Camera config: {camera_info}") + # Set up a simulated camera with fully configured simulated focuser. + device_config = { + 'model': 'simulator', + 'port': '/dev/camera/simulator', + 'focuser': {'model': 'simulator', + 'focus_port': '/dev/ttyFAKE', + 'initial_position': 20000, + 'autofocus_range': (40, 80), + 'autofocus_step': (10, 20), + 'autofocus_seconds': 0.1, + 'autofocus_size': 500}, + 'filterwheel': {'model': 'simulator', + 'filter_names': ['one', 'deux', 'drei', 'quattro'], + 'move_time': 0.1 * u.second, + 'timeout': 0.5 * u.second}, + 'readout_time': 0.5, + } + logger.debug(f"SimulatorCamera config: {device_config=}") primary_camera = None - for cam_num in range(num_cameras): cam_name = f'SimCam{cam_num:02d}' logger.debug(f'Using camera simulator {cam_name}') - # Set up a simulated camera with fully configured simulated focuser - device_config = { - 'model': 'simulator', - 'port': '/dev/camera/simulator', - 'focuser': {'model': 'simulator', - 'focus_port': '/dev/ttyFAKE', - 'initial_position': 20000, - 'autofocus_range': (40, 80), - 'autofocus_step': (10, 20), - 'autofocus_seconds': 0.1, - 'autofocus_size': 500}, - 'filterwheel': {'model': 'simulator', - 'filter_names': ['one', 'deux', 'drei', 'quattro'], - 'move_time': 0.1 * u.second, - 'timeout': 0.5 * u.second}, - 'readout_time': 0.5, - # Simulator config should always ignore local settings. - 'ignore_local_config': True - } camera_model = device_config['model'] logger.debug(f'Creating camera: {camera_model}') - try: - module = load_module(f'panoptes.pocs.camera.{camera_model}') - logger.debug(f'Camera module: {module}') - # Create the camera object - cam = module.Camera(name=cam_name, config_port=config_port, **device_config) - except error.NotFound: # pragma: no cover - logger.error(f"Cannot find camera module: {camera_model}") - except Exception as e: # pragma: no cover - logger.error(f"Cannot create camera type: {camera_model} {e!r}") - else: - is_primary = '' - if cam_num == 0: - cam.is_primary = True - primary_camera = cam - is_primary = ' [Primary]' + module = load_module(f'panoptes.pocs.camera.{camera_model}') + logger.debug(f'Camera module: {module}') - logger.debug(f"Camera created: {cam.name} {cam.uid}{is_primary}") + # Create the camera object + cam = module.Camera(name=cam_name, **device_config) - cameras[cam_name] = cam + is_primary = '' + if cam_num == 0: + cam.is_primary = True + primary_camera = cam + is_primary = ' [Primary]' - if len(cameras) == 0: - raise error.CameraNotFound(msg="No cameras available") + logger.debug(f"Camera created: {cam.name} {cam.uid}{is_primary}") + + cameras[cam_name] = cam logger.debug(f"Primary camera: {primary_camera}") logger.debug(f"{len(cameras)} cameras created") diff --git a/src/panoptes/pocs/camera/camera.py b/src/panoptes/pocs/camera/camera.py index 09e3e86a1..b802fac04 100644 --- a/src/panoptes/pocs/camera/camera.py +++ b/src/panoptes/pocs/camera/camera.py @@ -25,7 +25,7 @@ from panoptes.utils.serializers import from_yaml -def parse_config(lines): +def parse_config(lines): # pragma: no cover yaml_string = '' for line in lines: IsID = len(line.split('/')) > 1 @@ -69,8 +69,10 @@ class AbstractCamera(PanBase, metaclass=ABCMeta): Attributes: filter_type (str): Type of filter attached to camera, default RGGB. - focuser (`panoptes.pocs.focuser.AbstractFocuser`|None): Focuser for the camera, default None. - filter_wheel (`panoptes.pocs.filterwheel.AbstractFilterWheel`|None): Filter wheel for the camera, + focuser (`panoptes.pocs.focuser.AbstractFocuser`|None): Focuser for the camera, + default None. + filter_wheel (`panoptes.pocs.filterwheel.AbstractFilterWheel`|None): Filter wheel for the + camera, default None. is_primary (bool): If this camera is the primary camera for the system, default False. model (str): The model of camera, such as 'gphoto2', 'sbig', etc. Default 'simulator'. @@ -94,7 +96,8 @@ class AbstractCamera(PanBase, metaclass=ABCMeta): For these cameras serial_number should be passed to the constructor instead. For SBIG and FLI this should simply be the serial number engraved on the camera case, whereas for ZWO cameras this should be the 8 character ID string previously saved to the camera - firmware. This can be done using ASICAP, or `panoptes.pocs.camera.libasi.ASIDriver.set_ID()`. + firmware. This can be done using ASICAP, + or `panoptes.pocs.camera.libasi.ASIDriver.set_ID()`. """ _subcomponent_classes = {'Focuser', 'FilterWheel'} @@ -634,7 +637,8 @@ def get_thumbnail(self, seconds, file_path, thumbnail_size, keep_file=False, *ar return thumbnail @abstractmethod - def _start_exposure(self, seconds=None, filename=None, dark=False, header=None, *args, **kwargs): + def _start_exposure(self, seconds=None, filename=None, dark=False, header=None, *args, + **kwargs): """Responsible for the camera-specific process that start an exposure. This method is called from the `take_exposure` method and is used to handle @@ -734,7 +738,9 @@ def _setup_observation(self, observation, headers, filename, **kwargs): if observation.filter_name is not None: try: # Move the filterwheel - self.logger.debug(f'Moving filterwheel={self.filterwheel} to filter_name={observation.filter_name}') + self.logger.debug( + f'Moving filterwheel={self.filterwheel} to filter_name=' + f'{observation.filter_name}') self.filterwheel.move_to(observation.filter_name, blocking=True) except Exception as e: self.logger.error(f'Error moving filterwheel on {self} to' @@ -743,7 +749,8 @@ def _setup_observation(self, observation, headers, filename, **kwargs): else: self.logger.info(f'Filter {observation.filter_name} requested by' - f' observation but {self.filterwheel} is missing that filter, using' + f' observation but {self.filterwheel} is missing that filter, ' + f'using' f' {self.filter_type}.') if headers is None: @@ -756,7 +763,8 @@ def _setup_observation(self, observation, headers, filename, **kwargs): observation.seq_time = start_time # Get the filename - self.logger.debug(f'Setting image_dir={observation.directory}/{self.uid}/{observation.seq_time}') + self.logger.debug( + f'Setting image_dir={observation.directory}/{self.uid}/{observation.seq_time}') image_dir = os.path.join( observation.directory, self.uid, @@ -813,7 +821,8 @@ def _setup_observation(self, observation, headers, filename, **kwargs): metadata.update(headers) self.logger.debug( - f'Observation setup: exptime={exptime} file_path={file_path} image_id={image_id} metadata={metadata}') + f'Observation setup: exptime={exptime} file_path={file_path} image_id={image_id} ' + f'metadata={metadata}') return exptime, file_path, image_id, metadata def _process_fits(self, file_path, info): @@ -845,7 +854,8 @@ def _create_subcomponent(self, subcomponent, class_name): try: base_module = load_module(base_module_name) except error.NotFound as err: - self.logger.critical(f"Couldn't import {class_name} base class module {base_module_name}!") + self.logger.critical( + f"Couldn't import {class_name} base class module {base_module_name}!") raise err base_class = getattr(base_module, f"Abstract{class_name}") @@ -861,7 +871,7 @@ def _create_subcomponent(self, subcomponent, class_name): self.logger.critical(f"Couldn't import {class_name} module {module_name}!") raise err subcomponent_kwargs = copy.deepcopy(subcomponent) - subcomponent_kwargs.update({'camera': self, 'config_port': self._config_port}) + subcomponent_kwargs.update({'camera': self}) setattr(self, class_name_lower, getattr(module, class_name)(**subcomponent_kwargs)) diff --git a/src/panoptes/pocs/core.py b/src/panoptes/pocs/core.py index f00d3f26d..a6bd7a4da 100644 --- a/src/panoptes/pocs/core.py +++ b/src/panoptes/pocs/core.py @@ -31,7 +31,7 @@ class POCS(PanStateMachine, PanBase): class. POCS will call the `initialize` method of the observatory. state_machine_file(str): Filename of the state machine to use, defaults to 'simple_state_table'. - simulator(list): A list of the different modules that can run in simulator mode. Possible + simulators(list): A list of the different modules that can run in simulator mode. Possible modules include: all, mount, camera, weather, night. Defaults to an empty list. Attributes: @@ -44,11 +44,16 @@ def __init__( self, observatory, state_machine_file=None, + simulators=None, *args, **kwargs): - # Explicitly call the base classes in the order we want + # Explicitly call the base classes in the order we want. PanBase.__init__(self, *args, **kwargs) + if simulators: + self.logger.warning(f'Using {simulators=}') + self.set_config('simulator', simulators) + assert isinstance(observatory, Observatory) self.name = self.get_config('name', default='Generic PANOPTES Unit') @@ -61,26 +66,28 @@ def __init__( self.logger.info(f'Making a POCS state machine from {state_machine_file}') PanStateMachine.__init__(self, state_machine_file, **kwargs) - # Add observatory object, which does the bulk of the work + # Add observatory object, which does the bulk of the work. self.observatory = observatory self.is_initialized = False self._free_space = None + self.run_once = kwargs.get('run_once', False) self._obs_run_retries = self.get_config('pocs.RETRY_ATTEMPTS', default=3) + self.connected = True + self.interrupted = False # We want to call and record the status on a periodic interval. - def get_status(): - while True: + def get_periodic_status(): + while self.connected: status = self.status - self.logger.debug(f'Periodic status call: {status!r}') + self.logger.trace(f'Periodic status call: {status!r}') self.db.insert_current('status', status) CountdownTimer(self.get_config('status_check_interval', default=60)).sleep() - self._status_thread = Thread(target=get_status, daemon=True) + self._status_thread = Thread(target=get_periodic_status, daemon=True) self._status_thread.start() - self.connected = True self.say("Hi there!") @property @@ -116,14 +123,6 @@ def connected(self): def connected(self, new_value): self.set_config('pocs.CONNECTED', new_value) - @property - def keep_running(self): - return self.get_config('pocs.KEEP_RUNNING', default=True) - - @keep_running.setter - def keep_running(self, new_value): - self.set_config('pocs.KEEP_RUNNING', new_value) - @property def do_states(self): return self.get_config('pocs.DO_STATES', default=True) @@ -132,8 +131,30 @@ def do_states(self): def do_states(self, new_value): self.set_config('pocs.DO_STATES', new_value) + @property + def keep_running(self): + """If POCS should keep running. + + Currently reads: + + * `connected` + * `do_states` + * `observatory.can_observe` + + Returns: + bool: If POCS should keep running. + """ + return self.connected and self.do_states and self.observatory.can_observe + @property def run_once(self): + """If POCS should exit the run loop after a single iteration. + + This value reads the `pocs.RUN_ONCE` config value. + + Returns: + bool: if machine should stop after single iteration, default False. + """ return self.get_config('pocs.RUN_ONCE', default=False) @run_once.setter @@ -172,6 +193,10 @@ def initialize(self): bool: True if all initialization succeeded, False otherwise. """ + if not self.observatory.can_observe: + self.say("Looks like we're missing some required hardware.") + return False + if not self.is_initialized: self.logger.info('*' * 80) self.say("Initializing the system! Woohoo!") @@ -186,6 +211,7 @@ def initialize(self): self.power_down() else: self.is_initialized = True + self.do_states = True return self.is_initialized @@ -236,14 +262,12 @@ def power_down(self): # Observatory shut down self.observatory.power_down() - self.keep_running = True - self.do_states = True - self.is_initialized = False - self.interrupted = False self.connected = False + self._status_thread.join(1) + # Clear all the config items. - self.logger.info("Power down complete") + self.logger.success("Power down complete") def reset_observing_run(self): """Reset an observing run loop. """ @@ -316,6 +340,15 @@ def is_safe(self, no_warning=False, horizon='observe'): return safe + def _in_simulator(self, key): + """Checks the config server for the given simulator key value.""" + with suppress(KeyError): + if key in self.get_config('simulator', default=list()): + self.logger.debug(f'Using {key} simulator') + return True + + return False + def is_dark(self, horizon='observe'): """Is it dark @@ -323,7 +356,7 @@ def is_dark(self, horizon='observe'): entry `location.flat_horizon` by default. Args: - horizon (str, optional): Which horizon to use, 'flat''focus', or + horizon (str, optional): Which horizon to use, 'flat', 'focus', or 'observe' (default). Returns: @@ -334,11 +367,8 @@ def is_dark(self, horizon='observe'): is_dark = self.observatory.is_dark(horizon=horizon) self.logger.debug(f'Observatory is_dark: {is_dark}') - # Check simulator - with suppress(KeyError): - if 'night' in self.get_config('simulator', default=[]): - self.logger.debug(f'Using night simulator') - is_dark = True + if self._in_simulator('night'): + return True self.logger.debug(f"Dark Check: {is_dark}") return is_dark @@ -356,18 +386,12 @@ def is_weather_safe(self, stale=180): # Always assume False self.logger.debug("Checking weather safety") - is_safe = False - # Check if we are using weather simulator - simulator_values = self.get_config('simulator', default=[]) - if len(simulator_values): - self.logger.debug(f'simulator_values: {simulator_values}') - - if 'weather' in simulator_values: - self.logger.info("Weather simulator always safe") + if self._in_simulator('weather'): return True # Get current weather readings from database + is_safe = False try: record = self.db.get_current('weather') if record is None: @@ -378,12 +402,11 @@ def is_weather_safe(self, stale=180): timestamp = record['date'].replace(tzinfo=None) # current_time is timezone naive age = (current_time().datetime - timestamp).total_seconds() - self.logger.debug(f"Weather Safety: {is_safe} [{age:.0f} sec old - {timestamp:%Y-%m-%d %H:%M:%S}]") + self.logger.debug( + f"Weather Safety: {is_safe} [{age:.0f} sec old - {timestamp:%Y-%m-%d %H:%M:%S}]") - except (TypeError, KeyError) as e: - self.logger.warning(f"No record found in DB: {e!r}") except Exception as e: # pragma: no cover - self.logger.error(f"Error checking weather: {e!r}") + self.logger.error(f"No weather record in database: {e!r}") else: if age >= stale: self.logger.warning("Weather record looks stale, marking unsafe.") @@ -413,9 +436,9 @@ def has_free_space(self, required_space=0.25 * u.gigabyte, low_space_percent=1.5 has_space = bool(self._free_space.value >= req_space.value) if not has_space: - self.logger.error(f'No disk space: Free {self._free_space:.02f}\tReq: {req_space:.02f}') - elif space_is_low: - self.logger.warning(f'Low disk space: Free {self._free_space:.02f}\tReq: {req_space:.02f}') + self.logger.error(f'No disk space: Free {self._free_space:.02f}\t {req_space=:.02f}') + elif space_is_low: # pragma: no cover + self.logger.warning(f'Low disk space: Free {self._free_space:.02f}\t {req_space=:.02f}') return has_space @@ -438,11 +461,7 @@ def has_ac_power(self, stale=90): self.logger.debug("Checking for AC power") has_power = False - # TODO(wtgee): figure out if we really want to simulate no power - # Check if we are using power simulator - simulator_values = self.get_config('simulator', default=[]) - if 'power' in simulator_values: - self.logger.debug("AC power simulator always safe") + if self._in_simulator('power'): return True # Get current power readings from database @@ -459,7 +478,8 @@ def has_ac_power(self, stale=90): timestamp = record['date'].replace(tzinfo=None) # current_time is timezone naive age = (current_time().datetime - timestamp).total_seconds() - self.logger.debug(f"Power Safety: {has_power} [{age:.0f} sec old - {timestamp:%Y-%m-%d %H:%M:%S}]") + self.logger.debug( + f"Power Safety: {has_power} [{age:.0f} sec old - {timestamp:%Y-%m-%d %H:%M:%S}]") except (TypeError, KeyError) as e: self.logger.warning(f"No record found in DB: {e!r}") @@ -489,7 +509,7 @@ def wait(self, delay=None): delay {float|None} -- Number of seconds to wait. If default `None`, look up value in config, otherwise 2.5 seconds. """ - if delay is None: + if delay is None: # pragma: no cover delay = self.get_config('wait_delay', default=2.5) sleep_timer = CountdownTimer(delay) @@ -501,36 +521,3 @@ def wait(self, delay=None): is_expired = sleep_timer.expired() self.logger.debug(f'Leaving wait timer: expired={is_expired}') return is_expired - - ################################################################################################## - # Class Methods - ################################################################################################## - - @classmethod - def check_environment(cls): - """ Checks to see if environment is set up correctly - - There are a number of environmental variables that are expected - to be set in order for PANOPTES to work correctly. This method just - sanity checks our environment and shuts down otherwise. - - PANDIR Base directory for PANOPTES - POCS Base directory for POCS - """ - if sys.version_info[:2] < (3, 0): # pragma: no cover - warnings.warn("POCS requires Python 3.x to run") - - pandir = os.getenv('PANDIR') - if not os.path.exists(pandir): - sys.exit(f"$PANDIR dir does not exist or is empty: {pandir}") - - pocs = os.getenv('POCS') - if pocs is None: # pragma: no cover - sys.exit('Please make sure $POCS environment variable is set') - - if not os.path.exists(pocs): - sys.exit(f"$POCS directory does not exist or is empty: {pocs}") - - if not os.path.exists(f"{pandir}/logs"): - print(f"Creating log dir at {pandir}/logs") - os.makedirs(f"{pandir}/logs") diff --git a/src/panoptes/pocs/dome/__init__.py b/src/panoptes/pocs/dome/__init__.py index e04b3a551..faf846257 100644 --- a/src/panoptes/pocs/dome/__init__.py +++ b/src/panoptes/pocs/dome/__init__.py @@ -8,7 +8,7 @@ logger = get_logger() -def create_dome_from_config(config_port='6563', *args, **kwargs): +def create_dome_from_config(*args, **kwargs): """If there is a dome specified in the config, create a driver for it. A dome needs a config. We assume that there is at most one dome in the config, i.e. we don't @@ -17,7 +17,7 @@ def create_dome_from_config(config_port='6563', *args, **kwargs): by a single dome driver class. """ - dome_config = get_config('dome', port=config_port) + dome_config = get_config('dome') if dome_config is None: logger.info('No dome in config.') @@ -26,25 +26,25 @@ def create_dome_from_config(config_port='6563', *args, **kwargs): brand = dome_config['brand'] driver = dome_config['driver'] - logger.debug('Creating dome: brand={}, driver={}'.format(brand, driver)) + logger.debug(f'Creating dome: {brand=}, {driver=}') module = load_module(f'panoptes.pocs.dome.{driver}') - dome = module.Dome(config_port=config_port, *args, **kwargs) + dome = module.Dome(*args, **kwargs) logger.info(f'Created dome driver: brand={brand}, driver={driver}') return dome -def create_dome_simulator(config_port=6563, *args, **kwargs): - dome_config = get_config('dome', port=config_port) +def create_dome_simulator(*args, **kwargs): + dome_config = get_config('dome') brand = dome_config['brand'] driver = dome_config['driver'] - logger.debug('Creating dome simulator: brand={}, driver={}'.format(brand, driver)) + logger.debug(f'Creating dome simulator: {brand=}, {driver=}') module = load_module(f'panoptes.pocs.dome.{driver}') - dome = module.Dome(config_port=config_port, *args, **kwargs) - logger.info('Created dome driver: brand={}, driver={}'.format(brand, driver)) + dome = module.Dome(*args, **kwargs) + logger.info(f'Created dome driver: {brand=}, {driver=}') return dome diff --git a/src/panoptes/pocs/hardware.py b/src/panoptes/pocs/hardware.py index e0fbd723c..e6be88023 100644 --- a/src/panoptes/pocs/hardware.py +++ b/src/panoptes/pocs/hardware.py @@ -1,4 +1,5 @@ """Information about hardware supported by Panoptes.""" +from panoptes.utils.config.client import get_config ALL_NAMES = sorted([ 'camera', @@ -12,20 +13,38 @@ ]) -def get_all_names(all_names=ALL_NAMES, without=list()): +def get_all_names(all_names=ALL_NAMES, without=None): """Returns the names of all the categories of hardware that POCS supports. Note that this doesn't extend to the Arduinos for the telemetry and camera boards, for which no simulation is supported at this time. + + >>> from panoptes.pocs.hardware import get_all_names + >>> get_all_names() + ['camera', 'dome', 'mount', 'night', 'power', 'sensors', 'theskyx', 'weather'] + >>> get_all_names(without='mount') # Single item + ['camera', 'dome', 'night', 'power', 'sensors', 'theskyx', 'weather'] + >>> get_all_names(without=['mount', 'power']) # List + ['camera', 'dome', 'night', 'sensors', 'theskyx', 'weather'] + + >>> # You can alter available hardware if needed. + >>> get_all_names(['foo', 'bar', 'power'], without=['power']) + ['bar', 'foo'] + + Args: + all_names (list): The list of hardware. + without (iterable): Return all items expect those in the list. + + Returns: + list: The sorted list of available hardware except those listed in `without`. """ # Make sure that 'all' gets expanded. - if without: - without = get_simulator_names(simulator=without) + without = get_simulator_names(simulator=without) - return [v for v in all_names if v not in without] + return sorted([v for v in all_names if v not in without]) -def get_simulator_names(simulator=None, kwargs=None, config=None): +def get_simulator_names(simulator=None, kwargs=None): """Returns the names of the simulators to be used in lieu of hardware drivers. Note that returning a list containing 'X' doesn't mean that the config calls for a driver @@ -46,11 +65,18 @@ def get_simulator_names(simulator=None, kwargs=None, config=None): and once in the kwargs (which won't be examined). Python doesn't permit a keyword argument to be passed in twice. + >>> from panoptes.pocs.hardware import get_simulator_names + >>> get_simulator_names() + [] + >>> get_simulator_names('all') + ['camera', 'dome', 'mount', 'night', 'power', 'sensors', 'theskyx', 'weather'] + + Args: - simulator: An explicit list of names of hardware to be simulated (i.e. hardware drivers - to be replaced with simulators). - kwargs: The kwargs passed in to the caller, which is inspected for an arg called 'simulator'. - config: Dictionary created from panoptes.pocs.yaml or similar. + simulator (list): An explicit list of names of hardware to be simulated + (i.e. hardware drivers to be replaced with simulators). + kwargs: The kwargs passed in to the caller, which is inspected for an arg + called 'simulator'. Returns: List of names of the hardware to be simulated. @@ -60,7 +86,7 @@ def get_simulator_names(simulator=None, kwargs=None, config=None): def extract_simulator(d): return (d or empty).get('simulator') - for v in [simulator, extract_simulator(kwargs), extract_simulator(config)]: + for v in [simulator, extract_simulator(kwargs), extract_simulator(get_config())]: if not v: continue if isinstance(v, str): diff --git a/src/panoptes/pocs/mount/__init__.py b/src/panoptes/pocs/mount/__init__.py index 968b17fa9..54607b40c 100644 --- a/src/panoptes/pocs/mount/__init__.py +++ b/src/panoptes/pocs/mount/__init__.py @@ -12,8 +12,7 @@ logger = get_logger() -def create_mount_from_config(config_port='6563', - mount_info=None, +def create_mount_from_config(mount_info=None, earth_location=None, *args, **kwargs): """Create a mount instance based on the provided config. @@ -23,7 +22,6 @@ def create_mount_from_config(config_port='6563', and the class must be called Mount. Args: - config_port: The port number of the config server, default 6563. mount_info: Optional param which overrides the 'mount' entry in config if provided. Useful for testing. earth_location: `astropy.coordinates.EarthLocation` instance, representing the @@ -45,7 +43,7 @@ def create_mount_from_config(config_port='6563', # If mount_info was not passed as a parameter, check config. if mount_info is None: logger.debug('No mount info provided, using values from config.') - mount_info = get_config('mount', default=None, port=config_port) + mount_info = get_config('mount', default=None) # If nothing in config, raise exception. if mount_info is None: @@ -56,7 +54,7 @@ def create_mount_from_config(config_port='6563', logger.debug('No location provided, using values from config.') # Get details from config. - site_details = create_location_from_config(config_port=config_port) + site_details = create_location_from_config() earth_location = site_details['earth_location'] driver = mount_info.get('driver') @@ -67,13 +65,13 @@ def create_mount_from_config(config_port='6563', logger.debug(f'Mount: driver={driver} model={model}') # Check if we should be using a simulator - use_simulator = 'mount' in get_config('simulator', default=[], port=config_port) + use_simulator = 'mount' in get_config('simulator', default=[]) logger.debug(f'Mount is simulator: {use_simulator}') # Create simulator if requested if use_simulator or (driver == 'simulator'): logger.debug(f'Creating mount simulator') - return create_mount_simulator() + return create_mount_simulator(mount_info=mount_info, earth_location=earth_location) # See if we have a serial connection try: @@ -97,21 +95,23 @@ def create_mount_from_config(config_port='6563', raise error.MountNotFound(e) # Make the mount include site information - mount = module.Mount(config_port=config_port, location=earth_location, *args, **kwargs) + mount = module.Mount(location=earth_location, *args, **kwargs) logger.success(f'{driver} mount created') return mount -def create_mount_simulator(config_port='6563', *args, **kwargs): +def create_mount_simulator(mount_info=None, + earth_location=None, + *args, **kwargs): # Remove mount simulator - current_simulators = get_config('simulator', default=[], port=config_port) + current_simulators = get_config('simulator', default=[]) logger.warning(f'Current simulators: {current_simulators}') with suppress(ValueError): current_simulators.remove('mount') - mount_config = { + mount_config = mount_info or { 'model': 'simulator', 'driver': 'simulator', 'serial': { @@ -120,9 +120,9 @@ def create_mount_simulator(config_port='6563', *args, **kwargs): } # Set mount device info to simulator - set_config('mount', mount_config, port=config_port) + set_config('mount', mount_config) - earth_location = create_location_from_config(config_port=config_port)['earth_location'] + earth_location = earth_location or create_location_from_config()['earth_location'] logger.debug(f"Loading mount driver: pocs.mount.{mount_config['driver']}") try: @@ -130,7 +130,7 @@ def create_mount_simulator(config_port='6563', *args, **kwargs): except error.NotFound as e: raise error.MountNotFound(f'Error loading mount module: {e!r}') - mount = module.Mount(location=earth_location, config_port=config_port, *args, **kwargs) + mount = module.Mount(earth_location, *args, **kwargs) logger.success(f"{mount_config['driver']} mount created") diff --git a/src/panoptes/pocs/mount/mount.py b/src/panoptes/pocs/mount/mount.py index 742400a24..437d6014c 100644 --- a/src/panoptes/pocs/mount/mount.py +++ b/src/panoptes/pocs/mount/mount.py @@ -153,7 +153,7 @@ def is_connected(self): @property def is_initialized(self): - """ bool: Has mount been initialied with connection """ + """ bool: Has mount been initialised with connection """ return self._is_initialized @property diff --git a/src/panoptes/pocs/mount/simulator.py b/src/panoptes/pocs/mount/simulator.py index cc4ec2f7d..16e05f303 100644 --- a/src/panoptes/pocs/mount/simulator.py +++ b/src/panoptes/pocs/mount/simulator.py @@ -9,7 +9,6 @@ class Mount(AbstractMount): - """Mount class for a simulator. Use this when you don't actually have a mount attached. """ @@ -26,14 +25,13 @@ def __init__(self, location, commands=dict(), *args, **kwargs): self.logger.debug('Simulator mount created') + ################################################################################################## + # Properties + ################################################################################################## -################################################################################################## -# Properties -################################################################################################## - -################################################################################################## -# Public Methods -################################################################################################## + ################################################################################################## + # Public Methods + ################################################################################################## def initialize(self, unpark=False, *arg, **kwargs): """ Initialize the connection with the mount and setup for location. @@ -61,7 +59,7 @@ def initialize(self, unpark=False, *arg, **kwargs): if unpark: self.unpark() - return self.is_initialized + return self._is_initialized def connect(self): self.logger.debug("Connecting to mount simulator") @@ -196,16 +194,14 @@ def set_tracking_rate(self, direction='ra', delta=0.0): self.tracking_rate = 1.0 + delta self.logger.debug("Custom tracking rate sent") - -################################################################################################## -# Private Methods -################################################################################################## + ################################################################################################## + # Private Methods + ################################################################################################## def _setup_location_for_mount(self): """Sets the mount up to the current location. Mount must be initialized first. """ assert self.is_initialized, self.logger.warning('Mount has not been initialized') - assert self.location is not None, self.logger.warning( - 'Please set a location before attempting setup') + assert self.location is not None, self.logger.warning('Please set a location before attempting setup') self.logger.debug('Setting up mount for location') diff --git a/src/panoptes/pocs/observatory.py b/src/panoptes/pocs/observatory.py index b18ddb202..157cbff62 100644 --- a/src/panoptes/pocs/observatory.py +++ b/src/panoptes/pocs/observatory.py @@ -28,11 +28,14 @@ def __init__(self, cameras=None, scheduler=None, dome=None, mount=None, *args, * dates and weather station. Adds cameras, scheduler, dome and mount. """ super().__init__(*args, **kwargs) + self.scheduler = None + self.dome = None + self.mount = None self.logger.info('Initializing observatory') # Setup information about site location self.logger.info('Setting up location') - site_details = create_location_from_config(config_port=self.config_port) + site_details = create_location_from_config() self.location = site_details['location'] self.earth_location = site_details['earth_location'] self.observer = site_details['observer'] @@ -55,7 +58,8 @@ def __init__(self, cameras=None, scheduler=None, dome=None, mount=None, *args, * for cam_name, camera in cameras.items(): self.add_camera(cam_name, camera) - # TODO(jamessynge): Figure out serial port validation behavior here compared to that for the mount. + # TODO(jamessynge): Figure out serial port validation behavior here compared to that for + # the mount. self.set_dome(dome) self.set_scheduler(scheduler) @@ -172,7 +176,7 @@ def can_observe(self): if can_observe is False: for check_name, is_true in checks.items(): if not is_true: - self.logger.warning(f'{check_name.title()} not present, cannot observe') + self.logger.warning(f'{check_name.title()} not present') return can_observe @@ -190,7 +194,8 @@ def add_camera(self, cam_name, camera): assert isinstance(camera, AbstractCamera) self.logger.debug(f'Adding {cam_name}: {camera}') if cam_name in self.cameras: - self.logger.debug(f'{cam_name} already exists, replacing existing camera under that name.') + self.logger.debug( + f'{cam_name} already exists, replacing existing camera under that name.') self.cameras[cam_name] = camera if camera.is_primary: @@ -216,42 +221,35 @@ def set_scheduler(self, scheduler): Args: scheduler (`pocs.scheduler.BaseScheduler`): An instance of the `~BaseScheduler` class. """ - if isinstance(scheduler, BaseScheduler): - self.logger.info('Adding scheduler.') - self.scheduler = scheduler - elif scheduler is None: - self.logger.info('Removing scheduler.') - self.scheduler = None - else: - raise TypeError("Scheduler is not instance of BaseScheduler class, cannot add.") + self._set_hardware(scheduler, 'scheduler', BaseScheduler) def set_dome(self, dome): """Set's dome or remove the dome for the `Observatory`. Args: dome (`pocs.dome.AbstractDome`): An instance of the `~AbstractDome` class. """ - if isinstance(dome, AbstractDome): - self.logger.info('Adding dome.') - self.dome = dome - elif dome is None: - self.logger.info('Removing dome.') - self.dome = None - else: - raise TypeError('Dome is not instance of AbstractDome class, cannot add.') + self._set_hardware(dome, 'dome', AbstractDome) def set_mount(self, mount): """Sets the mount for the `Observatory`. Args: mount (`pocs.mount.AbstractMount`): An instance of the `~AbstractMount` class. """ - if isinstance(mount, AbstractMount): - self.logger.info('Adding mount') - self.mount = mount - elif mount is None: - self.logger.info('Removing mount') - self.mount = None + self._set_hardware(mount, 'mount', AbstractMount) + + def _set_hardware(self, new_hardware, hw_type, hw_class): + # Lookup the set method for the hardware type. + hw_attr = getattr(self, hw_type) + + if isinstance(new_hardware, hw_class): + self.logger.success(f'Adding {new_hardware=}') + setattr(self, hw_type, new_hardware) + elif new_hardware is None: + if hw_attr is not None: + self.logger.success(f'Removing {hw_attr=}') + setattr(self, hw_type, None) else: - raise TypeError("Mount is not instance of AbstractMount class, cannot add.") + raise TypeError(f"{hw_type.title()} is not an instance of {str(hw_class)} class") ########################################################################## # Methods @@ -265,7 +263,7 @@ def initialize(self): self.dome.connect() def power_down(self): - """Power down the observatory. Currently does nothing + """Power down the observatory. Currently just disconnects hardware. """ self.logger.debug("Shutting down observatory") if self.mount: @@ -281,13 +279,14 @@ def status(self): now = current_time() try: - if self.mount.is_initialized: + if self.mount and self.mount.is_initialized: status['mount'] = self.mount.status current_coords = self.mount.get_current_coordinates() status['mount']['current_ha'] = self.observer.target_hour_angle(now, current_coords) if self.mount.has_target: target_coords = self.mount.get_target_coordinates() - status['mount']['mount_target_ha'] = self.observer.target_hour_angle(now, target_coords) + status['mount']['mount_target_ha'] = self.observer.target_hour_angle(now, + target_coords) except Exception as e: # pragma: no cover self.logger.warning(f"Can't get mount status: {e!r}") @@ -300,7 +299,8 @@ def status(self): try: if self.current_observation: status['observation'] = self.current_observation.status - status['observation']['field_ha'] = self.observer.target_hour_angle(now, self.current_observation.field) + status['observation']['field_ha'] = self.observer.target_hour_angle(now, + self.current_observation.field) except Exception as e: # pragma: no cover self.logger.warning(f"Can't get observation status: {e!r}") @@ -501,7 +501,7 @@ def analyze_recent(self): # Get the image to compare image_id, image_path = self.current_observation.last_exposure - current_image = Image(image_path, location=self.earth_location, config_port=self._config_port) + current_image = Image(image_path, location=self.earth_location) solve_info = current_image.solve_field(skip_solved=False) diff --git a/src/panoptes/pocs/scheduler/__init__.py b/src/panoptes/pocs/scheduler/__init__.py index 88e15f0f7..691052824 100644 --- a/src/panoptes/pocs/scheduler/__init__.py +++ b/src/panoptes/pocs/scheduler/__init__.py @@ -6,8 +6,7 @@ from panoptes.pocs.scheduler.constraint import Duration from panoptes.pocs.scheduler.constraint import MoonAvoidance -# Below is needed for import -from panoptes.pocs.scheduler.scheduler import BaseScheduler # pragma: no flakes +from panoptes.pocs.scheduler.scheduler import BaseScheduler # noqa; needed for import from panoptes.utils import error from panoptes.utils import horizon as horizon_utils from panoptes.utils.library import load_module @@ -17,12 +16,12 @@ from panoptes.pocs.utils.location import create_location_from_config -def create_scheduler_from_config(config_port=6563, observer=None, *args, **kwargs): +def create_scheduler_from_config(observer=None, *args, **kwargs): """ Sets up the scheduler that will be used by the observatory """ logger = get_logger() - scheduler_config = get_config('scheduler', default=None, port=config_port) + scheduler_config = get_config('scheduler', default=None) logger.info(f'scheduler_config: {scheduler_config!r}') if scheduler_config is None or len(scheduler_config) == 0: @@ -31,15 +30,15 @@ def create_scheduler_from_config(config_port=6563, observer=None, *args, **kwarg if not observer: logger.debug(f'No Observer provided, creating from config.') - site_details = create_location_from_config(config_port=config_port) + site_details = create_location_from_config() observer = site_details['observer'] scheduler_type = scheduler_config.get('type', 'dispatch') # Read the targets from the file fields_file = scheduler_config.get('fields_file', 'simple.yaml') - fields_path = os.path.join(get_config('directories.targets', port=config_port), fields_file) - logger.debug('Creating scheduler: {}'.format(fields_path)) + fields_path = os.path.join(get_config('directories.targets'), fields_file) + logger.debug(f'Creating scheduler: {fields_path}') if os.path.exists(fields_path): @@ -47,9 +46,9 @@ def create_scheduler_from_config(config_port=6563, observer=None, *args, **kwarg # Load the required module module = load_module(f'panoptes.pocs.scheduler.{scheduler_type}') - obstruction_list = get_config('location.obstructions', default=[], port=config_port) + obstruction_list = get_config('location.obstructions', default=[]) default_horizon = get_config( - 'location.horizon', default=30 * u.degree, port=config_port) + 'location.horizon', default=30 * u.degree) horizon_line = horizon_utils.Horizon( obstructions=obstruction_list, @@ -58,21 +57,20 @@ def create_scheduler_from_config(config_port=6563, observer=None, *args, **kwarg # Simple constraint for now constraints = [ - Altitude(horizon=horizon_line, config_port=config_port), - MoonAvoidance(config_port=config_port), - Duration(default_horizon, weight=5., config_port=config_port) + Altitude(horizon=horizon_line), + MoonAvoidance(), + Duration(default_horizon, weight=5.) ] # Create the Scheduler instance scheduler = module.Scheduler(observer, fields_file=fields_path, constraints=constraints, - config_port=config_port) + *args, **kwargs) logger.debug("Scheduler created") except error.NotFound as e: raise error.NotFound(msg=e) else: - raise error.NotFound( - msg="Fields file does not exist: {}".format(fields_file)) + raise error.NotFound(msg=f"Fields file does not exist: {fields_file=}") return scheduler diff --git a/src/panoptes/pocs/scheduler/field.py b/src/panoptes/pocs/scheduler/field.py index 1b295faf7..d2055c721 100644 --- a/src/panoptes/pocs/scheduler/field.py +++ b/src/panoptes/pocs/scheduler/field.py @@ -14,18 +14,17 @@ def __init__(self, name, position, equinox='J2000', *args, **kwargs): Arguments: name {str} -- Name of the field, typically the name of object at - center `position` + center `position`. position {str} -- Center of field, can be anything accepted by - `~astropy.coordinates.SkyCoord` + `~astropy.coordinates.SkyCoord`. **kwargs {dict} -- Additional keywords to be passed to - `astroplan.ObservingBlock` + `astroplan.ObservingBlock`. """ PanBase.__init__(self, *args, **kwargs) - # Force an equinox - if equinox is None: - equinox = 'J2000' + # Force an equinox if they pass None (legacy). + equinox = equinox or 'J2000' super().__init__(SkyCoord(position, equinox=equinox, frame='icrs'), name=name, **kwargs) @@ -33,22 +32,10 @@ def __init__(self, name, position, equinox='J2000', *args, **kwargs): if not self._field_name: raise ValueError('Name is empty') - ################################################################################################## - # Properties - ################################################################################################## - @property def field_name(self): """ Flattened field name appropriate for paths """ return self._field_name - ################################################################################################## - # Methods - ################################################################################################## - - ################################################################################################## - # Private Methods - ################################################################################################## - def __str__(self): return self.name diff --git a/src/panoptes/pocs/scheduler/scheduler.py b/src/panoptes/pocs/scheduler/scheduler.py index dbe324a29..0c439aa7f 100644 --- a/src/panoptes/pocs/scheduler/scheduler.py +++ b/src/panoptes/pocs/scheduler/scheduler.py @@ -42,32 +42,26 @@ def __init__(self, observer, fields_list=None, fields_file=None, constraints=Non assert isinstance(observer, Observer) - self._fields_file = fields_file + self._observations = dict() + self._current_observation = None + self._fields_list = fields_list + + self.fields_file = fields_file # Setting the fields_list directly will clobber anything # from the fields_file. It comes second so we can specifically # clobber if passed. - self._fields_list = fields_list - self._observations = dict() self.observer = observer - self.constraints = constraints or list() - - self._current_observation = None self.observed_list = OrderedDict() - if not self.get_config('scheduler.check_file', default=False): + if self.get_config('scheduler.check_file', default=True): self.logger.debug("Reading initial set of fields") self.read_field_list() # Items common to each observation that shouldn't be computed each time. - self.common_properties = None - ########################################################################## - # Properties - ########################################################################## - @property def status(self): return { @@ -155,10 +149,7 @@ def fields_file(self, new_file): self.clear_available_observations() self._fields_file = new_file - if new_file is not None: - assert os.path.exists(new_file), \ - self.logger.error("Cannot load field list: {}".format(new_file)) - self.read_field_list() + self.read_field_list() @property def fields_list(self): @@ -184,10 +175,6 @@ def fields_list(self, new_list): self._fields_list = new_list self.read_field_list() - ########################################################################## - # Methods - ########################################################################## - def clear_available_observations(self): """Reset the list of available observations""" # Clear out existing list and observations @@ -229,27 +216,24 @@ def add_observation(self, field_config): Args: field_config (dict): Configuration items for `Observation` """ - if 'exptime' in field_config: - field_config['exptime'] = float(get_quantity_value( - field_config['exptime'], unit=u.second)) * u.second + with suppress(KeyError): + field_config['exptime'] = float(get_quantity_value(field_config['exptime'], unit=u.second)) * u.second - self.logger.debug("Adding {} to scheduler", field_config['name']) - field = Field(field_config['name'], field_config['position'], - config_port=self._config_port) - self.logger.debug("Created {} Field", field_config['name']) + self.logger.debug(f"Adding {field_config=} to scheduler") + field = Field(field_config['name'], field_config['position']) + self.logger.debug(f"Created {field.name=}") try: self.logger.debug(f"Creating observation for {field_config!r}") - obs = Observation(field, config_port=self._config_port, **field_config) - self.logger.debug(f"Observation created {obs}") + obs = Observation(field, **field_config) + self.logger.debug(f"Observation created for {field.name=}") except Exception as e: raise error.InvalidObservation(f"Skipping invalid field: {field_config!r} {e!r}") else: - self.logger.debug(f"Checking if {field.name} in self._observations") if field.name in self._observations: - self.logger.debug("Overriding existing entry for {}".format(field.name)) + self.logger.debug(f"Overriding existing entry for {field.name=}") self._observations[field.name] = obs - self.logger.debug(f"{obs} added") + self.logger.debug(f"{obs=} added") def remove_observation(self, field_name): """Removes an `Observation` from the scheduler @@ -265,8 +249,8 @@ def remove_observation(self, field_name): def read_field_list(self): """Reads the field file and creates valid `Observations` """ + self.logger.debug(f'Reading fields from file: {self.fields_file}') if self._fields_file is not None: - self.logger.debug('Reading fields from file: {}'.format(self.fields_file)) if not os.path.exists(self.fields_file): raise FileNotFoundError diff --git a/src/panoptes/pocs/state/machine.py b/src/panoptes/pocs/state/machine.py index 4a5c38f10..afc981706 100644 --- a/src/panoptes/pocs/state/machine.py +++ b/src/panoptes/pocs/state/machine.py @@ -20,7 +20,7 @@ class PanStateMachine(Machine): def __init__(self, state_machine_table, **kwargs): if isinstance(state_machine_table, str): - self.logger.info("Loading state table: {}".format(state_machine_table)) + self.logger.info(f"Loading state table: {state_machine_table}") state_machine_table = PanStateMachine.load_state_table( state_table_name=state_machine_table) @@ -62,7 +62,6 @@ def __init__(self, state_machine_table, **kwargs): self._state_machine_table = state_machine_table self.next_state = None - self.run_once = kwargs.get('run_once', False) self.logger.debug("State machine created") @@ -83,8 +82,8 @@ def next_state(self, value): # Methods ################################################################################################## - def run(self, exit_when_done=False, run_once=False): - """Runs the state machine loop + def run(self, exit_when_done=False, run_once=False, initial_next_state='ready'): + """Runs the state machine loop. This runs the state machine in a loop. Setting the machine property `is_running` to False will stop the loop. @@ -94,79 +93,80 @@ def run(self, exit_when_done=False, run_once=False): has become False, otherwise will wait (default) run_once (bool, optional): If the machine loop should only run one time, defaults to False to loop continuously. + initial_next_state (str, optional): The first state the machine should move to from + the `sleeping` state, default `ready`. """ - assert self.is_initialized, self.logger.error("POCS not initialized") + if not self.is_initialized: + self.logger.warning("POCS not initialized") + return False run_once = run_once or self.run_once - # Start with `get_ready` - self.next_state = 'ready' - - _loop_iteration = 0 - self.logger.debug(f'Starting run loop with keep_running={self.keep_running} ' - f'and connected={self.connected}') - while self.keep_running and self.connected: - state_changed = False - self.logger.info(f'Run loop: state={self.state} next_state={self.next_state} do_states={self.do_states}') - - # If we are processing the states - self.logger.debug(f'Observatory can_observe: {self.observatory.can_observe}') - if self.do_states and self.observatory.can_observe: - - # BEFORE TRANSITION - - # Wait for safety readying at given horizon level. - required_horizon = self._horizon_lookup.get(self.next_state, 'observe') - delay = self.get_config('wait_delay', default=60 * 3) # Check every 3 minutes - self.logger.debug(f'Checking for horizon={required_horizon} for {self.next_state}') - while not self.is_safe(no_warning=True, horizon=required_horizon): - self.logger.info(f'Waiting for horizon={required_horizon} for state={self.next_state}') - self.wait(delay=delay) - - # ENTER STATE - - self.logger.info(f'Going to next_state={self.next_state}') - try: - # The state's `on_enter` logic will be performed here. - state_changed = self.goto_next_state() - except Exception as e: - self.logger.critical(f"Problem going from {self.state} to {self.next_state}, exiting loop [{e!r}]") - # TODO should we automatically park here? - self.stop_states() - break + self.next_state = initial_next_state + + _transition_iteration = 0 + max_transition_attempts = self.get_config('max_transition_attempts', default=5) + + self.logger.debug(f'Starting run loop') + while self.keep_running: + # BEFORE TRANSITION TO STATE + self.logger.info(f'Run loop: {self.state=} {self.next_state=}') + + # Before moving to next state, check for required horizon level and wait if necessary. + required_horizon = self._horizon_lookup.get(self.next_state, 'observe') + self.logger.debug(f'Checking for {required_horizon=} for {self.next_state=}') + while not self.is_safe(no_warning=True, horizon=required_horizon): + self.logger.info(f'Waiting for {required_horizon=} for {self.next_state=}') + check_delay = self.get_config('wait_delay', + default=60 * 3) # Check every 3 minutes. + self.wait(delay=check_delay) + + # TRANSITION TO STATE + self.logger.info(f'Going to {self.next_state=}') + try: + # The state's `on_enter` logic will be performed here. + state_changed = self.goto_next_state() + except Exception as e: + self.logger.critical(f"Problem going from {self.state=} to {self.next_state=}" + f", exiting loop [{e!r}]") + # TODO should we automatically park here? + self.stop_states() + break - # AFTER TRANSITION - - # If we didn't successfully transition, wait a while then try again - max_iterations = self.get_config('pocs.MAX_TRANSITION_ATTEMPTS', default=5) - if not state_changed: - self.logger.warning(f"Failed to move from {self.state} to {self.next_state}") - if self.is_safe() is False: - self.logger.warning("Conditions have become unsafe; setting next state to 'parking'") - self.next_state = 'parking' - elif _loop_iteration > max_iterations: - self.logger.warning(f"Stuck in current state for {max_iterations} iterations, parking") - self.next_state = 'parking' - else: - _loop_iteration = _loop_iteration + 1 - self.logger.warning(f"Sleeping before trying again ({_loop_iteration}/{max_iterations})") - self.wait(with_status=False) + # AFTER TRANSITION TO STATE (NOW INSIDE STATE) + + # If we didn't successfully transition, wait a while then try again + if not state_changed: + self.logger.warning(f"Failed to move from {self.state=} to {self.next_state=}") + if self.is_safe() is False: + self.logger.warning( + "Conditions have become unsafe; setting next state to 'parking'") + self.next_state = 'parking' + elif _transition_iteration > max_transition_attempts: + self.logger.warning( + f"Stuck in current state for {max_transition_attempts=}, parking") + self.next_state = 'parking' else: - _loop_iteration = 0 - - # Note that `self.state` below has changed from above - # If we are in ready state then we are making one attempt through the loop. - if self.state == 'ready': - self._obs_run_retries -= 1 - - if self.state == 'sleeping' and run_once: + _transition_iteration = _transition_iteration + 1 + self.logger.warning( + f"Sleeping before trying again ({_transition_iteration}/" + f"{max_transition_attempts})") + self.wait(with_status=False, delay=7) # wait 7 seconds (no good reason) + else: + _transition_iteration = 0 + + # Note that `self.state` below has changed from above + + # We started in the sleeping state, so if we are back here we have + # done a full iteration. + if self.state == 'sleeping': + self._obs_run_retries -= 1 + if run_once: self.stop_states() - elif exit_when_done: - break - elif not self.interrupted: - # Sleep for one minute - self.logger.debug(f'Sleeping in run loop - why am I here?') - self.wait(delay=60) + + if exit_when_done: + self.logger.info(f'Leaving run loop {exit_when_done=}') + break def goto_next_state(self): """Make a transition to the next state. @@ -197,7 +197,7 @@ def goto_next_state(self): return state_changed def stop_states(self): - """ Stops the machine loop on the next iteration """ + """ Stops the machine loop on the next iteration by setting do_states=False """ self.logger.success("Stopping POCS states") self.do_states = False @@ -277,7 +277,8 @@ def after_state(self, event_data): event_data(transitions.EventData): Contains information about the event """ - self.logger.debug(f"After {event_data.event.name} transition. In {event_data.state.name} state") + self.logger.debug( + f"After {event_data.event.name} transition. In {event_data.state.name} state") ################################################################################################## # Class Methods @@ -310,7 +311,8 @@ def load_state_table(cls, state_table_name='simple_state_table'): with open(state_table_file, 'r') as f: state_table = from_yaml(f.read()) except Exception as err: - raise error.InvalidConfig(f'Problem loading state table yaml file: {err!r} {state_table_file}') + raise error.InvalidConfig( + f'Problem loading state table yaml file: {err!r} {state_table_file}') return state_table @@ -334,7 +336,6 @@ def _update_status(self, event_data): def _load_state(self, state, state_info=None): self.logger.debug(f"Loading state: {state}") - state_machine = None try: state_module = load_module('panoptes.{}.{}.{}'.format( self._states_location.replace("/", "."), @@ -347,7 +348,7 @@ def _load_state(self, state, state_info=None): on_enter_method = getattr(state_module, 'on_enter') setattr(self, f'on_enter_{state}', on_enter_method) - self.logger.debug(f"Added `on_enter` method from {state_module} {on_enter_method}") + self.logger.trace(f"Added `on_enter` method from {state_module} {on_enter_method}") if state_info is None: state_info = dict() @@ -357,7 +358,7 @@ def _load_state(self, state, state_info=None): self._horizon_lookup[state] = state_info['horizon'] del state_info['horizon'] - self.logger.debug(f"Creating state={state} with {state_info}") + self.logger.debug(f"Creating {state=} with {state_info=}") state_machine = MachineState(name=state, **state_info) # Add default callbacks. @@ -370,13 +371,11 @@ def _load_state(self, state, state_info=None): return state_machine def _load_transition(self, transition): - self.logger.debug(f"Loading transition: {transition}") - # Add `check_safety` as the first transition for all states conditions = listify(transition.get('conditions', [])) conditions.insert(0, 'check_safety') transition['conditions'] = conditions - self.logger.debug(f"Returning transition: {transition}") + self.logger.trace(f"Returning transition: {transition}") return transition diff --git a/src/panoptes/pocs/state/states/default/pointing.py b/src/panoptes/pocs/state/states/default/pointing.py index 945b91ec3..385617945 100644 --- a/src/panoptes/pocs/state/states/default/pointing.py +++ b/src/panoptes/pocs/state/states/default/pointing.py @@ -65,7 +65,6 @@ def waiting_cb(): pointing_image = Image( pointing_path, location=pocs.observatory.earth_location, - config_port=pocs.config_port ) pocs.logger.debug(f"Pointing image: {pointing_image}") diff --git a/src/panoptes/pocs/tests/bisque/test_mount.py b/src/panoptes/pocs/tests/bisque/test_mount.py index f82396b57..669f3340b 100644 --- a/src/panoptes/pocs/tests/bisque/test_mount.py +++ b/src/panoptes/pocs/tests/bisque/test_mount.py @@ -14,8 +14,8 @@ @pytest.fixture -def location(dynamic_config_server, config_port): - config = get_config(port=config_port) +def location(): + config = get_config() loc = config['location'] return EarthLocation(lon=loc['longitude'], lat=loc['latitude'], height=loc['elevation']) diff --git a/src/panoptes/pocs/tests/bisque/test_run.py b/src/panoptes/pocs/tests/bisque/test_run.py index da7cfa822..26a5014cd 100644 --- a/src/panoptes/pocs/tests/bisque/test_run.py +++ b/src/panoptes/pocs/tests/bisque/test_run.py @@ -14,8 +14,8 @@ @pytest.fixture -def location(dynamic_config_server, config_port): - config = get_config(port=config_port) +def location(): + config = get_config() loc = config['location'] return EarthLocation(lon=loc['longitude'], lat=loc['latitude'], height=loc['elevation']) @@ -31,13 +31,13 @@ def target_down(location): @pytest.fixture -def pocs(target, dynamic_config_server, config_port): +def pocs(target): try: del os.environ['POCSTIME'] except KeyError: pass - config = get_config(port=config_port) + config = get_config() pocs = POCS(simulator=['weather', 'night', 'camera'], run_once=True, config=config, db='panoptes_testing') diff --git a/src/panoptes/pocs/tests/test_astrohaven_dome.py b/src/panoptes/pocs/tests/test_astrohaven_dome.py index be29f9b75..3069a3cfe 100644 --- a/src/panoptes/pocs/tests/test_astrohaven_dome.py +++ b/src/panoptes/pocs/tests/test_astrohaven_dome.py @@ -12,18 +12,18 @@ @pytest.fixture(scope='function') -def dome(dynamic_config_server, config_port): +def dome(): # Install our test handlers for the duration. serial.protocol_handler_packages.append('panoptes.pocs.dome') # Modify the config so that the dome uses the right controller and port. - set_config('simulator', hardware.get_all_names(without=['dome']), port=config_port) + set_config('simulator', hardware.get_all_names(without=['dome'])) set_config('dome', { 'brand': 'Astrohaven', 'driver': 'astrohaven', 'port': 'astrohaven_simulator://', - }, port=config_port) - the_dome = create_dome_simulator(config_port=config_port) + }) + the_dome = create_dome_simulator() yield the_dome with suppress(Exception): diff --git a/src/panoptes/pocs/tests/test_base.py b/src/panoptes/pocs/tests/test_base.py index 25bb16e7b..fb9de8b77 100644 --- a/src/panoptes/pocs/tests/test_base.py +++ b/src/panoptes/pocs/tests/test_base.py @@ -2,32 +2,13 @@ from panoptes.pocs.base import PanBase -from panoptes.utils.config.client import set_config from panoptes.utils.database import PanDB -# def test_mount_in_config(dynamic_config_server, config_port): -# set_config('mount', {}, port=config_port) -# with pytest.raises(SystemExit): -# PanBase(config_port=config_port) +def test_with_logger(): + PanBase() -# def test_directories_in_config(dynamic_config_server, config_port): -# set_config('directories', {}, port=config_port) -# with pytest.raises(SystemExit): -# PanBase(config_port=config_port) - - -# def test_state_machine_in_config(dynamic_config_server, config_port): -# set_config('state_machine', {}, port=config_port) -# with pytest.raises(SystemExit): -# PanBase(config_port=config_port) - - -def test_with_logger(dynamic_config_server, config_port): - PanBase(config_port=config_port) - - -def test_with_db(dynamic_config_server, config_port): - base = PanBase(config_port=config_port, db=PanDB(db_type='memory', db_name='tester')) +def test_with_db(): + base = PanBase(db=PanDB(db_type='memory', db_name='tester')) assert isinstance(base, PanBase) diff --git a/src/panoptes/pocs/tests/test_base_scheduler.py b/src/panoptes/pocs/tests/test_base_scheduler.py index b9ad4c898..d3e620740 100644 --- a/src/panoptes/pocs/tests/test_base_scheduler.py +++ b/src/panoptes/pocs/tests/test_base_scheduler.py @@ -14,18 +14,18 @@ @pytest.fixture -def constraints(dynamic_config_server, config_port): - return [MoonAvoidance(config_port=config_port), Duration(30 * u.deg, config_port=config_port)] +def constraints(): + return [MoonAvoidance(), Duration(30 * u.deg)] @pytest.fixture -def simple_fields_file(dynamic_config_server, config_port): - return get_config('directories.targets', port=config_port) + '/simulator.yaml' +def simple_fields_file(): + return get_config('directories.targets') + '/simulator.yaml' @pytest.fixture -def observer(dynamic_config_server, config_port): - loc = get_config('location', port=config_port) +def observer(): + loc = get_config('location') location = EarthLocation(lon=loc['longitude'], lat=loc['latitude'], height=loc['elevation']) return Observer(location=location, name="Test Observer", timezone=loc['timezone']) @@ -75,72 +75,58 @@ def field_list(): @pytest.fixture -def scheduler(dynamic_config_server, config_port, field_list, observer, constraints): +def scheduler(field_list, observer, constraints): return Scheduler(observer, fields_list=field_list, - constraints=constraints, - config_port=config_port) + constraints=constraints) -def test_scheduler_load_no_params(dynamic_config_server, config_port): +def test_scheduler_load_no_params(): with pytest.raises(TypeError): - Scheduler(config_port=config_port) + Scheduler() -def test_no_observer(dynamic_config_server, config_port, simple_fields_file): +def test_no_observer(simple_fields_file): with pytest.raises(TypeError): - Scheduler(fields_file=simple_fields_file, config_port=config_port) + Scheduler(fields_file=simple_fields_file) -def test_bad_observer(dynamic_config_server, config_port, simple_fields_file, constraints): +def test_bad_observer(simple_fields_file, constraints): with pytest.raises(TypeError): Scheduler(fields_file=simple_fields_file, - constraints=constraints, - config_port=config_port) + constraints=constraints) -def test_loading_target_file_check_file(dynamic_config_server, - config_port, - observer, +def test_loading_target_file_check_file(observer, simple_fields_file, constraints): - set_config('scheduler.check_file', False, port=config_port) + set_config('scheduler.check_file', False) scheduler = Scheduler(observer, fields_file=simple_fields_file, - constraints=constraints, - config_port=config_port - ) + constraints=constraints) # Check the hidden property as the public one # will populate if not found. assert len(scheduler._observations) -def test_loading_target_file_no_check_file(dynamic_config_server, - config_port, - observer, - simple_fields_file, - constraints): - # If check_file is True then we will check the file - # before each call to `get_observation`, but *not* - # when the Scheduler is initialized. - set_config('scheduler.check_file', True, port=config_port) +def test_loading_target_file_check_file(observer, + simple_fields_file, + constraints): + set_config('scheduler.check_file', True) scheduler = Scheduler(observer, fields_file=simple_fields_file, constraints=constraints, - config_port=config_port ) # Check the hidden property as the public one # will populate if not found. - assert len(scheduler._observations) == 0 + assert len(scheduler._observations) > 0 -def test_loading_target_file_via_property(dynamic_config_server, - config_port, - simple_fields_file, +def test_loading_target_file_via_property(simple_fields_file, observer, constraints): scheduler = Scheduler(observer, fields_file=simple_fields_file, - constraints=constraints, config_port=config_port) + constraints=constraints) scheduler._observations = dict() assert scheduler.observations is not None @@ -149,9 +135,9 @@ def test_with_location(scheduler): assert isinstance(scheduler, Scheduler) -def test_loading_bad_target_file(dynamic_config_server, config_port, observer): +def test_loading_bad_target_file(observer): with pytest.raises(FileNotFoundError): - Scheduler(observer, fields_file='/var/path/foo.bar', config_port=config_port) + Scheduler(observer, fields_file='/var/path/foo.bar') def test_new_fields_file(scheduler, simple_fields_file): diff --git a/src/panoptes/pocs/tests/test_camera.py b/src/panoptes/pocs/tests/test_camera.py index 779ae8af6..f488d4800 100644 --- a/src/panoptes/pocs/tests/test_camera.py +++ b/src/panoptes/pocs/tests/test_camera.py @@ -62,13 +62,13 @@ 'simulator', 'simulator_focuser', 'simulator_filterwheel', 'simulator_sdk', 'sbig', 'fli', 'zwo' ]) -def camera(request, dynamic_config_server, config_port): +def camera(request): CamClass = request.param[0] cam_params = request.param[1] if isinstance(cam_params, dict): # Simulator - camera = CamClass(config_port=config_port, **cam_params) + camera = CamClass(**cam_params) else: # Lookup real hardware device name in real life config server. for cam_config in get_config('cameras.devices'): @@ -80,7 +80,7 @@ def camera(request, dynamic_config_server, config_port): assert camera.is_ready yield camera - # simulator_sdk needs this explictly removed for some reason. + # simulator_sdk needs this explicitly removed for some reason. with suppress(AttributeError): type(camera)._assigned_cameras.discard(camera.uid) @@ -112,90 +112,89 @@ def test_create_camera_simulator(): create_camera_simulator(num_cameras=0) -def test_create_cameras_from_config_no_autodetect(dynamic_config_server, config_port): - set_config('cameras.auto_detect', False, port=config_port) +def test_create_cameras_from_config_no_autodetect(): + set_config('cameras.auto_detect', False) set_config('cameras.devices', [ dict(model='canon_gphoto2', port='/dev/fake01'), dict(model='canon_gphoto2', port='/dev/fake02'), - ], port=config_port) + ]) with pytest.raises(error.CameraNotFound): - create_cameras_from_config(config_port=config_port) + create_cameras_from_config() -def test_create_cameras_from_config_autodetect(dynamic_config_server, config_port): - set_config('cameras.auto_detect', True, port=config_port) +def test_create_cameras_from_config_autodetect(): + set_config('cameras.auto_detect', True) with pytest.raises(error.CameraNotFound): - create_cameras_from_config(config_port=config_port) + create_cameras_from_config() # Hardware independent tests, mostly use simulator: -def test_sim_create_focuser(dynamic_config_server, config_port): - sim_camera = SimCamera(focuser={'model': 'simulator', 'focus_port': '/dev/ttyFAKE'}, - config_port=config_port) +def test_sim_create_focuser(): + sim_camera = SimCamera(focuser={'model': 'simulator', 'focus_port': '/dev/ttyFAKE'}) assert isinstance(sim_camera.focuser, Focuser) -def test_sim_passed_focuser(dynamic_config_server, config_port): - sim_focuser = Focuser(port='/dev/ttyFAKE', config_port=config_port) - sim_camera = SimCamera(focuser=sim_focuser, config_port=config_port) +def test_sim_passed_focuser(): + sim_focuser = Focuser(port='/dev/ttyFAKE') + sim_camera = SimCamera(focuser=sim_focuser) assert sim_camera.focuser is sim_focuser -def test_sim_bad_focuser(dynamic_config_server, config_port): +def test_sim_bad_focuser(): with pytest.raises((NotFound)): - SimCamera(focuser={'model': 'NOTAFOCUSER'}, config_port=config_port) + SimCamera(focuser={'model': 'NOTAFOCUSER'}) -def test_sim_worse_focuser(dynamic_config_server, config_port): - sim_camera = SimCamera(focuser='NOTAFOCUSER', config_port=config_port) +def test_sim_worse_focuser(): + sim_camera = SimCamera(focuser='NOTAFOCUSER') # Will log an error but raise no exceptions assert sim_camera.focuser is None -def test_sim_string(dynamic_config_server, config_port): - sim_camera = SimCamera(config_port=config_port) +def test_sim_string(): + sim_camera = SimCamera() assert str(sim_camera) == 'Simulated Camera ({}) on None'.format(sim_camera.uid) - sim_camera = SimCamera(name='Sim', port='/dev/ttyFAKE', config_port=config_port) + sim_camera = SimCamera(name='Sim', port='/dev/ttyFAKE') assert str(sim_camera) == 'Sim ({}) on /dev/ttyFAKE'.format(sim_camera.uid) -def test_sim_file_extension(dynamic_config_server, config_port): - sim_camera = SimCamera(config_port=config_port) +def test_sim_file_extension(): + sim_camera = SimCamera() assert sim_camera.file_extension == 'fits' - sim_camera = SimCamera(file_extension='FIT', config_port=config_port) + sim_camera = SimCamera(file_extension='FIT') assert sim_camera.file_extension == 'FIT' -def test_sim_readout_time(dynamic_config_server, config_port): - sim_camera = SimCamera(config_port=config_port) +def test_sim_readout_time(): + sim_camera = SimCamera() assert sim_camera.readout_time == 1.0 - sim_camera = SimCamera(readout_time=2.0, config_port=config_port) + sim_camera = SimCamera(readout_time=2.0) assert sim_camera.readout_time == 2.0 -def test_sdk_no_serial_number(dynamic_config_server, config_port): +def test_sdk_no_serial_number(): with pytest.raises(ValueError): - SimSDKCamera(config_port=config_port) + SimSDKCamera() -def test_sdk_camera_not_found(dynamic_config_server, config_port): +def test_sdk_camera_not_found(): with pytest.raises(error.PanError): - SimSDKCamera(serial_number='SSC404', config_port=config_port) + SimSDKCamera(serial_number='SSC404') -def test_sdk_already_in_use(dynamic_config_server, config_port): - sim_camera = SimSDKCamera(serial_number='SSC999', config_port=config_port) +def test_sdk_already_in_use(): + sim_camera = SimSDKCamera(serial_number='SSC999') assert sim_camera with pytest.raises(error.PanError): - SimSDKCamera(serial_number='SSC999', config_port=config_port) + SimSDKCamera(serial_number='SSC999') # Hardware independent tests for SBIG camera -def test_sbig_driver_bad_path(dynamic_config_server, config_port): +def test_sbig_driver_bad_path(): """ Manually specify an incorrect path for the SBIG shared library. The CDLL loader should raise OSError when it fails. Can't test a successful @@ -203,11 +202,11 @@ def test_sbig_driver_bad_path(dynamic_config_server, config_port): CDLL unload problem. """ with pytest.raises(OSError): - SBIGDriver(library_path='no_library_here', config_port=config_port) + SBIGDriver(library_path='no_library_here') @pytest.mark.filterwarnings('ignore:Could not connect to SBIG Camera') -def test_sbig_bad_serial(dynamic_config_server, config_port): +def test_sbig_bad_serial(): """ Attempt to create an SBIG camera instance for a specific non-existent camera. No actual cameras are required to run this test but the SBIG @@ -216,7 +215,7 @@ def test_sbig_bad_serial(dynamic_config_server, config_port): if find_library('sbigudrv') is None: pytest.skip("Test requires SBIG camera driver to be installed") with pytest.raises(error.PanError): - SBIGCamera(serial_number='NOTAREALSERIALNUMBER', config_port=config_port) + SBIGCamera(serial_number='NOTAREALSERIALNUMBER') # *Potentially* hardware dependant tests: @@ -463,12 +462,12 @@ def test_exposure_timeout(camera, tmpdir, caplog): assert exposure_event.is_set() -def test_observation(dynamic_config_server, config_port, camera, images_dir): +def test_observation(camera, images_dir): """ Tests functionality of take_observation() """ - field = Field('Test Observation', '20h00m43.7135s +22d42m39.0645s', config_port=config_port) - observation = Observation(field, exptime=1.5 * u.second, config_port=config_port) + field = Field('Test Observation', '20h00m43.7135s +22d42m39.0645s') + observation = Observation(field, exptime=1.5 * u.second) observation.seq_time = '19991231T235959' camera.take_observation(observation, headers={}) time.sleep(7) @@ -479,12 +478,12 @@ def test_observation(dynamic_config_server, config_port, camera, images_dir): os.remove(fn) -def test_observation_nofilter(dynamic_config_server, config_port, camera, images_dir): +def test_observation_nofilter(camera, images_dir): """ Tests functionality of take_observation() """ - field = Field('Test Observation', '20h00m43.7135s +22d42m39.0645s', config_port=config_port) - observation = Observation(field, exptime=1.5 * u.second, filter_name=None, config_port=config_port) + field = Field('Test Observation', '20h00m43.7135s +22d42m39.0645s') + observation = Observation(field, exptime=1.5 * u.second, filter_name=None) observation.seq_time = '19991231T235959' camera.take_observation(observation, headers={}) time.sleep(7) diff --git a/src/panoptes/pocs/tests/test_constraints.py b/src/panoptes/pocs/tests/test_constraints.py index 80a8df555..bebc07551 100644 --- a/src/panoptes/pocs/tests/test_constraints.py +++ b/src/panoptes/pocs/tests/test_constraints.py @@ -23,16 +23,16 @@ @pytest.fixture(scope='function') -def observer(dynamic_config_server, config_port): - loc = get_config('location', port=config_port) +def observer(): + loc = get_config('location') location = EarthLocation(lon=loc['longitude'], lat=loc['latitude'], height=loc['elevation']) return Observer(location=location, name="Test Observer", timezone=loc['timezone']) @pytest.fixture(scope='function') -def horizon_line(dynamic_config_server, config_port): - obstruction_list = get_config('location.obstructions', default=list(), port=config_port) - default_horizon = get_config('location.horizon', port=config_port) +def horizon_line(): + obstruction_list = get_config('location.obstructions', default=list()) + default_horizon = get_config('location.horizon') horizon_line = horizon_utils.Horizon( obstructions=obstruction_list, diff --git a/src/panoptes/pocs/tests/test_dispatch_scheduler.py b/src/panoptes/pocs/tests/test_dispatch_scheduler.py index 58176fd0c..a1b51bb1a 100644 --- a/src/panoptes/pocs/tests/test_dispatch_scheduler.py +++ b/src/panoptes/pocs/tests/test_dispatch_scheduler.py @@ -15,24 +15,24 @@ @pytest.fixture -def constraints(dynamic_config_server, config_port): - return [MoonAvoidance(config_port=config_port), Duration(30 * u.deg, config_port=config_port)] +def constraints(): + return [MoonAvoidance(), Duration(30 * u.deg)] @pytest.fixture -def observer(dynamic_config_server, config_port): - loc = get_config('location', port=config_port) +def observer(): + loc = get_config('location') location = EarthLocation(lon=loc['longitude'], lat=loc['latitude'], height=loc['elevation']) return Observer(location=location, name="Test Observer", timezone=loc['timezone']) @pytest.fixture() -def field_file(dynamic_config_server, config_port): - scheduler_config = get_config('scheduler', default={}, port=config_port) +def field_file(): + scheduler_config = get_config('scheduler', default={}) # Read the targets from the file fields_file = scheduler_config.get('fields_file', 'simple.yaml') - fields_path = os.path.join(get_config('directories.targets', port=config_port), fields_file) + fields_path = os.path.join(get_config('directories.targets'), fields_file) return fields_path @@ -82,19 +82,17 @@ def field_list(): @pytest.fixture -def scheduler(dynamic_config_server, config_port, field_list, observer, constraints): +def scheduler(field_list, observer, constraints): return Scheduler(observer, fields_list=field_list, - constraints=constraints, - config_port=config_port) + constraints=constraints) @pytest.fixture -def scheduler_from_file(dynamic_config_server, config_port, field_file, observer, constraints): +def scheduler_from_file(field_file, observer, constraints): return Scheduler(observer, fields_file=field_file, - constraints=constraints, - config_port=config_port) + constraints=constraints) def test_get_observation(scheduler): @@ -106,9 +104,7 @@ def test_get_observation(scheduler): assert isinstance(best[1], float) -def test_get_observation_reread(dynamic_config_server, - config_port, - field_list, +def test_get_observation_reread(field_list, observer, temp_file, constraints): @@ -120,8 +116,7 @@ def test_get_observation_reread(dynamic_config_server, scheduler = Scheduler(observer, fields_file=temp_file, - constraints=constraints, - config_port=config_port) + constraints=constraints) # Get observation as above best = scheduler.get_observation(time=time) diff --git a/src/panoptes/pocs/tests/test_dome_simulator.py b/src/panoptes/pocs/tests/test_dome_simulator.py index 08a58b948..fe1af389f 100644 --- a/src/panoptes/pocs/tests/test_dome_simulator.py +++ b/src/panoptes/pocs/tests/test_dome_simulator.py @@ -7,13 +7,13 @@ @pytest.fixture(scope="function") -def dome(dynamic_config_server, config_port): +def dome(): set_config('dome', { 'brand': 'Simulacrum', 'driver': 'simulator', - }, port=config_port) + }) - the_dome = create_dome_simulator(config_port=config_port) + the_dome = create_dome_simulator() yield the_dome diff --git a/src/panoptes/pocs/tests/test_filterwheel.py b/src/panoptes/pocs/tests/test_filterwheel.py index dcdc55808..0e5197b6c 100644 --- a/src/panoptes/pocs/tests/test_filterwheel.py +++ b/src/panoptes/pocs/tests/test_filterwheel.py @@ -10,11 +10,10 @@ @pytest.fixture(scope='function') -def filterwheel(dynamic_config_server, config_port): +def filterwheel(): sim_filterwheel = SimFilterWheel(filter_names=['one', 'deux', 'drei', 'quattro'], move_time=0.1 * u.second, - timeout=0.5 * u.second, - config_port=config_port) + timeout=0.5 * u.second) return sim_filterwheel @@ -26,31 +25,30 @@ def test_init(filterwheel): assert filterwheel.is_connected -def test_camera_init(dynamic_config_server, config_port): +def test_camera_init(): sim_camera = SimCamera(filterwheel={'model': 'simulator', - 'filter_names': ['one', 'deux', 'drei', 'quattro']}, - config_port=config_port) + 'filter_names': ['one', 'deux', 'drei', 'quattro']}) assert isinstance(sim_camera.filterwheel, SimFilterWheel) assert sim_camera.filterwheel.is_connected assert sim_camera.filterwheel.uid assert sim_camera.filterwheel.camera is sim_camera -def test_camera_no_filterwheel(dynamic_config_server, config_port): - sim_camera = SimCamera(config_port=config_port) +def test_camera_no_filterwheel(): + sim_camera = SimCamera() assert sim_camera.filterwheel is None -def test_camera_association_on_init(dynamic_config_server, config_port): - sim_camera = SimCamera(config_port=config_port) +def test_camera_association_on_init(): + sim_camera = SimCamera() sim_filterwheel = SimFilterWheel(filter_names=['one', 'deux', 'drei', 'quattro'], - camera=sim_camera, config_port=config_port) + camera=sim_camera) assert sim_filterwheel.camera is sim_camera -def test_with_no_name(dynamic_config_server, config_port): +def test_with_no_name(): with pytest.raises(ValueError): - SimFilterWheel(config_port=config_port) + SimFilterWheel() # Basic property getting and (not) setting @@ -137,11 +135,10 @@ def test_move_bad_name(filterwheel): filterwheel.move_to('cinco') -def test_move_timeout(dynamic_config_server, config_port, caplog): +def test_move_timeout(caplog): slow_filterwheel = SimFilterWheel(filter_names=['one', 'deux', 'drei', 'quattro'], move_time=0.5, - timeout=0.2, - config_port=config_port) + timeout=0.2) with pytest.raises(error.Timeout): # Move should take 0.3 seconds, more than timeout. slow_filterwheel.position = 4 @@ -152,12 +149,11 @@ def test_move_timeout(dynamic_config_server, config_port, caplog): @pytest.mark.parametrize("name, unidirectional, expected", [("unidirectional", True, 0.3), ("bidirectional", False, 0.1)]) -def test_move_times(dynamic_config_server, config_port, name, unidirectional, expected): +def test_move_times(name, unidirectional, expected): sim_filterwheel = SimFilterWheel(filter_names=['one', 'deux', 'drei', 'quattro'], move_time=0.1 * u.second, unidirectional=unidirectional, - timeout=0.5 * u.second, - config_port=config_port) + timeout=0.5 * u.second) sim_filterwheel.position = 1 assert timeit("sim_filterwheel.position = 2", number=1, globals=locals()) == \ pytest.approx(0.1, rel=1e-1) @@ -167,10 +163,9 @@ def test_move_times(dynamic_config_server, config_port, name, unidirectional, ex pytest.approx(expected, rel=2e-1) -def test_move_exposing(dynamic_config_server, config_port, tmpdir): +def test_move_exposing(tmpdir): sim_camera = SimCamera(filterwheel={'model': 'simulator', - 'filter_names': ['one', 'deux', 'drei', 'quattro']}, - config_port=config_port) + 'filter_names': ['one', 'deux', 'drei', 'quattro']}) fits_path = str(tmpdir.join('test_exposure.fits')) exp_event = sim_camera.take_exposure(filename=fits_path, seconds=0.1) with pytest.raises(error.PanError): diff --git a/src/panoptes/pocs/tests/test_focuser.py b/src/panoptes/pocs/tests/test_focuser.py index a9fb15f96..93c84125c 100644 --- a/src/panoptes/pocs/tests/test_focuser.py +++ b/src/panoptes/pocs/tests/test_focuser.py @@ -3,7 +3,7 @@ import pytest from threading import Thread -from panoptes.utils.config import load_config +from panoptes.utils.config.helpers import load_config from panoptes.pocs.focuser.simulator import Focuser as SimFocuser from panoptes.pocs.focuser.birger import Focuser as BirgerFocuser @@ -13,11 +13,12 @@ params = [SimFocuser, BirgerFocuser, FocusLynxFocuser] ids = ['simulator', 'birger', 'focuslynx'] + # Ugly hack to access id inside fixture @pytest.fixture(scope='function', params=zip(params, ids), ids=ids) -def focuser(request, dynamic_config_server, config_port): +def focuser(request): if request.param[0] == SimFocuser: # Simulated focuser, just create one and return it return request.param[0]() @@ -44,7 +45,7 @@ def focuser(request, dynamic_config_server, config_port): request.param[1])) # Create and return a Focuser based on the first config - return request.param[0](**focuser_configs[0], config_port=config_port) + return request.param[0](**focuser_configs[0]) @pytest.fixture(scope='function') diff --git a/src/panoptes/pocs/tests/test_images.py b/src/panoptes/pocs/tests/test_images.py index b1ccfa567..846e67c30 100644 --- a/src/panoptes/pocs/tests/test_images.py +++ b/src/panoptes/pocs/tests/test_images.py @@ -21,47 +21,45 @@ def copy_file_to_dir(to_dir, file): return result -def test_fits_exists(dynamic_config_server, config_port, unsolved_fits_file): +def test_fits_exists(unsolved_fits_file): with pytest.raises(AssertionError): - Image(unsolved_fits_file.replace('.fits', '.fit'), config_port=config_port) + Image(unsolved_fits_file.replace('.fits', '.fit')) -def test_fits_extension(dynamic_config_server, config_port): +def test_fits_extension(): with pytest.raises(AssertionError): - Image(os.path.join(os.environ['POCS'], 'pocs', 'images.py'), config_port=config_port) + Image(os.path.join(os.environ['POCS'], 'pocs', 'images.py')) -def test_fits_noheader(dynamic_config_server, config_port, noheader_fits_file): +def test_fits_noheader(noheader_fits_file): with pytest.raises(KeyError): - Image(noheader_fits_file, config_port=config_port) + Image(noheader_fits_file) -def test_solve_timeout(dynamic_config_server, config_port, tiny_fits_file): +def test_solve_timeout(tiny_fits_file): with tempfile.TemporaryDirectory() as tmpdir: tiny_fits_file = copy_file_to_dir(tmpdir, tiny_fits_file) - im0 = Image(tiny_fits_file, config_port=config_port) + im0 = Image(tiny_fits_file) assert str(im0) with pytest.raises(Timeout): im0.solve_field(verbose=True, replace=False, radius=4, timeout=1) -def test_fail_solve(dynamic_config_server, config_port, tiny_fits_file): +def test_fail_solve(tiny_fits_file): with tempfile.TemporaryDirectory() as tmpdir: tiny_fits_file = copy_file_to_dir(tmpdir, tiny_fits_file) - im0 = Image(tiny_fits_file, config_port=config_port) + im0 = Image(tiny_fits_file) assert str(im0) with pytest.raises(SolveError): im0.solve_field(verbose=True, replace=False, radius=4) -def test_solve_field_unsolved(dynamic_config_server, - config_port, - unsolved_fits_file, +def test_solve_field_unsolved(unsolved_fits_file, solved_fits_file): # We place the input images into a temp directory so that output images # are also in the temp directory. with tempfile.TemporaryDirectory() as tmpdir: - im0 = Image(copy_file_to_dir(tmpdir, unsolved_fits_file), config_port=config_port) + im0 = Image(copy_file_to_dir(tmpdir, unsolved_fits_file)) assert isinstance(im0, Image) assert im0.wcs is None @@ -77,15 +75,15 @@ def test_solve_field_unsolved(dynamic_config_server, assert im0.ha is not None # Compare it to another file of known offset. - im1 = Image(copy_file_to_dir(tmpdir, solved_fits_file), config_port=config_port) + im1 = Image(copy_file_to_dir(tmpdir, solved_fits_file)) offset_info = im0.compute_offset(im1) # print('offset_info:', offset_info) expected_offset = [10.1 * u.arcsec, 5.29 * u.arcsec, 8.77 * u.arcsec] assert u.allclose(offset_info, expected_offset, rtol=0.005) -def test_solve_field_solved(dynamic_config_server, config_port, solved_fits_file): - im0 = Image(solved_fits_file, config_port=config_port) +def test_solve_field_solved(solved_fits_file): + im0 = Image(solved_fits_file) assert isinstance(im0, Image) assert im0.wcs is not None @@ -100,24 +98,22 @@ def test_solve_field_solved(dynamic_config_server, config_port, solved_fits_file assert isinstance(im0.pointing, SkyCoord) -def test_pointing_error_no_wcs(dynamic_config_server, config_port, unsolved_fits_file): - im0 = Image(unsolved_fits_file, config_port=config_port) +def test_pointing_error_no_wcs(unsolved_fits_file): + im0 = Image(unsolved_fits_file) with pytest.raises(AssertionError): im0.pointing_error -def test_pointing_error_passed_wcs(dynamic_config_server, - config_port, - unsolved_fits_file, +def test_pointing_error_passed_wcs(unsolved_fits_file, solved_fits_file): - im0 = Image(unsolved_fits_file, wcs_file=solved_fits_file, config_port=config_port) + im0 = Image(unsolved_fits_file, wcs_file=solved_fits_file) assert isinstance(im0.pointing_error, OffsetError) -def test_pointing_error(dynamic_config_server, config_port, solved_fits_file): - im0 = Image(solved_fits_file, config_port=config_port) +def test_pointing_error(solved_fits_file): + im0 = Image(solved_fits_file) im0.solve_field(verbose=True, replace=False, radius=4) diff --git a/src/panoptes/pocs/tests/test_ioptron.py b/src/panoptes/pocs/tests/test_ioptron.py index 470d3322a..1f9a7829f 100644 --- a/src/panoptes/pocs/tests/test_ioptron.py +++ b/src/panoptes/pocs/tests/test_ioptron.py @@ -13,13 +13,13 @@ @pytest.fixture -def location(dynamic_config_server, config_port): - loc = get_config('location', port=config_port) +def location(): + loc = get_config('location') return EarthLocation(lon=loc['longitude'], lat=loc['latitude'], height=loc['elevation']) @pytest.fixture(scope="function") -def mount(dynamic_config_server, config_port, location): +def mount(location): with suppress(KeyError): del os.environ['POCSTIME'] @@ -27,9 +27,9 @@ def mount(dynamic_config_server, config_port, location): { 'brand': 'bisque', 'template_dir': 'resources/bisque', - }, port=config_port) + }) - return Mount(location=location, config_port=config_port) + return Mount(location=location) @pytest.mark.with_mount @@ -42,13 +42,10 @@ def test_loading_without_config(): @pytest.mark.with_mount class TestMount(object): - """ Test the mount """ @pytest.fixture(autouse=True) def setup(self): - - # Don't use config_port because we use real live config_server location = create_location_from_config() # Can't supply full location, need earth_location @@ -91,7 +88,6 @@ def test_unpark_park(self): def test_get_tracking_correction(mount): - offsets = [ # HA, ΔRA, ΔDec, Magnitude (2, -13.0881456, 1.4009, 12.154), @@ -148,7 +144,6 @@ def test_get_tracking_correction(mount): def test_get_tracking_correction_custom(mount): - min_tracking = 105 max_tracking = 950 diff --git a/src/panoptes/pocs/tests/test_mount.py b/src/panoptes/pocs/tests/test_mount.py index 8dcb43abe..42923dbaf 100644 --- a/src/panoptes/pocs/tests/test_mount.py +++ b/src/panoptes/pocs/tests/test_mount.py @@ -10,6 +10,21 @@ from panoptes.utils import error from panoptes.utils.config.client import get_config from panoptes.utils.config.client import set_config +from panoptes.utils.serializers import to_json + +import requests + +config_host = 'localhost' +config_port = 6563 +url = f'http://{config_host}:{config_port}/reset-config' + + +def reset_conf(): + response = requests.post(url, + data=to_json({'reset': True}), + headers={'Content-Type': 'application/json'} + ) + assert response.ok def test_create_mount_simulator(): @@ -18,72 +33,81 @@ def test_create_mount_simulator(): assert isinstance(mount, AbstractMount) is True -def test_create_mount_simulator_with_config(dynamic_config_server, config_port): +def test_create_mount_simulator_with_config(): # Remove mount from list of simulators. set_config('simulator', hardware.get_all_names(without=['mount'])) # But setting the driver to `simulator` should return simulator. - set_config('mount.driver', 'simulator', port=config_port) + set_config('mount.driver', 'simulator') - mount = create_mount_from_config(config_port=config_port) + mount = create_mount_from_config() assert isinstance(mount, AbstractMount) is True + reset_conf() -def test_create_mount_without_mount_info(dynamic_config_server, config_port): +def test_create_mount_without_mount_info(): # Set the mount config to none and then don't pass anything for error. - set_config('mount', None, port=config_port) + set_config('mount', None) set_config('simulator', hardware.get_all_names(without=['mount'])) with pytest.raises(error.MountNotFound): - create_mount_from_config(config_port=config_port, mount_info=None) + create_mount_from_config(mount_info=None) + reset_conf() -def test_create_mount_with_mount_info(dynamic_config_server, config_port): + +def test_create_mount_with_mount_info(): # Pass the mount info directly with nothing in config. - mount_info = get_config('mount', port=config_port) + mount_info = get_config('mount', default=dict()) mount_info['driver'] = 'simulator' # Remove info from config. - set_config('mount', None, port=config_port) + set_config('mount', None) set_config('simulator', hardware.get_all_names(without=['mount'])) - assert isinstance(create_mount_from_config(config_port=config_port, - mount_info=mount_info), AbstractMount) is True + assert isinstance(create_mount_from_config(mount_info=mount_info), AbstractMount) is True + + reset_conf() -def test_create_mount_with_earth_location(dynamic_config_server, config_port): +def test_create_mount_with_earth_location(): # Get location to pass manually. loc = create_location_from_config() # Set config to not have a location. - set_config('location', None, port=config_port) - assert isinstance(create_mount_from_config(config_port=config_port, - earth_location=loc), AbstractMount) is True + set_config('location', None) + set_config('simulator', hardware.get_all_names()) + assert isinstance(create_mount_from_config(earth_location=loc['earth_location']), AbstractMount) is True + + reset_conf() -def test_create_mount_without_earth_location(dynamic_config_server, config_port): - set_config('location', None, port=config_port) +def test_create_mount_without_earth_location(): + set_config('location', None) with pytest.raises(error.PanError): - create_mount_from_config(config_port=config_port, earth_location=None) + create_mount_from_config(earth_location=None) + reset_conf() -def test_bad_mount_port(dynamic_config_server, config_port): +def test_bad_mount_port(): # Remove the mount from the list of simulators so it thinks we have a real one. - simulators = get_config('simulator', port=config_port) - with suppress(KeyError): + simulators = get_config('simulator') + with suppress(KeyError, AttributeError): simulators.remove('mount') - set_config('simulator', simulators, port=config_port) + set_config('simulator', simulators) # Set a bad port, which should cause a fail before actual mount creation. set_config('mount.serial.port', 'foobar') with pytest.raises(error.MountNotFound): - create_mount_from_config(config_port=config_port) + create_mount_from_config() + reset_conf() -def test_bad_mount_driver(dynamic_config_server, config_port): +def test_bad_mount_driver(): # Remove the mount from the list of simulators so it thinks we have a real one. - simulators = get_config('simulator', port=config_port) - with suppress(KeyError): + simulators = get_config('simulator') + with suppress(KeyError, AttributeError): simulators.remove('mount') - set_config('simulator', simulators, port=config_port) + set_config('simulator', simulators) # Set a bad port, which should cause a fail before actual mount creation. set_config('mount.serial.driver', 'foobar') with pytest.raises(error.MountNotFound): - create_mount_from_config(config_port=config_port) + create_mount_from_config() + reset_conf() diff --git a/src/panoptes/pocs/tests/test_mount_simulator.py b/src/panoptes/pocs/tests/test_mount_simulator.py index 32b49d087..0012ab140 100644 --- a/src/panoptes/pocs/tests/test_mount_simulator.py +++ b/src/panoptes/pocs/tests/test_mount_simulator.py @@ -12,8 +12,8 @@ @pytest.fixture -def location(dynamic_config_server, config_port): - loc = get_config('location', port=config_port) +def location(): + loc = get_config('location') return EarthLocation(lon=loc['longitude'], lat=loc['latitude'], height=loc['elevation']) @@ -22,14 +22,14 @@ def target(location): return altaz_to_radec(obstime='2016-08-13 21:03:01', location=location, alt=45, az=90) -def test_no_location(dynamic_config_server, config_port): +def test_no_location(): with pytest.raises(TypeError): - Mount(config_port=config_port) + Mount() @pytest.fixture(scope='function') -def mount(dynamic_config_server, config_port, location): - return Mount(location=location, config_port=config_port) +def mount(location): + return Mount(location=location) def test_connect(mount): @@ -83,22 +83,22 @@ def test_status(mount): assert 'mount_target_ra' in status2 -def test_update_location_no_init(dynamic_config_server, config_port, mount): - loc = get_config('location', port=config_port) +def test_update_location_no_init(mount): + loc = get_config('location') location2 = EarthLocation( lon=loc['longitude'], lat=loc['latitude'], height=loc['elevation'] - - 1000 * - u.meter) + 1000 * + u.meter) with pytest.raises(AssertionError): mount.location = location2 -def test_update_location(dynamic_config_server, config_port, mount): - loc = get_config('location', port=config_port) +def test_update_location(mount): + loc = get_config('location') mount.initialize() @@ -107,8 +107,8 @@ def test_update_location(dynamic_config_server, config_port, mount): lon=loc['longitude'], lat=loc['latitude'], height=loc['elevation'] - - 1000 * - u.meter) + 1000 * + u.meter) mount.location = location2 assert location1 != location2 diff --git a/src/panoptes/pocs/tests/test_observation.py b/src/panoptes/pocs/tests/test_observation.py index b40893215..3c3af7651 100644 --- a/src/panoptes/pocs/tests/test_observation.py +++ b/src/panoptes/pocs/tests/test_observation.py @@ -7,103 +7,103 @@ @pytest.fixture -def field(dynamic_config_server, config_port): - return Field('Test Observation', '20h00m43.7135s +22d42m39.0645s', config_port=config_port) +def field(): + return Field('Test Observation', '20h00m43.7135s +22d42m39.0645s') -def test_create_observation_no_field(dynamic_config_server, config_port): +def test_create_observation_no_field(): with pytest.raises(TypeError): - Observation(config_port=config_port) + Observation() -def test_create_observation_bad_field(dynamic_config_server, config_port): +def test_create_observation_bad_field(): with pytest.raises(AssertionError): - Observation('20h00m43.7135s +22d42m39.0645s', config_port=config_port) + Observation('20h00m43.7135s +22d42m39.0645s') -def test_create_observation_exptime_no_units(dynamic_config_server, config_port, field): +def test_create_observation_exptime_no_units(field): with pytest.raises(TypeError): - Observation(field, exptime=1.0, config_port=config_port) + Observation(field, exptime=1.0) -def test_create_observation_exptime_bad(dynamic_config_server, config_port, field): +def test_create_observation_exptime_bad(field): with pytest.raises(AssertionError): - Observation(field, exptime=0.0 * u.second, config_port=config_port) + Observation(field, exptime=0.0 * u.second) -def test_create_observation_exptime_minutes(dynamic_config_server, config_port, field): - obs = Observation(field, exptime=5.0 * u.minute, config_port=config_port) +def test_create_observation_exptime_minutes(field): + obs = Observation(field, exptime=5.0 * u.minute) assert obs.exptime == 300 * u.second -def test_bad_priority(dynamic_config_server, config_port, field): +def test_bad_priority(field): with pytest.raises(AssertionError): - Observation(field, priority=-1, config_port=config_port) + Observation(field, priority=-1) -def test_good_priority(dynamic_config_server, config_port, field): - obs = Observation(field, priority=5.0, config_port=config_port) +def test_good_priority(field): + obs = Observation(field, priority=5.0) assert obs.priority == 5.0 -def test_priority_str(dynamic_config_server, config_port, field): - obs = Observation(field, priority="5", config_port=config_port) +def test_priority_str(field): + obs = Observation(field, priority="5") assert obs.priority == 5.0 -def test_bad_min_set_combo(dynamic_config_server, config_port, field): +def test_bad_min_set_combo(field): with pytest.raises(AssertionError): - Observation(field, exp_set_size=7, config_port=config_port) + Observation(field, exp_set_size=7) with pytest.raises(AssertionError): - Observation(field, min_nexp=57, config_port=config_port) + Observation(field, min_nexp=57) -def test_small_sets(dynamic_config_server, config_port, field): +def test_small_sets(field): obs = Observation(field, exptime=1 * u.second, min_nexp=1, - exp_set_size=1, config_port=config_port) + exp_set_size=1) assert obs.minimum_duration == 1 * u.second assert obs.set_duration == 1 * u.second -def test_good_min_set_combo(dynamic_config_server, config_port, field): - obs = Observation(field, min_nexp=21, exp_set_size=3, config_port=config_port) +def test_good_min_set_combo(field): + obs = Observation(field, min_nexp=21, exp_set_size=3) assert isinstance(obs, Observation) -def test_default_min_duration(dynamic_config_server, config_port, field): - obs = Observation(field, config_port=config_port) +def test_default_min_duration(field): + obs = Observation(field) assert obs.minimum_duration == 7200 * u.second -def test_default_set_duration(dynamic_config_server, config_port, field): - obs = Observation(field, config_port=config_port) +def test_default_set_duration(field): + obs = Observation(field) assert obs.set_duration == 1200 * u.second -def test_print(dynamic_config_server, config_port, field): +def test_print(field): obs = Observation(field, exptime=17.5 * u.second, min_nexp=27, - exp_set_size=9, config_port=config_port) + exp_set_size=9) test_str = "Test Observation: 17.5 s exposures in blocks of 9, minimum 27, priority 100" assert str(obs) == test_str -def test_seq_time(dynamic_config_server, config_port, field): +def test_seq_time(field): obs = Observation(field, exptime=17.5 * u.second, min_nexp=27, - exp_set_size=9, config_port=config_port) + exp_set_size=9) assert obs.seq_time is None -def test_no_exposures(dynamic_config_server, config_port, field): +def test_no_exposures(field): obs = Observation(field, exptime=17.5 * u.second, min_nexp=27, - exp_set_size=9, config_port=config_port) + exp_set_size=9) assert obs.first_exposure is None assert obs.last_exposure is None assert obs.pointing_image is None -def test_last_exposure_and_reset(dynamic_config_server, config_port, field): +def test_last_exposure_and_reset(field): obs = Observation(field, exptime=17.5 * u.second, min_nexp=27, - exp_set_size=9, config_port=config_port) + exp_set_size=9) status = obs.status assert status['current_exp'] == obs.current_exp_num diff --git a/src/panoptes/pocs/tests/test_observatory.py b/src/panoptes/pocs/tests/test_observatory.py index acf74309e..c33f58f66 100644 --- a/src/panoptes/pocs/tests/test_observatory.py +++ b/src/panoptes/pocs/tests/test_observatory.py @@ -7,6 +7,7 @@ from panoptes.pocs import __version__ from panoptes.utils import error from panoptes.utils.config.client import set_config +from panoptes.utils.serializers import to_json from panoptes.pocs import hardware from panoptes.pocs.mount import AbstractMount @@ -21,26 +22,39 @@ from panoptes.pocs.scheduler import create_scheduler_from_config from panoptes.pocs.utils.location import create_location_from_config +import requests + +config_host = 'localhost' +config_port = 6563 +url = f'http://{config_host}:{config_port}/reset-config' + + +def reset_conf(): + response = requests.post(url, + data=to_json({'reset': True}), + headers={'Content-Type': 'application/json'} + ) + assert response.ok + @pytest.fixture(scope='function') -def cameras(dynamic_config_server, config_port): - return create_camera_simulator(config_port=config_port) +def cameras(): + return create_camera_simulator() @pytest.fixture(scope='function') -def mount(dynamic_config_server, config_port): +def mount(): return create_mount_simulator() @pytest.fixture -def observatory(dynamic_config_server, config_port, mount, cameras, images_dir): +def observatory(mount, cameras, images_dir): """Return a valid Observatory instance with a specific config.""" - site_details = create_location_from_config(config_port=config_port) - scheduler = create_scheduler_from_config(config_port=config_port, - observer=site_details['observer']) + site_details = create_location_from_config() + scheduler = create_scheduler_from_config(observer=site_details['observer']) - obs = Observatory(scheduler=scheduler, config_port=config_port) + obs = Observatory(scheduler=scheduler) obs.set_mount(mount) for cam_name, cam in cameras.items(): obs.add_camera(cam_name, cam) @@ -48,64 +62,63 @@ def observatory(dynamic_config_server, config_port, mount, cameras, images_dir): return obs -def test_camera_already_exists(dynamic_config_server, config_port, observatory, cameras): +def test_camera_already_exists(observatory, cameras): for cam_name, cam in cameras.items(): observatory.add_camera(cam_name, cam) -def test_remove_cameras(dynamic_config_server, config_port, observatory, cameras): +def test_remove_cameras(observatory, cameras): for cam_name, cam in cameras.items(): observatory.remove_camera(cam_name) -def test_bad_site(dynamic_config_server, config_port): - set_config('location', {}, port=config_port) +def test_bad_site(): + set_config('location', {}) with pytest.raises(error.PanError): - Observatory(config_port=config_port) + Observatory() + + reset_conf() -def test_cannot_observe(dynamic_config_server, config_port, caplog): - obs = Observatory(config_port=config_port) +def test_cannot_observe(caplog): + obs = Observatory() - site_details = create_location_from_config(config_port=config_port) + site_details = create_location_from_config() cameras = create_camera_simulator() assert obs.can_observe is False time.sleep(0.5) # log sink time log_record = caplog.records[-1] - assert log_record.message.endswith("not present, cannot observe") and log_record.levelname == "WARNING" - obs.scheduler = create_scheduler_from_config(observer=site_details['observer'], config_port=config_port) + assert log_record.message.endswith("not present") and log_record.levelname == "WARNING" + obs.scheduler = create_scheduler_from_config(observer=site_details['observer']) assert obs.can_observe is False time.sleep(0.5) # log sink time log_record = caplog.records[-1] - assert log_record.message.endswith("not present, cannot observe") and log_record.levelname == "WARNING" + assert log_record.message.endswith("not present") and log_record.levelname == "WARNING" for cam_name, cam in cameras.items(): obs.add_camera(cam_name, cam) assert obs.can_observe is False log_record = caplog.records[-1] time.sleep(0.5) # log sink time - assert log_record.message.endswith("not present, cannot observe") and log_record.levelname == "WARNING" + assert log_record.message.endswith("not present") and log_record.levelname == "WARNING" -def test_camera_wrong_type(dynamic_config_server, config_port): +def test_camera_wrong_type(): # Remove mount simulator - set_config('simulator', hardware.get_all_names(without='camera'), port=config_port) + set_config('simulator', hardware.get_all_names(without='camera')) with pytest.raises(AttributeError): - Observatory(cameras=[Time.now()], - config_port=config_port) + Observatory(cameras=[Time.now()]) with pytest.raises(AssertionError): - Observatory(cameras={'Cam00': Time.now()}, - config_port=config_port) + Observatory(cameras={'Cam00': Time.now()}) -def test_camera(dynamic_config_server, config_port): - cameras = create_camera_simulator(config_port=config_port) - obs = Observatory(cameras=cameras, - config_port=config_port) +def test_camera(): + cameras = create_camera_simulator() + obs = Observatory(cameras=cameras) assert obs.has_cameras @@ -118,15 +131,20 @@ def test_primary_camera_no_primary_camera(observatory): assert observatory.primary_camera is not None -def test_set_scheduler(dynamic_config_server, config_port, observatory, caplog): - site_details = create_location_from_config(config_port=config_port) - scheduler = create_scheduler_from_config( - observer=site_details['observer'], config_port=config_port) +def test_set_scheduler(observatory, caplog): + site_details = create_location_from_config() + scheduler = create_scheduler_from_config(observer=site_details['observer']) + + assert observatory.current_observation is None + observatory.set_scheduler(scheduler=None) assert observatory.scheduler is None + observatory.set_scheduler(scheduler=scheduler) + assert observatory.scheduler is not None - err_msg = 'Scheduler is not instance of BaseScheduler class, cannot add.' + err_msg = 'Scheduler is not an instance of .*BaseScheduler' + with pytest.raises(TypeError, match=err_msg): observatory.set_scheduler('scheduler') err_msg = ".*missing 1 required positional argument.*" @@ -134,20 +152,20 @@ def test_set_scheduler(dynamic_config_server, config_port, observatory, caplog): observatory.set_scheduler() -def test_set_dome(dynamic_config_server, config_port): +def test_set_dome(): set_config('dome', { 'brand': 'Simulacrum', 'driver': 'simulator', - }, port=config_port) - dome = create_dome_simulator(config_port=config_port) + }) + dome = create_dome_simulator() - obs = Observatory(dome=dome, config_port=config_port) + obs = Observatory(dome=dome) assert obs.has_dome is True obs.set_dome(dome=None) assert obs.has_dome is False obs.set_dome(dome=dome) assert obs.has_dome is True - err_msg = 'Dome is not instance of AbstractDome class, cannot add.' + err_msg = 'Dome is not an instance of .*AbstractDome' with pytest.raises(TypeError, match=err_msg): obs.set_dome('dome') err_msg = ".*missing 1 required positional argument.*" @@ -155,8 +173,8 @@ def test_set_dome(dynamic_config_server, config_port): obs.set_dome() -def test_set_mount(dynamic_config_server, config_port): - obs = Observatory(config_port=config_port) +def test_set_mount(): + obs = Observatory() assert obs.mount is None obs.set_mount(mount=None) @@ -166,12 +184,12 @@ def test_set_mount(dynamic_config_server, config_port): 'brand': 'Simulacrum', 'driver': 'simulator', 'model': 'simulator', - }, port=config_port) - mount = create_mount_from_config(config_port=config_port) + }) + mount = create_mount_from_config() obs.set_mount(mount=mount) assert isinstance(obs.mount, AbstractMount) is True - err_msg = 'Mount is not instance of AbstractMount class, cannot add.' + err_msg = 'Mount is not an instance of .*AbstractMount' with pytest.raises(TypeError, match=err_msg): obs.set_mount(mount='mount') err_msg = ".*missing 1 required positional argument.*" @@ -235,10 +253,11 @@ def test_standard_headers(observatory): observatory.scheduler.fields_file = None observatory.scheduler.fields_list = [ - {'name': 'HAT-P-20', - 'priority': '100', - 'position': '07h27m39.89s +24d20m14.7s', - }, + { + 'name': 'HAT-P-20', + 'priority': '100', + 'position': '07h27m39.89s +24d20m14.7s', + }, ] observatory.get_observation() @@ -254,7 +273,8 @@ def test_standard_headers(observatory): 'moon_fraction': 0.7880103086091879, 'moon_separation': 148.34401, 'observer': 'Generic PANOPTES Unit', - 'origin': 'Project PANOPTES'} + 'origin': 'Project PANOPTES' + } assert headers['airmass'] == pytest.approx(test_headers['airmass'], rel=1e-2) assert headers['ha_mnt'] == pytest.approx(test_headers['ha_mnt'], rel=1e-2) @@ -276,7 +296,7 @@ def test_sidereal_time(observatory): assert abs(st.value - 9.145547849536634) < 1e-4 -def test_get_observation(observatory): +def test_get_observation(observatory, caplog): os.environ['POCSTIME'] = '2016-08-13 15:00:00' observation = observatory.get_observation() assert isinstance(observation, Observation) @@ -376,21 +396,21 @@ def test_no_dome(observatory): assert observatory.close_dome() -def test_operate_dome(dynamic_config_server, config_port): +def test_operate_dome(): # Remove dome and night simulator - set_config('simulator', hardware.get_all_names(without=['dome', 'night']), port=config_port) + set_config('simulator', hardware.get_all_names(without=['dome', 'night'])) set_config('dome', { 'brand': 'Simulacrum', 'driver': 'simulator', - }, port=config_port) + }) set_config('dome', { 'brand': 'Simulacrum', 'driver': 'simulator', - }, port=config_port) - dome = create_dome_simulator(config_port=config_port) - observatory = Observatory(dome=dome, config_port=config_port) + }) + dome = create_dome_simulator() + observatory = Observatory(dome=dome) assert observatory.has_dome assert observatory.open_dome() diff --git a/src/panoptes/pocs/tests/test_pocs.py b/src/panoptes/pocs/tests/test_pocs.py index 4092447a2..a84b299e6 100644 --- a/src/panoptes/pocs/tests/test_pocs.py +++ b/src/panoptes/pocs/tests/test_pocs.py @@ -3,13 +3,16 @@ import time import pytest +import requests + +from astropy import units as u from panoptes.pocs import hardware from panoptes.pocs.core import POCS from panoptes.pocs.observatory import Observatory -from panoptes.utils import CountdownTimer from panoptes.utils.config.client import set_config +from panoptes.utils.serializers import to_json, to_yaml from panoptes.pocs.mount import create_mount_simulator from panoptes.pocs.camera import create_cameras_from_config @@ -17,32 +20,44 @@ from panoptes.pocs.scheduler import create_scheduler_from_config from panoptes.pocs.utils.location import create_location_from_config +config_host = 'localhost' +config_port = 6563 +url = f'http://{config_host}:{config_port}/reset-config' + + +def reset_conf(): + response = requests.post(url, + data=to_json({'reset': True}), + headers={'Content-Type': 'application/json'} + ) + assert response.ok + @pytest.fixture(scope='function') -def cameras(dynamic_config_server, config_port): - return create_cameras_from_config(config_port=config_port) +def cameras(): + return create_cameras_from_config() @pytest.fixture(scope='function') -def mount(dynamic_config_server, config_port): +def mount(): return create_mount_simulator() @pytest.fixture(scope='function') -def site_details(dynamic_config_server, config_port): - return create_location_from_config(config_port=config_port) +def site_details(): + return create_location_from_config() @pytest.fixture(scope='function') -def scheduler(dynamic_config_server, config_port, site_details): - return create_scheduler_from_config(config_port=config_port, observer=site_details['observer']) +def scheduler(site_details): + return create_scheduler_from_config(observer=site_details['observer']) @pytest.fixture(scope='function') -def observatory(dynamic_config_server, config_port, cameras, mount, site_details, scheduler): +def observatory(cameras, mount, site_details, scheduler): """Return a valid Observatory instance with a specific config.""" - obs = Observatory(scheduler=scheduler, config_port=config_port) + obs = Observatory(scheduler=scheduler, simulator=['power', 'weather']) for cam_name, cam in cameras.items(): obs.add_camera(cam_name, cam) @@ -52,26 +67,27 @@ def observatory(dynamic_config_server, config_port, cameras, mount, site_details @pytest.fixture(scope='function') -def dome(config_port): +def dome(): set_config('dome', { 'brand': 'Simulacrum', 'driver': 'simulator', - }, port=config_port) + }) - return create_dome_simulator(config_port=config_port) + return create_dome_simulator() @pytest.fixture(scope='function') -def pocs(dynamic_config_server, config_port, observatory): +def pocs(observatory): os.environ['POCSTIME'] = '2020-01-01 08:00:00' - pocs = POCS(observatory, run_once=True, config_port=config_port) + pocs = POCS(observatory, run_once=True, simulators=['power']) yield pocs pocs.power_down() + reset_conf() @pytest.fixture(scope='function') -def pocs_with_dome(dynamic_config_server, config_port, pocs, dome): +def pocs_with_dome(pocs, dome): # Add dome to config os.environ['POCSTIME'] = '2020-01-01 08:00:00' pocs.observatory.set_dome(dome) @@ -81,52 +97,38 @@ def pocs_with_dome(dynamic_config_server, config_port, pocs, dome): @pytest.fixture(scope='module') def valid_observation(): - return {'name': 'HIP 36850', - 'position': '113.65 deg +31.887 deg', - 'priority': '100', - 'exptime': 2, - 'min_nexp': 2, - 'exp_set_size': 2, - } - - -def test_bad_pandir_env(pocs): - pandir = os.getenv('PANDIR') - os.environ['PANDIR'] = '/foo/bar' - with pytest.raises(SystemExit): - POCS.check_environment() - os.environ['PANDIR'] = pandir - - -def test_bad_pocs_env(pocs): - pocs_dir = os.getenv('POCS') - os.environ['POCS'] = '/foo/bar' - with pytest.raises(SystemExit): - POCS.check_environment() - os.environ['POCS'] = pocs_dir - - -def test_make_log_dir(tmp_path, pocs): - log_dir = tmp_path / 'logs' - assert os.path.exists(log_dir) is False - - old_pandir = os.environ['PANDIR'] - os.environ['PANDIR'] = str(tmp_path.resolve()) - POCS.check_environment() - - assert os.path.exists(log_dir) is True - os.removedirs(log_dir) - - os.environ['PANDIR'] = old_pandir + return { + 'name': 'HIP 36850', + 'position': '113.65 deg +31.887 deg', + 'priority': '100', + 'exptime': 2, + 'min_nexp': 2, + 'exp_set_size': 2, + } + + +def test_observatory_cannot_observe(pocs): + scheduler = pocs.observatory.scheduler + pocs.observatory.scheduler = None + assert pocs.initialize() is False + pocs.observatory.scheduler = scheduler + assert pocs.initialize() + assert pocs.is_initialized + # Make sure we can do it twice. + assert pocs.initialize() + assert pocs.is_initialized -def test_simple_simulator(pocs): +def test_simple_simulator(pocs, caplog): assert isinstance(pocs, POCS) + pocs.set_config('simulator', 'all') assert pocs.is_initialized is not True - with pytest.raises(AssertionError): - pocs.run() + # Not initialized returns false and gives warning. + assert pocs.run() is False + log_record = caplog.records[-1] + assert log_record.message == 'POCS not initialized' and log_record.levelname == "WARNING" pocs.initialize() assert pocs.is_initialized @@ -143,30 +145,30 @@ def test_simple_simulator(pocs): assert pocs.is_safe() -def test_is_weather_and_dark_simulator(dynamic_config_server, config_port, pocs): +def test_is_weather_and_dark_simulator(pocs): pocs.initialize() # Night simulator - set_config('simulator', ['camera', 'mount', 'weather', 'night'], port=config_port) + pocs.set_config('simulator', 'all') os.environ['POCSTIME'] = '2020-01-01 08:00:00' # is dark assert pocs.is_dark() is True os.environ['POCSTIME'] = '2020-01-01 18:00:00' # is day assert pocs.is_dark() is True # No night simulator - set_config('simulator', ['camera', 'mount', 'weather'], port=config_port) + pocs.set_config('simulator', hardware.get_all_names(without=['night'])) os.environ['POCSTIME'] = '2020-01-01 08:00:00' # is dark assert pocs.is_dark() is True os.environ['POCSTIME'] = '2020-01-01 18:00:00' # is day assert pocs.is_dark() is False - set_config('simulator', ['camera', 'mount', 'weather', 'night'], port=config_port) + pocs.set_config('simulator', ['camera', 'mount', 'weather', 'night']) assert pocs.is_weather_safe() is True -def test_is_weather_safe_no_simulator(dynamic_config_server, config_port, pocs): +def test_is_weather_safe_no_simulator(pocs): pocs.initialize() - set_config('simulator', ['camera', 'mount', 'night'], port=config_port) + pocs.set_config('simulator', hardware.get_all_names(without=['weather'])) # Set a specific time os.environ['POCSTIME'] = '2020-01-01 18:00:00' @@ -180,7 +182,8 @@ def test_is_weather_safe_no_simulator(dynamic_config_server, config_port, pocs): assert pocs.is_weather_safe() is False -def test_unsafe_park(dynamic_config_server, config_port, pocs): +def test_unsafe_park(pocs): + pocs.set_config('simulator', 'all') pocs.initialize() assert pocs.is_initialized is True os.environ['POCSTIME'] = '2020-01-01 08:00:00' @@ -192,7 +195,7 @@ def test_unsafe_park(dynamic_config_server, config_port, pocs): # My time goes fast... os.environ['POCSTIME'] = '2020-01-01 18:00:00' - set_config('simulator', hardware.get_all_names(without=['night']), port=config_port) + pocs.set_config('simulator', hardware.get_all_names(without=['night'])) assert pocs.is_safe() is False @@ -204,12 +207,12 @@ def test_unsafe_park(dynamic_config_server, config_port, pocs): pocs.power_down() -def test_no_ac_power(dynamic_config_server, config_port, pocs): +def test_no_ac_power(pocs): # Simulator makes AC power safe assert pocs.has_ac_power() is True # Remove 'power' from simulator - set_config('simulator', hardware.get_all_names(without=['power']), port=config_port) + pocs.set_config('simulator', hardware.get_all_names(without=['power'])) pocs.initialize() @@ -265,9 +268,9 @@ def test_power_down_dome_while_running(pocs_with_dome): assert not pocs.observatory.dome.is_connected -def test_run_no_targets_and_exit(dynamic_config_server, config_port, pocs): - os.environ['POCSTIME'] = '2020-01-01 18:00:00' - set_config('simulator', hardware.get_all_names(), port=config_port) +def test_run_no_targets_and_exit(pocs): + os.environ['POCSTIME'] = '2020-01-01 19:00:00' + pocs.set_config('simulator', 'all') pocs.state = 'sleeping' @@ -278,24 +281,6 @@ def test_run_no_targets_and_exit(dynamic_config_server, config_port, pocs): assert pocs.state == 'sleeping' -def test_run_complete(dynamic_config_server, config_port, pocs, valid_observation): - os.environ['POCSTIME'] = '2020-01-01 08:00:00' - set_config('simulator', hardware.get_all_names(), port=config_port) - - pocs.state = 'sleeping' - pocs._do_states = True - - pocs.observatory.scheduler.clear_available_observations() - pocs.observatory.scheduler.add_observation(valid_observation) - - pocs.initialize() - assert pocs.is_initialized is True - - pocs.run(exit_when_done=True, run_once=True) - assert pocs.state == 'sleeping' - pocs.power_down() - - def test_pocs_park_to_ready_with_observations(pocs): # We don't want to run_once here pocs.run_once = False @@ -327,16 +312,21 @@ def test_pocs_park_to_ready_with_observations(pocs): def test_pocs_park_to_ready_without_observations(pocs): os.environ['POCSTIME'] = '2020-01-01 08:00:00' + pocs.logger.warning(f'Inserting safe weather reading') + pocs.db.insert_current('weather', {'safe': True}) assert pocs.is_safe() is True assert pocs.state == 'sleeping' pocs.next_state = 'ready' assert pocs.initialize() + pocs.logger.warning(f'Moving to ready') assert pocs.goto_next_state() assert pocs.state == 'ready' + pocs.logger.warning(f'Moving to scheduling') assert pocs.goto_next_state() assert pocs.observatory.current_observation is not None pocs.next_state = 'parking' + pocs.logger.warning(f'Moving to parking') assert pocs.goto_next_state() assert pocs.state == 'parking' assert pocs.observatory.current_observation is None @@ -356,7 +346,6 @@ def test_pocs_park_to_ready_without_observations(pocs): def test_run_wait_until_safe(observatory, valid_observation, - config_port, ): os.environ['POCSTIME'] = '2020-01-01 08:00:00' @@ -365,22 +354,24 @@ def test_run_wait_until_safe(observatory, observatory.logger.info('start_pocs ENTER') # Remove weather simulator, else it would always be safe. - set_config('simulator', hardware.get_all_names(without=['weather']), port=config_port) + observatory.set_config('simulator', hardware.get_all_names(without=['weather'])) - pocs = POCS(observatory, config_port=config_port) + pocs = POCS(observatory) pocs.set_config('wait_delay', 5) # Check safety every 5 seconds. pocs.observatory.scheduler.clear_available_observations() pocs.observatory.scheduler.add_observation(valid_observation) + assert pocs.connected is True + assert pocs.is_initialized is False pocs.initialize() pocs.logger.info('Starting observatory run') # Weather is bad and unit is is connected but not set. assert pocs.is_weather_safe() is False + assert pocs.is_initialized assert pocs.connected assert pocs.do_states - assert pocs.is_initialized assert pocs.next_state is None pocs.set_config('wait_delay', 1) @@ -414,7 +405,8 @@ def start_pocs(): assert pocs.is_safe() is True while pocs.next_state != 'slewing': - pocs.logger.warning(f'Waiting to get to scheduling state. Currently next_state={pocs.next_state}') + pocs.logger.warning( + f'Waiting to get to scheduling state. Currently next_state={pocs.next_state}') time.sleep(1) pocs.logger.warning(f'Stopping states via pocs.DO_STATES') @@ -426,17 +418,16 @@ def start_pocs(): assert pocs_thread.is_alive() is False -def test_run_power_down_interrupt(config_port, - observatory, +def test_run_power_down_interrupt(observatory, valid_observation, ): os.environ['POCSTIME'] = '2020-01-01 08:00:00' observatory.logger.info('start_pocs ENTER') # Remove weather simulator, else it would always be safe. - set_config('simulator', hardware.get_all_names(), port=config_port) + observatory.set_config('simulator', 'all') - pocs = POCS(observatory, config_port=config_port) + pocs = POCS(observatory) pocs.set_config('wait_delay', 5) # Check safety every 5 seconds. pocs.observatory.scheduler.clear_available_observations() @@ -461,7 +452,8 @@ def start_pocs(): pocs_thread.start() while pocs.next_state != 'scheduling': - pocs.logger.debug(f'Waiting to get to scheduling state. Currently next_state={pocs.next_state}') + pocs.logger.debug( + f'Waiting to get to scheduling state. Currently next_state={pocs.next_state}') time.sleep(1) pocs.logger.warning(f'Stopping states via pocs.DO_STATES') @@ -471,3 +463,41 @@ def start_pocs(): pocs_thread.join(timeout=300) assert pocs_thread.is_alive() is False + + +def test_custom_state_file(observatory, temp_file): + state_table = POCS.load_state_table() + assert isinstance(state_table, dict) + + with open(temp_file, 'w') as f: + f.write(to_yaml(state_table)) + + file_path = os.path.abspath(temp_file) + + pocs = POCS(observatory, state_machine_file=file_path, run_once=True, simulators=['power']) + pocs.initialize() + pocs.power_down() + reset_conf() + + +def test_free_space(pocs, caplog): + assert pocs.has_free_space() + + assert pocs.has_free_space(required_space=999 * u.terabyte) is False + assert 'No disk space' in caplog.records[-1].message + assert caplog.records[-1].levelname == 'ERROR' + + +def test_run_complete(pocs, valid_observation): + os.environ['POCSTIME'] = '2020-01-01 08:00:00' + pocs.set_config('simulator', 'all') + + pocs.observatory.scheduler.clear_available_observations() + pocs.observatory.scheduler.add_observation(valid_observation) + + pocs.initialize() + assert pocs.is_initialized is True + + pocs.run(exit_when_done=True, run_once=True) + assert pocs.state == 'sleeping' + pocs.power_down() diff --git a/src/panoptes/pocs/tests/test_scheduler.py b/src/panoptes/pocs/tests/test_scheduler.py index 34e56969b..7c7e9cb51 100644 --- a/src/panoptes/pocs/tests/test_scheduler.py +++ b/src/panoptes/pocs/tests/test_scheduler.py @@ -1,32 +1,51 @@ import pytest +import requests from panoptes.utils import error from panoptes.utils.config.client import set_config from panoptes.pocs.scheduler import create_scheduler_from_config from panoptes.pocs.scheduler import BaseScheduler from panoptes.pocs.utils.location import create_location_from_config +from panoptes.utils.serializers import to_json +config_host = 'localhost' +config_port = 6563 +url = f'http://{config_host}:{config_port}/reset-config' -def test_bad_scheduler_type(dynamic_config_server, config_port): - set_config('scheduler.type', 'foobar', port=config_port) - site_details = create_location_from_config(config_port=config_port) + +def reset_conf(): + response = requests.post(url, + data=to_json({'reset': True}), + headers={'Content-Type': 'application/json'} + ) + assert response.ok + + +def test_bad_scheduler_type(): + set_config('scheduler.type', 'foobar') + site_details = create_location_from_config() with pytest.raises(error.NotFound): - create_scheduler_from_config(observer=site_details['observer'], config_port=config_port) + create_scheduler_from_config(observer=site_details['observer']) + reset_conf() -def test_bad_scheduler_fields_file(dynamic_config_server, config_port): - set_config('scheduler.fields_file', 'foobar', port=config_port) - site_details = create_location_from_config(config_port=config_port) + +def test_bad_scheduler_fields_file(): + set_config('scheduler.fields_file', 'foobar') + site_details = create_location_from_config() with pytest.raises(error.NotFound): - create_scheduler_from_config(observer=site_details['observer'], config_port=config_port) + create_scheduler_from_config(observer=site_details['observer']) + + reset_conf() def test_no_observer(): assert isinstance(create_scheduler_from_config(observer=None), BaseScheduler) is True -def test_no_scheduler_in_config(dynamic_config_server, config_port): - set_config('scheduler', None, port=config_port) - site_details = create_location_from_config(config_port=config_port) +def test_no_scheduler_in_config(): + set_config('scheduler', None) + site_details = create_location_from_config() assert create_scheduler_from_config( - observer=site_details['observer'], config_port=config_port) is None + observer=site_details['observer']) is None + reset_conf() diff --git a/src/panoptes/pocs/tests/test_state_machine.py b/src/panoptes/pocs/tests/test_state_machine.py index e99ef69c1..52779e4b4 100644 --- a/src/panoptes/pocs/tests/test_state_machine.py +++ b/src/panoptes/pocs/tests/test_state_machine.py @@ -8,8 +8,8 @@ @pytest.fixture -def observatory(dynamic_config_server, config_port): - observatory = Observatory(simulator=['all'], config_port=config_port) +def observatory(): + observatory = Observatory(simulator=['all']) yield observatory @@ -19,8 +19,8 @@ def test_bad_state_machine_file(): POCS.load_state_table(state_table_name='foo') -def test_load_bad_state(dynamic_config_server, config_port, observatory): - pocs = POCS(observatory, config_port=config_port) +def test_load_bad_state(observatory): + pocs = POCS(observatory) with pytest.raises(error.InvalidConfig): pocs._load_state('foo') diff --git a/src/panoptes/pocs/utils/location.py b/src/panoptes/pocs/utils/location.py index 3201c2c0a..d596c0e35 100644 --- a/src/panoptes/pocs/utils/location.py +++ b/src/panoptes/pocs/utils/location.py @@ -9,26 +9,25 @@ logger = get_logger() -def create_location_from_config(config_port=6563): +def create_location_from_config(): """ - Sets up the site and location details. - - Note: - These items are read from the 'site' config directive and include: - * name - * latitude - * longitude - * timezone - * pressure - * elevation - * horizon + Sets up the site and location details. + + These items are read from the 'site' config directive and include: + * name + * latitude + * longitude + * timezone + * pressure + * elevation + * horizon """ logger.debug('Setting up site details') try: - config_site = get_config('location', default=None, port=config_port) + config_site = get_config('location', default=None) if config_site is None: raise error.PanError(msg='location information not found in config.') diff --git a/src/panoptes/pocs/utils/logger.py b/src/panoptes/pocs/utils/logger.py index d9408b37f..fe7bd0576 100644 --- a/src/panoptes/pocs/utils/logger.py +++ b/src/panoptes/pocs/utils/logger.py @@ -16,7 +16,7 @@ def __init__(self): # Level Time_UTC Time_Local dynamic_padding Message self.fmt = "{level:.1s} " \ "{time:MM-DD HH:mm:ss.ss!UTC}" \ - "({time:HH:mm:ss.ss}) " \ + " ({time:HH:mm:ss.ss}) " \ "| {name} {function}:{line}{extra[padding]} | " \ "{message}\n" self.handlers = dict() @@ -86,6 +86,7 @@ def get_logger(profile='panoptes', colorize=True, backtrace=True, diagnose=True, + catch=True, compression='gz', level=log_level) LOGGER_INFO.handlers['console'] = console_id diff --git a/tests/pocs_testing.yaml b/tests/pocs_testing.yaml index b59135be3..7d4f0685f 100644 --- a/tests/pocs_testing.yaml +++ b/tests/pocs_testing.yaml @@ -13,6 +13,10 @@ name: Generic PANOPTES Unit pan_id: PAN000 +pocs: + INITIALIZED: false + CONNECTED: false + INTERRUPTED: false location: name: Mauna Loa Observatory latitude: 19.54 deg @@ -34,11 +38,11 @@ directories: targets: POCS/resources/targets mounts: POCS/resources/mounts db: - name: panoptes + name: panoptes_testing type: file scheduler: type: dispatch - fields_file: simple.yaml + fields_file: simulator.yaml check_file: False mount: brand: ioptron From 4efb870715e9fa89fd8b0837a8d8c6a336c23499 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Sun, 14 Jun 2020 16:14:15 -1000 Subject: [PATCH 05/22] Update README.md --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5fe691d5e..ec11cefbb 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,13 @@ If you are running a PANOPTES unit then you will most likely want an entire PANO There is a bash shell script that will install an entire working POCS system on your computer. Some folks even report that it works on a Mac. -To test the script, open a terminal and enter: +The script will ask if you want to install in "developer" mode or not. If so, you should fork this repo, [panoptes-utils](https://github.com/panoptes/panoptes-utils), and [panoptes-tutorials](https://github.com/panoptes/panoptes-tutorials), +and then give your github username when prompted. + +The non-developer mode of the script is intended for PANOPTES units. + + +To install POCS via the script, open a terminal and enter: ```bash curl -L https://install.projectpanoptes.org | bash From 91d4507fbf126e83821b209376e43165190c252b Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Sun, 14 Jun 2020 18:18:50 -1000 Subject: [PATCH 06/22] Update README.md Fixing documentation badge. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ec11cefbb..dace6da9e 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Welcome to POCS documentation!


-[![GHA Status](https://img.shields.io/endpoint.svg?url=https%3A%2F%2Factions-badge.atrox.dev%2Fpanoptes%2FPOCS%2Fbadge%3Fref%3Ddevelop&style=flat)](https://actions-badge.atrox.dev/panoptes/POCS/goto?ref=develop) [![Travis Status](https://travis-ci.com/panoptes/POCS.svg?branch=develop)](https://travis-ci.com/panoptes/POCS) [![codecov](https://codecov.io/gh/panoptes/POCS/branch/develop/graph/badge.svg)](https://codecov.io/gh/panoptes/POCS) [![Documentation Status](https://readthedocs.org/projects/panoptes-pocs/badge/?version=latest)](https://panoptes-pocs.readthedocs.io/en/latest/?badge=latest) [![PyPI version](https://badge.fury.io/py/panoptes-pocs.svg)](https://badge.fury.io/py/panoptes-pocs) +[![GHA Status](https://img.shields.io/endpoint.svg?url=https%3A%2F%2Factions-badge.atrox.dev%2Fpanoptes%2FPOCS%2Fbadge%3Fref%3Ddevelop&style=flat)](https://actions-badge.atrox.dev/panoptes/POCS/goto?ref=develop) [![Travis Status](https://travis-ci.com/panoptes/POCS.svg?branch=develop)](https://travis-ci.com/panoptes/POCS) [![codecov](https://codecov.io/gh/panoptes/POCS/branch/develop/graph/badge.svg)](https://codecov.io/gh/panoptes/POCS) [![Documentation Status](https://readthedocs.org/projects/pocs/badge/?version=latest)](https://pocs.readthedocs.io/en/latest/?badge=latest) [![PyPI version](https://badge.fury.io/py/panoptes-pocs.svg)](https://badge.fury.io/py/panoptes-pocs) # Project PANOPTES From 16c90f95c3b7ba954b79c86a83f33b25e114b258 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Wed, 17 Jun 2020 20:32:12 -1000 Subject: [PATCH 07/22] Logger and other minor improvements for developers (#977) * Mount simulator better name and stringify. * Global db. * Logger * Add a `console_log_level` and `stderr_log_level`. The former is written to the log file in $PANLOG and is meant to be `tail`ed in the console. The `stderr_log_level` is what would be displayed, e.g. in a jupyter notebook. * The `panoptes` logger is enabled inside `get_logger`. --- CHANGELOG.rst | 3 ++ src/panoptes/pocs/base.py | 9 +++- src/panoptes/pocs/camera/camera.py | 3 +- src/panoptes/pocs/mount/__init__.py | 6 +-- src/panoptes/pocs/mount/mount.py | 11 ++++- src/panoptes/pocs/observatory.py | 4 +- src/panoptes/pocs/tests/utils/test_logger.py | 5 +-- src/panoptes/pocs/utils/logger.py | 43 ++++++++++++++++---- 8 files changed, 64 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e5e022ef9..a8bbc517b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -49,6 +49,9 @@ Changed * Simplification of the `run` method and the various predicates used to control it. Now just use the computed `keep_running`. * Adding some action flags to the `pocs.yaml` file. * Remove `POCS.check_environment` class method. + * Add a `console_log_level` and `stderr_log_level`. The former is written to the log file in `$PANLOG` and is meant to be tailed in the console. The `stderr_log_level` is what would be displayed, e.g. in a jupyter notebook. (#977) + * Mount simulator better name and stringify. (#977) + * Global db object for `PanBase` (#977) * Camera simulator cleanup. (#974) * Scheduler (#974) diff --git a/src/panoptes/pocs/base.py b/src/panoptes/pocs/base.py index b6e2308c3..621428e35 100644 --- a/src/panoptes/pocs/base.py +++ b/src/panoptes/pocs/base.py @@ -6,6 +6,9 @@ from panoptes.pocs.utils.logger import get_logger from panoptes.pocs import hardware +# Global database. +PAN_DB_OBJ = None + class PanBase(object): """ Base class for other classes within the PANOPTES ecosystem @@ -24,7 +27,11 @@ def __init__(self, config_port='6563', *args, **kwargs): db_type = kwargs.get('db_type', self.get_config('db.type', default='file')) db_name = kwargs.get('db_name', self.get_config('db.name', default='panoptes')) - self.db = PanDB(db_type=db_type, db_name=db_name) + global PAN_DB_OBJ + if PAN_DB_OBJ is None: + PAN_DB_OBJ = PanDB(db_type=db_type, db_name=db_name) + + self.db = PAN_DB_OBJ def get_config(self, *args, **kwargs): """Thin-wrapper around client based get_config that sets default port. diff --git a/src/panoptes/pocs/camera/camera.py b/src/panoptes/pocs/camera/camera.py index b802fac04..5d418f5df 100644 --- a/src/panoptes/pocs/camera/camera.py +++ b/src/panoptes/pocs/camera/camera.py @@ -901,7 +901,8 @@ def __str__(self): else: s += f" & {subcomponent.name}" sub_count += 1 - except Exception: + except Exception as e: + self.logger.warning(f'Unable to stringify camera: {e=}') s = str(self.__class__) return s diff --git a/src/panoptes/pocs/mount/__init__.py b/src/panoptes/pocs/mount/__init__.py index 54607b40c..2da0f085b 100644 --- a/src/panoptes/pocs/mount/__init__.py +++ b/src/panoptes/pocs/mount/__init__.py @@ -112,10 +112,10 @@ def create_mount_simulator(mount_info=None, current_simulators.remove('mount') mount_config = mount_info or { - 'model': 'simulator', + 'model': 'Mount Simulator', 'driver': 'simulator', 'serial': { - 'port': 'simulator' + 'port': '/dev/FAKE' } } @@ -132,6 +132,6 @@ def create_mount_simulator(mount_info=None, mount = module.Mount(earth_location, *args, **kwargs) - logger.success(f"{mount_config['driver']} mount created") + logger.success(f"{mount_config['driver'].title()} mount created") return mount diff --git a/src/panoptes/pocs/mount/mount.py b/src/panoptes/pocs/mount/mount.py index 437d6014c..cb0fc486d 100644 --- a/src/panoptes/pocs/mount/mount.py +++ b/src/panoptes/pocs/mount/mount.py @@ -1,4 +1,5 @@ import time +from contextlib import suppress from astropy import units as u from astropy.coordinates import EarthLocation @@ -43,7 +44,7 @@ def __init__(self, location, commands=None, *args, **kwargs): # Create an object for just the mount config items self.mount_config = self.get_config('mount') - self.logger.debug("Mount config: {}".format(self.mount_config)) + self.logger.debug(f"Mount config: {self.mount_config}") # setup commands for mount self.logger.debug("Setting up commands for mount") @@ -87,6 +88,14 @@ def __init__(self, location, commands=None, *args, **kwargs): self._current_coordinates = None self._park_coordinates = None + def __str__(self): + brand = self.mount_config.get('brand', '') + model = self.mount_config.get('model', '') + port = '' + with suppress(KeyError): + port = self.mount_config['serial']['port'] + return f'{brand} {model} ({port})' + def connect(self): # pragma: no cover raise NotImplementedError diff --git a/src/panoptes/pocs/observatory.py b/src/panoptes/pocs/observatory.py index 157cbff62..1f5bdb961 100644 --- a/src/panoptes/pocs/observatory.py +++ b/src/panoptes/pocs/observatory.py @@ -54,7 +54,7 @@ def __init__(self, cameras=None, scheduler=None, dome=None, mount=None, *args, * self._primary_camera = None if cameras: - self.logger.info(f'Adding the cameras to the observatory: {cameras}') + self.logger.info(f'Adding cameras to the observatory: {cameras}') for cam_name, camera in cameras.items(): self.add_camera(cam_name, camera) @@ -242,7 +242,7 @@ def _set_hardware(self, new_hardware, hw_type, hw_class): hw_attr = getattr(self, hw_type) if isinstance(new_hardware, hw_class): - self.logger.success(f'Adding {new_hardware=}') + self.logger.success(f'Adding {new_hardware}') setattr(self, hw_type, new_hardware) elif new_hardware is None: if hw_attr is not None: diff --git a/src/panoptes/pocs/tests/utils/test_logger.py b/src/panoptes/pocs/tests/utils/test_logger.py index cec6409ab..7a78d43ca 100644 --- a/src/panoptes/pocs/tests/utils/test_logger.py +++ b/src/panoptes/pocs/tests/utils/test_logger.py @@ -11,9 +11,8 @@ def profile(): def test_base_logger(caplog, profile, tmp_path): logger = get_logger(log_dir=str(tmp_path), - full_log_file=None, - profile=profile) + full_log_file=None) logger.debug('Hello') - time.sleep(0.5) + time.sleep(1) # Wait for log to make it there. assert caplog.records[-1].message == 'Hello' assert caplog.records[-1].levelname == 'DEBUG' diff --git a/src/panoptes/pocs/utils/logger.py b/src/panoptes/pocs/utils/logger.py index fe7bd0576..4732ad6b0 100644 --- a/src/panoptes/pocs/utils/logger.py +++ b/src/panoptes/pocs/utils/logger.py @@ -1,4 +1,7 @@ import os +import sys +from contextlib import suppress + from loguru import logger as loguru_logger @@ -7,7 +10,8 @@ class PanLogger: Also provides a `handlers` dictionary to track attached handlers by id. - See https://loguru.readthedocs.io/en/stable/resources/recipes.html#dynamically-formatting-messages-to-properly-align-values-with-padding + See https://loguru.readthedocs.io/en/stable/resources/recipes.html#dynamically-formatting + -messages-to-properly-align-values-with-padding """ @@ -32,11 +36,12 @@ def format(self, record): LOGGER_INFO = PanLogger() -def get_logger(profile='panoptes', - console_log_file='panoptes.log', +def get_logger(console_log_file='panoptes.log', full_log_file='panoptes_{time:YYYYMMDD!UTC}.log', log_dir=None, - log_level='DEBUG'): + console_log_level='DEBUG', + stderr_log_level='INFO', + ): """Creates a root logger for PANOPTES used by the PanBase object. Two log files are created, one suitable for viewing on the console (via `tail`) @@ -49,7 +54,6 @@ def get_logger(profile='panoptes', `$PANDIR/logs` if `$PANDIR` exists, otherwise defaults to `.`. Args: - profile (str, optional): The name of the logger to use, defaults to 'panoptes'. console_log_file (str|None, optional): Filename for the file that is suitable for tailing in a shell (i.e., read by humans). This file is rotated daily however the files are not retained. @@ -58,9 +62,11 @@ def get_logger(profile='panoptes', website. Defaults to `panoptes_{time:YYYYMMDD!UTC}.log.gz` with a daily rotation at 11:30am and a 7 day retention policy. If `None` then no file will be generated. log_dir (str|None, optional): The directory to place the log file, see note. - log_level (str, optional): Log level for console output, defaults to 'DEBUG'. + stderr_log_level (str, optional): The log level to show on stderr, default INFO. + console_log_level (str, optional): Log level for console file output, defaults to 'DEBUG'. Note that it should be a string that matches standard `logging` levels and - also includes `TRACE` (below `DEBUG`) and `SUCCESS` (above `INFO`). + also includes `TRACE` (below `DEBUG`) and `SUCCESS` (above `INFO`). Also note this + is not the stderr output, but the output to the file to be tailed. Returns: `loguru.logger`: A configured instance of the logger. @@ -74,6 +80,22 @@ def get_logger(profile='panoptes', log_dir = os.path.normpath(log_dir) os.makedirs(log_dir, exist_ok=True) + if 'stderr' not in LOGGER_INFO.handlers: + # Remove default and add in custom stderr. + with suppress(ValueError): + loguru_logger.remove(0) + + stderr_format = "{level:.1s} " \ + "{time:MM-DD HH:mm:ss.ss!UTC} " \ + "{message}" + + stderr_id = loguru_logger.add( + sys.stdout, + format=stderr_format, + level=stderr_log_level + ) + LOGGER_INFO.handlers['stderr'] = stderr_id + # Log file for tailing on the console. if 'console' not in LOGGER_INFO.handlers: console_log_path = os.path.normpath(os.path.join(log_dir, console_log_file)) @@ -88,7 +110,7 @@ def get_logger(profile='panoptes', diagnose=True, catch=True, compression='gz', - level=log_level) + level=console_log_level) LOGGER_INFO.handlers['console'] = console_id # Log file for ingesting into log file service. @@ -106,10 +128,13 @@ def get_logger(profile='panoptes', level='TRACE') LOGGER_INFO.handlers['archive'] = archive_id - # Customize colors + # Customize colors. loguru_logger.level('TRACE', color='') loguru_logger.level('DEBUG', color='') loguru_logger.level('INFO', color='') loguru_logger.level('SUCCESS', color='') + # Enable the logging. + loguru_logger.enable('panoptes') + return loguru_logger From c1984b4cb105c7c98119e19dd1cc2298114c4a00 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Sun, 5 Jul 2020 14:43:26 -1000 Subject: [PATCH 08/22] New docker base image (#978) * * Fix curl install script step. * Docker * Use buildx for an arm build. * Increase timeout for build. * Copy git folder into build context for now for version. * `latest` image is installed from a directory. * Remove `pendulum` to make `arm` build easier. * * Building arm images. * Remove `readline` as it is difficult to install on `arm` and will be remove soon (only used in the shells). * * Use the `latest` image for running tests. * * Use the python3 * * Ignore import errors. * * Small change (also want to kick tests again). * Docker changes * `latest` is again from pip. * `develop` is from local source and used for testing. * Cloudbuilding will only build `latest`. * Switch back to the root user. * Minor edits. --- .dockerignore | 7 ++++- .gcloudignore | 24 +++++++++++++++ .travis.yml | 14 +++++---- CHANGELOG.rst | 5 +++- docker/build-image.sh | 2 +- docker/cloudbuild.yaml | 48 ++++++++++++++++++++---------- docker/develop.Dockerfile | 50 ++++++++++++++++++++------------ scripts/install/install-pocs.sh | 7 +++-- scripts/testing/run-tests.sh | 20 ++++++------- setup.cfg | 2 -- src/panoptes/peas/sensors.py | 5 ++-- src/panoptes/pocs/observatory.py | 4 +-- 12 files changed, 126 insertions(+), 62 deletions(-) create mode 100644 .gcloudignore diff --git a/.dockerignore b/.dockerignore index d04f9f888..0421e4de3 100644 --- a/.dockerignore +++ b/.dockerignore @@ -22,4 +22,9 @@ notebooks examples docs build -coverage.xml \ No newline at end of file +coverage.xml + +# Build and docs folder/files +build/* +dist/* +sdist/* diff --git a/.gcloudignore b/.gcloudignore new file mode 100644 index 000000000..ef5031a05 --- /dev/null +++ b/.gcloudignore @@ -0,0 +1,24 @@ +.idea +.venv +venv + +.git +.github + +*.md +!README*.md + +logs/ +**/.ipynb_checkpoints +**/.pytest_cache +**/.eggs +**/*.pdf +**/*.log +**/*.egg-info +**/*.pyc +**/__pycache__ +notebooks +examples +docs +build +coverage.xml diff --git a/.travis.yml b/.travis.yml index f061e9f88..119b62fe6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,14 +6,16 @@ addons: packages: - docker-ce python: + # Doesn't matter because we run inside the docker container. - "3.6" services: -- docker + - docker before_install: -- sed -i s'/^\.git$/\!\.git/' .dockerignore -- docker build -t panoptes-pocs:develop -f ${TRAVIS_BUILD_DIR}/docker/develop.Dockerfile ${TRAVIS_BUILD_DIR} + # Make sure git goes to the build context. + - sed -i s'/^\.git$/\!\.git/' .dockerignore + - docker build -t panoptes-pocs:develop -f ${TRAVIS_BUILD_DIR}/docker/develop.Dockerfile ${TRAVIS_BUILD_DIR} install: true script: -- docker run -it - panoptes-pocs:develop - scripts/testing/run-tests.sh + - docker run -it + panoptes-pocs:develop + scripts/testing/run-tests.sh diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a8bbc517b..382c1531e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -12,11 +12,11 @@ adheres to `Semantic Versioning `__. Changed ~~~~~~~ -* Python moved to 3.8. (#974) * `panoptes-utils` to `0.2.20`. (#974) * Install script. (#974) * Env var file is sourced for zshrc and bashrc. + * Fix the clone of the repos in install script. (#978) * Development Environment (#974) @@ -32,6 +32,9 @@ Changed * Use new ``arduino-cli`` installer. * Add ``bin/panoptes-develop`` and ``bin/wait-for-it.sh`` to installed scripts. * Add ``docker/setup-local-environment.sh``, a convenience script for building local images. + * Python moved to 3.8. (#974) + * Docker images are now built with buildx to get an arm version running. (#978) + * Removing readline and pendulum dependencies. (#978) * Testing (#974) diff --git a/docker/build-image.sh b/docker/build-image.sh index e559eced3..bb0009b59 100755 --- a/docker/build-image.sh +++ b/docker/build-image.sh @@ -8,7 +8,7 @@ cd "${SOURCE_DIR}" echo "Building gcr.io/panoptes-exp/panoptes-pocs:${TAG}" gcloud builds submit \ - --timeout="1h" \ + --timeout="5h" \ --substitutions="_TAG=${TAG}" \ --config "${SOURCE_DIR}/docker/${BASE_CLOUD_FILE}" \ "${SOURCE_DIR}" diff --git a/docker/cloudbuild.yaml b/docker/cloudbuild.yaml index 0372db9ac..b8ecf6d4f 100644 --- a/docker/cloudbuild.yaml +++ b/docker/cloudbuild.yaml @@ -1,22 +1,40 @@ steps: + # Set up multiarch support - name: 'gcr.io/cloud-builders/docker' - id: 'amd64-build' + id: 'setup-buildx' + env: + - 'DOCKER_CLI_EXPERIMENTAL=enabled' args: - - 'build' - - '--build-arg' - - 'IMAGE_URL=gcr.io/panoptes-exp/panoptes-utils:${_TAG}' - - '-f' - - 'docker/${_TAG}.Dockerfile' - - '--tag' - - 'gcr.io/${PROJECT_ID}/panoptes-pocs:${_TAG}' - - '.' + - 'run' + - '--privileged' + - '--rm' + - 'docker/binfmt:a7996909642ee92942dcd6cff44b9b95f08dad64' + waitFor: ['-'] + # Build builder - name: 'gcr.io/cloud-builders/docker' - id: 'amd64-push' + id: 'build-builder' + env: + - 'DOCKER_CLI_EXPERIMENTAL=enabled' args: - - 'push' - - 'gcr.io/${PROJECT_ID}/panoptes-pocs:${_TAG}' - waitFor: ['amd64-build'] + - 'buildx' + - 'create' + - '--use' + - '--driver=docker-container' + waitFor: ['setup-buildx'] -images: - - 'gcr.io/${PROJECT_ID}/panoptes-pocs:${_TAG}' + # Build + - name: 'gcr.io/cloud-builders/docker' + id: 'build-images' + env: + - 'DOCKER_CLI_EXPERIMENTAL=enabled' + args: + - 'buildx' + - 'build' + - '--push' + - '--platform=linux/amd64,linux/arm64' + - '-f=docker/latest.Dockerfile' + - '--tag=gcr.io/${PROJECT_ID}/panoptes-pocs:latest' + - '--cache-from=gcr.io/${PROJECT_ID}/panoptes-pocs:latest' + - '.' + waitFor: ['build-builder'] diff --git a/docker/develop.Dockerfile b/docker/develop.Dockerfile index 7afa710e0..c1057919a 100644 --- a/docker/develop.Dockerfile +++ b/docker/develop.Dockerfile @@ -1,38 +1,52 @@ -ARG IMAGE_URL=gcr.io/panoptes-exp/panoptes-pocs:latest -FROM ${IMAGE_URL} +ARG IMAGE_URL=gcr.io/panoptes-exp/panoptes-utils:latest +FROM ${IMAGE_URL} AS pocs-base -LABEL description="Installs the local folder in develop mode (i.e. pip install .e). \ -Used for running the tests and as a base for the for developer-env image." +LABEL description="Installs the panoptes-pocs module in local editable \ +mode. This requires the entire git history to be present. Used for testing." LABEL maintainers="developers@projectpanoptes.org" LABEL repo="github.com/panoptes/POCS" -ARG pan_dir=/var/panoptes -ARG pocs_dir="${pan_dir}/POCS" +ARG pandir=/var/panoptes +ARG arduino_url="https://raw.githubusercontent.com/arduino/arduino-cli/master/install.sh" +ARG gphoto2_url="https://raw.githubusercontent.com/gonzalo/gphoto2-updater/master/gphoto2-updater.sh" -ENV DEBIAN_FRONTEND=noninteractive ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 ENV SHELL /bin/zsh - -ENV PANUSER=panoptes -ENV PANDIR $pan_dir -ENV POCS $pocs_dir +ENV PANDIR $pandir +ENV POCS ${PANDIR}/POCS ENV SOLVE_FIELD /usr/bin/solve-field +RUN apt-get update \ + && apt-get install --no-install-recommends --yes \ + gcc libncurses5-dev udev \ + # GPhoto2 + && wget $gphoto2_url \ + && chmod +x gphoto2-updater.sh \ + && /bin/bash gphoto2-updater.sh --stable \ + && rm gphoto2-updater.sh \ + # arduino-cli + && curl -fsSL $arduino_url | BINDIR="/usr/local/bin" sh + # panoptes-utils USER ${PANUSER} COPY --chown=panoptes:panoptes . "${PANDIR}/POCS/" RUN cd "${PANDIR}/POCS" && \ - pip install -U -e ".[testing,google]" + pip3 install -U -e ".[testing,google]" # Cleanup apt. USER root -RUN apt-get autoremove --purge -y && \ +RUN apt-get autoremove --purge -y \ + autoconf \ + automake \ + autopoint \ + build-essential \ + gcc \ + gettext \ + libtool \ + pkg-config && \ + apt-get autoremove --purge -y && \ apt-get -y clean && \ - rm -rf /var/lib/apt/lists/* && \ - chown -R "${PANUSER}:${PANUSER}" "${PANDIR}" && \ - chmod -R 777 /astrometry + rm -rf /var/lib/apt/lists/* WORKDIR ${POCS} - -# Entrypoint runs gosu with panoptes user. CMD ["/bin/zsh"] diff --git a/scripts/install/install-pocs.sh b/scripts/install/install-pocs.sh index a22c5a729..d65251219 100644 --- a/scripts/install/install-pocs.sh +++ b/scripts/install/install-pocs.sh @@ -10,7 +10,7 @@ usage() { # # This script is meant for quick & easy install via: # -# $ curl -L https://install.projectpanoptes.org | bash +# $ curl -fsSL https://install.projectpanoptes.org | bash # or # $ wget -O - https://install.projectpanoptes.org | bash # @@ -125,6 +125,7 @@ function make_directories { sudo mkdir -p "${PANDIR}" else echo "WARNING ${PANDIR} already exists. You can exit and specify an alternate directory with --pandir or continue." + echo "Would you like to continue with the existing directory?" select yn in "Yes" "No"; do case ${yn} in Yes ) echo "Proceeding with existing directory"; break;; @@ -195,10 +196,10 @@ function get_repos { for repo in "${repos[@]}"; do if [[ ! -d "${PANDIR}/${repo}" ]]; then cd "${PANDIR}" - echo "Cloning ${repo}" + echo "Cloning ${GITHUB_URL}/${repo}" # Just redirect the errors because otherwise looks like it hangs. # TODO handle errors if repo doesn't exist (e.g. bad github name). - git clone "https://github.com/${GITHUB_USER}/${repo}.git" >> "${LOGFILE}" 2>&1 + git clone "${GITHUB_URL}/${repo}.git" >> "${LOGFILE}" 2>&1 # Set panoptes as upstream cd "${repo}" diff --git a/scripts/testing/run-tests.sh b/scripts/testing/run-tests.sh index c8cb25ae3..776842590 100755 --- a/scripts/testing/run-tests.sh +++ b/scripts/testing/run-tests.sh @@ -1,24 +1,22 @@ -#!/usr/bin/env bash -set -e - -cd "${POCS}" -PYTEST_CMD="$(command -v pytest)" - -echo "Run test params: ${PYTEST_CMD} $@" +#!/bin/bash -e REPORT_FILE=${REPORT_FILE:-coverage.xml} # This assumes we are always running in a docker container. -export COVERAGE_PROCESS_START="/var/panoptes/panoptes-pocs/setup.cfg" +export COVERAGE_PROCESS_START="/var/panoptes/POCS/setup.cfg" + +coverage erase # Run coverage over the pytest suite. echo "Starting tests" +coverage run "$(command -v pytest-3)" -coverage erase -coverage run "${PYTEST_CMD}" - +echo "Combining coverage" coverage combine + +echo "Making XML coverage report at ${REPORT_FILE}" coverage xml -o "${REPORT_FILE}" + coverage report --show-missing exit 0 diff --git a/setup.cfg b/setup.cfg index e82d97014..5c3d4cf3a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -49,8 +49,6 @@ install_requires = panoptes-utils>=0.2.20 pyserial>=3.1.1 PyYaml - pendulum - readline requests responses scalpl diff --git a/src/panoptes/peas/sensors.py b/src/panoptes/peas/sensors.py index 85b6e27b4..6f6ef44e2 100644 --- a/src/panoptes/peas/sensors.py +++ b/src/panoptes/peas/sensors.py @@ -6,13 +6,14 @@ from panoptes.utils.config.client import get_config from panoptes.utils.database import PanDB -from panoptes.pocs.utils.logger import get_logger from panoptes.utils.rs232 import SerialData from panoptes.utils import error +from panoptes.pocs.utils.logger import get_logger + class ArduinoSerialMonitor(object): - """Monitors the serial lines and tries to parse any data recevied as JSON. + """Monitors the serial lines and tries to parse any data received as JSON. Checks for the `camera_box` and `computer_box` entries in the config and tries to connect. Values are updated in the database. diff --git a/src/panoptes/pocs/observatory.py b/src/panoptes/pocs/observatory.py index 1f5bdb961..97621e0d2 100644 --- a/src/panoptes/pocs/observatory.py +++ b/src/panoptes/pocs/observatory.py @@ -1,7 +1,7 @@ import os import subprocess from collections import OrderedDict -import pendulum +from datetime import datetime from astropy import units as u from astropy.coordinates import get_moon @@ -308,7 +308,7 @@ def status(self): status['observer'] = { 'siderealtime': str(self.sidereal_time), 'utctime': now, - 'localtime': pendulum.now(), + 'localtime': datetime.now(), 'local_evening_astro_time': self._evening_astro_time, 'local_morning_astro_time': self._morning_astro_time, 'local_sun_set_time': self._local_sunset, From 0b24db2aa223867a98b2334d8a712f87f63c2fa9 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Sun, 5 Jul 2020 18:01:55 -1000 Subject: [PATCH 09/22] New utils and db path (#979) * Bumping to `panoptes-utils 0.2.21` * * Allowing custom folder for metadata. Closes #957 * * Changelog. * * Install script * Bumping docker-compose version to `1.26.2` * Adding script version. * * Install script * Fix README instruction. * Changelog. --- CHANGELOG.rst | 5 +++++ README.md | 2 +- conf_files/pocs.yaml | 1 + scripts/install/install-pocs.sh | 6 +++--- setup.cfg | 2 +- src/panoptes/pocs/base.py | 3 ++- 6 files changed, 13 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 382c1531e..27f2d9b9b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -12,11 +12,14 @@ adheres to `Semantic Versioning `__. Changed ~~~~~~~ +* `panoptes-utils` to `0.2.21`. (#979) * `panoptes-utils` to `0.2.20`. (#974) * Install script. (#974) * Env var file is sourced for zshrc and bashrc. * Fix the clone of the repos in install script. (#978) + * Adding a date version to script. (#979) + * `docker-compose` version bumped to `1.26.2` (#979) * Development Environment (#974) @@ -55,6 +58,8 @@ Changed * Add a `console_log_level` and `stderr_log_level`. The former is written to the log file in `$PANLOG` and is meant to be tailed in the console. The `stderr_log_level` is what would be displayed, e.g. in a jupyter notebook. (#977) * Mount simulator better name and stringify. (#977) * Global db object for `PanBase` (#977) + * Allow for custom folder for metadata. (#979) + * Default changed to `metadata`. * Camera simulator cleanup. (#974) * Scheduler (#974) diff --git a/README.md b/README.md index dace6da9e..5983c0cb5 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ The non-developer mode of the script is intended for PANOPTES units. To install POCS via the script, open a terminal and enter: ```bash -curl -L https://install.projectpanoptes.org | bash +curl -fsSL https://install.projectpanoptes.org | bash ``` Or using `wget`: diff --git a/conf_files/pocs.yaml b/conf_files/pocs.yaml index 5c85fec59..e062a5ec4 100644 --- a/conf_files/pocs.yaml +++ b/conf_files/pocs.yaml @@ -37,6 +37,7 @@ directories: db: name: panoptes type: file + folder: metadata wait_delay: 180 # time in seconds before checking safety/etc while waiting. max_transition_attempts: 5 # number of transitions attempts. diff --git a/scripts/install/install-pocs.sh b/scripts/install/install-pocs.sh index d65251219..40320b1c1 100644 --- a/scripts/install/install-pocs.sh +++ b/scripts/install/install-pocs.sh @@ -5,6 +5,8 @@ usage() { echo -n "################################################## # Install POCS and friends. # +# Script Version: 2020-07-05 +# # This script is designed to install the PANOPTES Observatory # Control System (POCS) on a cleanly installed Ubuntu system. # @@ -27,7 +29,6 @@ usage() { # ${DOCKER_BASE}/panoptes-utils # ${DOCKER_BASE}/pocs # -# # The script will ask if it should be installed in "developer" mode or not. # # The regular install is for running units and will not create local (to the @@ -54,7 +55,6 @@ usage() { " } - DEVELOPER=${DEVELOPER:-false} PANUSER=${PANUSER:-$USER} PANDIR=${PANDIR:-/var/panoptes} @@ -63,7 +63,7 @@ OS="$(uname -s)" ARCH="$(uname -m)" ENV_FILE="${PANDIR}/env" -DOCKER_COMPOSE_VERSION="${DOCKER_COMPOSE_VERSION:-1.26.0}" +DOCKER_COMPOSE_VERSION="${DOCKER_COMPOSE_VERSION:-1.26.2}" DOCKER_COMPOSE_INSTALL="https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-${OS}-${ARCH}" DOCKER_BASE=${DOCKER_BASE:-"gcr.io/panoptes-exp"} diff --git a/setup.cfg b/setup.cfg index 5c3d4cf3a..887c22ba0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -46,7 +46,7 @@ install_requires = Flask matplotlib numpy - panoptes-utils>=0.2.20 + panoptes-utils>=0.2.21 pyserial>=3.1.1 PyYaml requests diff --git a/src/panoptes/pocs/base.py b/src/panoptes/pocs/base.py index 621428e35..84d7b6109 100644 --- a/src/panoptes/pocs/base.py +++ b/src/panoptes/pocs/base.py @@ -26,10 +26,11 @@ def __init__(self, config_port='6563', *args, **kwargs): # If the user requests a db_type then update runtime config db_type = kwargs.get('db_type', self.get_config('db.type', default='file')) db_name = kwargs.get('db_name', self.get_config('db.name', default='panoptes')) + db_folder = kwargs.get('db_folder', self.get_config('db.folder', default='json_store')) global PAN_DB_OBJ if PAN_DB_OBJ is None: - PAN_DB_OBJ = PanDB(db_type=db_type, db_name=db_name) + PAN_DB_OBJ = PanDB(db_type=db_type, db_name=db_name, storage_dir=db_folder) self.db = PAN_DB_OBJ From 86d1643b3e410777227852d404315b708113da26 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Tue, 7 Jul 2020 06:56:55 -1000 Subject: [PATCH 10/22] Minor docker setup fixes (#981) * Pass the git folder to the build context for now. * Properly pass the params for `panoptes-develop` --- .dockerignore | 2 +- .gcloudignore | 2 +- bin/panoptes-develop | 3 ++- docker/developer-env.Dockerfile | 2 +- docker/latest.Dockerfile | 2 +- docker/setup-local-environment.sh | 4 ++-- 6 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.dockerignore b/.dockerignore index 0421e4de3..f277c06c8 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,7 +3,7 @@ docs/* .venv venv -.git +!.git .github *.md diff --git a/.gcloudignore b/.gcloudignore index ef5031a05..d5f5d4964 100644 --- a/.gcloudignore +++ b/.gcloudignore @@ -2,7 +2,7 @@ .venv venv -.git +!.git .github *.md diff --git a/bin/panoptes-develop b/bin/panoptes-develop index b62212326..aea51f7e9 100755 --- a/bin/panoptes-develop +++ b/bin/panoptes-develop @@ -2,6 +2,7 @@ set -e SUBCMD=$1 +PARAMS=${@:2} export PANDIR=${PANDIR:-/var/panoptes} export IMAGE="${IMAGE:-panoptes-pocs}" @@ -9,7 +10,7 @@ export TAG="${TAG:-developer-env}" cd "${PANDIR}" -## Add the deamon option by default. +## Add the daemon option by default. if [[ "${SUBCMD}" == "up" ]]; then SUBCMD="up -d" export CONTAINER_NAME="pocs-developer-env" diff --git a/docker/developer-env.Dockerfile b/docker/developer-env.Dockerfile index 0a692d6ed..b3af9afef 100644 --- a/docker/developer-env.Dockerfile +++ b/docker/developer-env.Dockerfile @@ -38,7 +38,7 @@ USER $PANUSER COPY --chown=panoptes:panoptes . ${PANDIR}/POCS/ RUN cd ${PANDIR}/POCS && \ # Install everything! - pip install -e ".[google,developer,plotting,testing]" && \ + pip3 install -e ".[google,developer,plotting,testing]" && \ # Set some jupyterlab defaults. mkdir -p /home/panoptes/.jupyter && \ jupyter-lab --generate-config && \ diff --git a/docker/latest.Dockerfile b/docker/latest.Dockerfile index 3ae5e31ee..0d9575bfa 100644 --- a/docker/latest.Dockerfile +++ b/docker/latest.Dockerfile @@ -27,7 +27,7 @@ RUN apt-get update \ # arduino-cli && curl -fsSL $arduino_url | BINDIR="/usr/local/bin" sh \ # Install the module. - && pip install -U "panoptes-pocs[google]" + && pip3 install -U "panoptes-pocs[google]" # Cleanup apt. RUN apt-get autoremove --purge -y \ diff --git a/docker/setup-local-environment.sh b/docker/setup-local-environment.sh index f88e901b8..a1d5af13b 100755 --- a/docker/setup-local-environment.sh +++ b/docker/setup-local-environment.sh @@ -15,7 +15,7 @@ echo "Building local panoptes-utils" cd "${POCS}" # In the local develop we need to pass git to the docker build context. -sed -i s'/^\.git$/\!\.git/' .dockerignore +#sed -i s'/^\.git$/\!\.git/' .dockerignore echo "Building local panoptes-pocs:latest from panoptes-utils:develop" docker build \ @@ -42,7 +42,7 @@ docker build \ "${POCS}" # Revert our .dockerignore changes. -sed -i s'/^!\.git$/\.git/' .dockerignore +#sed -i s'/^!\.git$/\.git/' .dockerignore docker system prune --force From 2bf10f1585fff7ee5c6d830ae0b1e532199387fc Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Tue, 7 Jul 2020 08:47:06 -1000 Subject: [PATCH 11/22] Minor Updates to logging and dome (#983) * Minor Dome Updates * Removing source of random testing coverage. * Fixing logging calls. --- src/panoptes/peas/sensors.py | 16 +++---- src/panoptes/pocs/dome/astrohaven.py | 19 ++++---- .../dome/protocol_astrohaven_simulator.py | 8 ++-- src/panoptes/pocs/focuser/birger.py | 30 ++++++------- src/panoptes/pocs/observatory.py | 7 ++- src/panoptes/pocs/sensors/arduino_io.py | 44 +++++++++---------- 6 files changed, 59 insertions(+), 65 deletions(-) diff --git a/src/panoptes/peas/sensors.py b/src/panoptes/peas/sensors.py index 6f6ef44e2..fe505b257 100644 --- a/src/panoptes/peas/sensors.py +++ b/src/panoptes/peas/sensors.py @@ -130,7 +130,7 @@ def capture(self, store_result=True): self.db.insert_current('power', data['power']) except Exception as e: - self.logger.warning('Exception while reading from sensor {}: {}', sensor_name, e) + self.logger.warning(f'Exception while reading from {sensor_name=}: {e!r}') return sensor_data @@ -165,14 +165,14 @@ def detect_board_on_port(port): Else returns None. """ logger = get_logger() - logger.debug('Attempting to connect to serial port: {}'.format(port)) + logger.debug(f'Attempting to connect to serial {port=}') try: serial_reader = SerialData(port=port, baudrate=9600, retry_limit=1, retry_delay=0) if not serial_reader.is_connected: serial_reader.connect() - logger.debug('Connected to {}', port) + logger.debug(f'Connected to {port=}') except Exception: - logger.warning('Could not connect to port: {}'.format(port)) + logger.warning(f'Could not connect to {port=}') return None try: reading = serial_reader.get_and_parse_reading(retry_limit=3) @@ -183,10 +183,10 @@ def detect_board_on_port(port): result = (data['name'], serial_reader) serial_reader = None return result - logger.warning('Unable to find board name in reading: {!r}', reading) + logger.warning(f'Unable to find board name in {reading=!r}') return None except Exception as e: - logger.error('Exception while auto-detecting port {!r}: {!r}'.format(port, e)) + logger.error(f'Exception while auto-detecting port {port=!r}: {e!r}') finally: if serial_reader: serial_reader.disconnect() @@ -196,11 +196,11 @@ def detect_board_on_port(port): if __name__ == '__main__': devices = find_arduino_devices() if devices: - print("Arduino devices: {}".format(", ".join(devices))) + print(f'Arduino devices: {",".join(devices)}') else: print("No Arduino devices found.") sys.exit(1) names_and_readers = auto_detect_arduino_devices() for (name, serial_reader) in names_and_readers: - print('Device {} has name {}'.format(serial_reader.name, name)) + print(f'Device {serial_reader.name} has {name=}') serial_reader.disconnect() diff --git a/src/panoptes/pocs/dome/astrohaven.py b/src/panoptes/pocs/dome/astrohaven.py index d7eeccd8a..0a2f0723d 100644 --- a/src/panoptes/pocs/dome/astrohaven.py +++ b/src/panoptes/pocs/dome/astrohaven.py @@ -78,7 +78,7 @@ def open(self): v = self._read_state_until_stable() if v == Protocol.BOTH_OPEN: return True - self.logger.warning('AstrohavenDome.open wrong final state: {!r}', v) + self.logger.warning(f'AstrohavenDome.open wrong final state: {v!r}') return False @property @@ -94,7 +94,7 @@ def close(self): v = self._read_state_until_stable() if v == Protocol.BOTH_CLOSED: return True - self.logger.warning('AstrohavenDome.close wrong final state: {!r}', v) + self.logger.warning(f'AstrohavenDome.close wrong final state: {v!r}') return False @property @@ -150,7 +150,7 @@ def _read_state_until_stable(self): c = chr(data[-1]) if c in Protocol.STABLE_STATES: return c - self.logger.debug('_read_state_until_stable not yet stable: {!r}', data) + self.logger.debug(f'_read_state_until_stable not yet stable: {data=!r}') if time.time() < end_by: continue pass @@ -188,26 +188,23 @@ def _full_move(self, send, target_feedback, feedback_countdown=1): c = chr(data[-1]) if c == target_feedback: feedback_countdown -= 1 - self.logger.debug('Got target_feedback, feedback_countdown={}', - feedback_countdown) + self.logger.debug(f'Got target_feedback, {feedback_countdown=}') if feedback_countdown <= 0: # Woot! Moved the dome and got the desired response. return True elif c == send: have_seen_send = True - elif not have_seen_send and c in Protocol.STABLE_STATES: + elif not have_seen_send and c in Protocol.STABLE_STATES: # pragma: no cover # At the start of looping, we may see the previous stable state until # we start seeing the echo of `send`. pass else: - self.logger.warning( - 'Unexpected value from dome! send={!r} expected={!r} actual={!r}', - send, target_feedback, data) + self.logger.warning(f'Unexpected value from dome! {send=!r} {target_feedback=!r} {data=!r}') if time.time() < end_by: continue self.logger.error( - 'Timed out moving the dome. Check for hardware or communications ' + - 'problem. send={!r} expected={!r} actual={!r}', send, target_feedback, data) + f'Timed out moving the dome. Check for hardware or communications problem. ' + f'{send=!r} {target_feedback=!r} {data=!r}') return False finally: self.serial.ser.timeout = saved_timeout diff --git a/src/panoptes/pocs/dome/protocol_astrohaven_simulator.py b/src/panoptes/pocs/dome/protocol_astrohaven_simulator.py index f62b95177..23f41f83e 100644 --- a/src/panoptes/pocs/dome/protocol_astrohaven_simulator.py +++ b/src/panoptes/pocs/dome/protocol_astrohaven_simulator.py @@ -139,7 +139,7 @@ def do_output(self): c = self.next_output_code if not c: c = self.compute_state() - self.logger.debug('AstrohavenPLCSimulator.compute_state -> {!r}', c) + self.logger.debug(f'AstrohavenPLCSimulator.compute_state -> {c!r}') self.next_output_code = None # We drop output if the queue is full. if not self.status_queue.full(): @@ -147,7 +147,7 @@ def do_output(self): self.next_output_time = datetime.datetime.now() + self.delta def handle_input(self, c): - self.logger.debug('AstrohavenPLCSimulator.handle_input {!r}', c) + self.logger.debug(f'AstrohavenPLCSimulator.handle_input {c!r}') (a_acted, a_resp) = self.shutter_a.handle_input(c) (b_acted, b_resp) = self.shutter_b.handle_input(c) # Use a_resp if a_acted or if there is no b_resp @@ -256,7 +256,7 @@ def read(self, size=1): if timeout_obj.expired(): break response = bytes(response) - self.logger.debug('AstrohavenSerialSimulator.read({}) -> {!r}', size, response) + self.logger.debug(f'AstrohavenSerialSimulator.read({size}) -> {response!r}') return response @property @@ -302,7 +302,7 @@ def write(self, data): if not isinstance(data, (bytes, bytearray)): raise ValueError("write takes bytes") data = bytes(data) # Make sure it can't change. - self.logger.info('AstrohavenSerialSimulator.write({!r})', data) + self.logger.info(f'AstrohavenSerialSimulator.write({data!r})') count = 0 timeout_obj = serialutil.Timeout(self.write_timeout) for b in data: diff --git a/src/panoptes/pocs/focuser/birger.py b/src/panoptes/pocs/focuser/birger.py index 4f4da4199..2140807bd 100644 --- a/src/panoptes/pocs/focuser/birger.py +++ b/src/panoptes/pocs/focuser/birger.py @@ -149,9 +149,9 @@ def __del__(self): self._serial_port.close() self.logger.debug('Closed serial port {}'.format(self._port)) -################################################################################################## -# Properties -################################################################################################## + ################################################################################################## + # Properties + ################################################################################################## @property def is_connected(self): @@ -211,9 +211,9 @@ def is_moving(self): """ True if the focuser is currently moving. """ return self._is_moving -################################################################################################## -# Public Methods -################################################################################################## + ################################################################################################## + # Public Methods + ################################################################################################## def connect(self, port): try: @@ -304,9 +304,9 @@ def move_by(self, increment): self.logger.debug("Moved by {} encoder units".format(moved_by)) return moved_by -################################################################################################## -# Private Methods -################################################################################################## + ################################################################################################## + # Private Methods + ################################################################################################## def _send_command(self, command, response_length=None, ignore_response=False): """ @@ -461,15 +461,15 @@ def _initialise_aperture(self): self.logger.debug('Initialising aperture motor') response = self._send_command('in', response_length=1)[0].rstrip() if response != 'DONE': - self.logger.error("{} got '{}', expected 'DONE'!".format(self, response)) + self.logger.error(f"{self} got {response=}, expected 'DONE'!") def _move_zero(self): response = self._send_command('mz', response_length=1)[0].rstrip() if response[:4] != 'DONE': - self.logger.error("{} got '{}', expected 'DONENNNNN,1'!".format(self, response)) + self.logger.error(f"{self} got {response=}, expected 'DONENNNNN,1'!") else: r = response[4:].rstrip() - self.logger.debug("Moved {} encoder units to close stop".format(r[:-2])) + self.logger.debug(f"Moved {r[:-2]} encoder units to close stop") return int(r[:-2]) def _zero_encoder(self): @@ -480,15 +480,15 @@ def _learn_focus_range(self): self.logger.debug('Learning absolute focus range') response = self._send_command('la', response_length=1)[0].rstrip() if response != 'DONE:LA': - self.logger.error("{} got '{}', expected 'DONE:LA'!".format(self, response)) + self.logger.error(f"{self} got {response=}, expected 'DONE:LA'!") def _move_inf(self): response = self._send_command('mi', response_length=1)[0].rstrip() if response[:4] != 'DONE': - self.logger.error("{} got '{}', expected 'DONENNNNN,1'!".format(self, response)) + self.logger.error(f"{self} got {response=}, expected 'DONENNNNN,1'!") else: r = response[4:].rstrip() - self.logger.debug("Moved {} encoder units to far stop".format(r[:-2])) + self.logger.debug(f"Moved {r[:-2]} encoder units to far stop") return int(r[:-2]) def _add_fits_keywords(self, header): diff --git a/src/panoptes/pocs/observatory.py b/src/panoptes/pocs/observatory.py index 97621e0d2..9c22c5ccd 100644 --- a/src/panoptes/pocs/observatory.py +++ b/src/panoptes/pocs/observatory.py @@ -395,18 +395,17 @@ def cleanup_observations(self, upload_images=None, make_timelapse=None, keep_jpg 'fields', observation.field.field_name ) - self.logger.debug('Searching directory: {}', observation_dir) + self.logger.debug(f'Searching directory: {observation_dir}') for cam_name, camera in self.cameras.items(): - self.logger.debug('Cleanup for camera {} [{}]'.format( - cam_name, camera.uid)) + self.logger.debug(f'Cleanup for camera {cam_name} [{camera.uid}]') seq_dir = os.path.join( observation_dir, camera.uid, seq_time ) - self.logger.info('Cleaning directory {}'.format(seq_dir)) + self.logger.info(f'Cleaning directory {seq_dir}') process_cmd = [ process_script_path, diff --git a/src/panoptes/pocs/sensors/arduino_io.py b/src/panoptes/pocs/sensors/arduino_io.py index a379d8dba..be4c8a2ef 100644 --- a/src/panoptes/pocs/sensors/arduino_io.py +++ b/src/panoptes/pocs/sensors/arduino_io.py @@ -52,7 +52,7 @@ def detect_board_on_port(port): attribute in the top-level object. Else returns None. """ logger = get_logger() - logger.debug('Attempting to connect to serial port: {}'.format(port)) + logger.debug(f'Attempting to connect to serial port: {port}') serial_reader = None try: # First open a connection to the device. @@ -60,9 +60,9 @@ def detect_board_on_port(port): serial_reader = open_serial_device(port) if not serial_reader.is_connected: serial_reader.connect() - logger.debug('Connected to {}', port) + logger.debug(f'Connected to {port=}') except Exception: - logger.warning('Could not connect to port: {}'.format(port)) + logger.warning(f'Could not connect to {port=}') return None try: reading = serial_reader.get_and_parse_reading(retry_limit=3) @@ -71,10 +71,10 @@ def detect_board_on_port(port): (ts, data) = reading if isinstance(data, dict) and 'name' in data and isinstance(data['name'], str): return data['name'] - logger.warning('Unable to find board name in reading: {}', reading) + logger.warning(f'Unable to find board name in {reading=}') return None except Exception as e: # pragma: no cover - logger.error('Exception while auto-detecting port {}: {}', port, e) + logger.error(f'Exception while auto-detecting {port=}: {e!r}') finally: if serial_reader: serial_reader.disconnect() @@ -141,7 +141,7 @@ def __init__(self, board, serial_data, db): # Using threading.Event rather than just a boolean field so that any thread # can get and set the stop_running property. self._stop_running = threading.Event() - self._logger.info('Created ArduinoIO instance for board {}', self.board) + self._logger.info(f'Created ArduinoIO instance for {self.board=}') @property def stop_running(self): @@ -153,7 +153,7 @@ def stop_running(self, value): self._stop_running.set() else: self._stop_running.clear() - self._logger.info('Updated ArduinoIO.stop_running to {!r}', self.stop_running) + self._logger.info(f'Updated ArduinoIO.stop_running to {self.stop_running!r}') def run(self): """Main loop for recording data and reading commands. @@ -180,13 +180,11 @@ def read_and_record(self): if not reading: # Consider adding an error counter. if not self._report_next_reading: - self._logger.warning( - 'Unable to read from {}. Will report when next successful read.', - self.port) + self._logger.warning(f'Unable to read from {self.port=}. Will report when next successful read.') self._report_next_reading = True return False if self._report_next_reading: - self._logger.info('Succeeded in reading from {}; got:\n{}', self.port, reading) + self._logger.info(f'Succeeded in reading from {self.port=}; got:\n{reading}') self._report_next_reading = False self.handle_reading(reading) return True @@ -205,7 +203,7 @@ def disconnect(self): if self._serial_data.is_connected: self._serial_data.disconnect() except Exception as e: - self._logger.error('Failed to disconnect from {} due to: {}', self.port, e) + self._logger.error(f'Failed to disconnect from {self.port=} due to: {e!r}') def reconnect(self): """Disconnect from and connect to the serial port. @@ -216,13 +214,13 @@ def reconnect(self): try: self.disconnect() except Exception: - self._logger.error('Unable to disconnect from {}', self.port) + self._logger.error(f'Unable to disconnect from {self.port=}') return False try: self.connect() return True except Exception: - self._logger.error('Unable to reconnect to {}', self.port) + self._logger.error(f'Unable to reconnect to {self.port=}') return False def get_reading(self): @@ -232,8 +230,8 @@ def get_reading(self): try: return self._serial_data.get_and_parse_reading(retry_limit=1) except serial.SerialException as e: - self._logger.error('Exception raised while reading from port {}', self.port) - self._logger.error('Exception: {}', "\n".join(traceback.format_exc())) + self._logger.error(f'Exception raised while reading from {self.port=}') + self._logger.error("\n".join(traceback.format_exc())) if self.reconnect(): return None raise e @@ -244,7 +242,7 @@ def handle_reading(self, reading): # instead of a string. Obviously it needs to be serialized eventually. timestamp, data = reading if data.get('name', self.board) != self.board: - msg = 'Board reports name {}, expected {}'.format(data['name'], self.board) + msg = f'Board reports {data["name"]}, expected {self.board}' self._logger.critical(msg) raise ArduinoDataError(msg) reading = dict(name=self.board, timestamp=timestamp, data=data) @@ -268,13 +266,13 @@ def handle_commands(self): topic, msg_obj = self._sub.receive_message(blocking=True, timeout_ms=0.05) if not topic: continue - self._logger.debug('Received a message for topic {}', topic) + self._logger.debug(f'Received a message for {topic=}') if topic.lower() == self._cmd_topic: try: self.handle_command(msg_obj) except Exception as e: - self._logger.error('Exception while handling command: {}', e) - self._logger.error('msg_obj: {}', msg_obj) + self._logger.error(f'Exception while handling command: {e!r}') + self._logger.error(f'{msg_obj=}') def handle_command(self, msg): """Handle one relay command. @@ -284,15 +282,15 @@ def handle_command(self, msg): exists on this device. """ if msg['command'] == 'shutdown': - self._logger.info('Received command to shutdown ArduinoIO for board {}', self.board) + self._logger.info(f'Received command to shutdown ArduinoIO for {self.board=}') self.stop_running = True elif msg['command'] == 'write_line': line = msg['line'].rstrip('\r\n') - self._logger.debug('Sending line to board {}: {}', self.board, line) + self._logger.debug(f'Sending line to {self.board=}: {line}') line = line + '\n' self.write(line) else: - self._logger.error('Ignoring command: {}', msg) + self._logger.error(f'Ignoring command: {msg}') def write(self, text): """Writes text (a string) to the port. From 82dafff2f77d6ce3aaa97e2d6054d80bec96765d Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Tue, 7 Jul 2020 09:34:38 -1000 Subject: [PATCH 12/22] * Allow for checking of space in different folders, including symlinks. Closes #919. Replaces #958. (#982) * Allow for checking of space in different folders, including symlinks. Closes #919. Replaces #958. (#982) --- src/panoptes/pocs/core.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/panoptes/pocs/core.py b/src/panoptes/pocs/core.py index a6bd7a4da..3776b283d 100644 --- a/src/panoptes/pocs/core.py +++ b/src/panoptes/pocs/core.py @@ -314,8 +314,12 @@ def is_safe(self, no_warning=False, horizon='observe'): # Check weather is_safe_values['good_weather'] = self.is_weather_safe() - # Hard-drive space - is_safe_values['free_space'] = self.has_free_space() + # Hard-drive space in root + is_safe_values['free_space_root'] = self.has_free_space('/') + + # Hard-drive space in images directory. + images_dir = self.get_config('directories.images') + is_safe_values['free_space_images'] = self.has_free_space(images_dir) safe = all(is_safe_values.values()) @@ -414,10 +418,12 @@ def is_weather_safe(self, stale=180): return is_safe - def has_free_space(self, required_space=0.25 * u.gigabyte, low_space_percent=1.5): + def has_free_space(self, directory=None, required_space=0.25 * u.gigabyte, low_space_percent=1.5): """Does hard drive have disk space (>= 0.5 GB). Args: + directory (str, optional): The path to check free space on, the default + `None` will check `$PANDIR`. required_space (u.gigabyte, optional): Amount of free space required for operation low_space_percent (float, optional): Give warning if space is less @@ -427,8 +433,9 @@ def has_free_space(self, required_space=0.25 * u.gigabyte, low_space_percent=1.5 Returns: bool: True if enough space """ + directory = directory or os.getenv('PANDIR') req_space = required_space.to(u.gigabyte) - self._free_space = get_free_space() + self._free_space = get_free_space(dir=directory) space_is_low = self._free_space.value <= (req_space.value * low_space_percent) @@ -436,9 +443,9 @@ def has_free_space(self, required_space=0.25 * u.gigabyte, low_space_percent=1.5 has_space = bool(self._free_space.value >= req_space.value) if not has_space: - self.logger.error(f'No disk space: Free {self._free_space:.02f}\t {req_space=:.02f}') + self.logger.error(f'No disk space for {directory=}: Free {self._free_space:.02f}\t {req_space=:.02f}') elif space_is_low: # pragma: no cover - self.logger.warning(f'Low disk space: Free {self._free_space:.02f}\t {req_space=:.02f}') + self.logger.warning(f'Low disk space for {directory=}: Free {self._free_space:.02f}\t {req_space=:.02f}') return has_space From cacd26e0b4aceee61e0f06255a974eee4a65b98a Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Wed, 8 Jul 2020 06:48:13 -1000 Subject: [PATCH 13/22] Update install-pocs.sh (#980) Merging before pushing newer PR. * Update install-pocs.sh Fixing how the environment variables are written to the file. They should only be written if the file does not already exist. Cleanup of the menu prompts and other code. * Update install-pocs.sh A better `select` prompt via `PS3` * Update scripts/install/install-pocs.sh Fixing case select option. Co-authored-by: Joshua Liberman <65671429+jlibermann@users.noreply.github.com> * Update scripts/install/install-pocs.sh Break from select. Co-authored-by: Joshua Liberman <65671429+jlibermann@users.noreply.github.com> * Update scripts/install/install-pocs.sh Break from select. Co-authored-by: Joshua Liberman <65671429+jlibermann@users.noreply.github.com> * Update install-pocs.sh Better one-line if test. Co-authored-by: Joshua Liberman <65671429+jlibermann@users.noreply.github.com> --- scripts/install/install-pocs.sh | 91 +++++++++++++++++++-------------- 1 file changed, 52 insertions(+), 39 deletions(-) diff --git a/scripts/install/install-pocs.sh b/scripts/install/install-pocs.sh index 40320b1c1..c27d77d52 100644 --- a/scripts/install/install-pocs.sh +++ b/scripts/install/install-pocs.sh @@ -5,7 +5,7 @@ usage() { echo -n "################################################## # Install POCS and friends. # -# Script Version: 2020-07-05 +# Script Version: 2020-07-06 # # This script is designed to install the PANOPTES Observatory # Control System (POCS) on a cleanly installed Ubuntu system. @@ -42,6 +42,11 @@ usage() { # # The script has been tested with a fresh install of Ubuntu 20.04 # but may work on other linux systems. +# +# Changes: +# * 2020-07-05 - Initial release of versioned script. +# * 2020-07-06 (wtgee) - Fix the writing of the env file. Cleanup. +# ############################################################# $ $(basename $0) [--developer] [--user panoptes] [--pandir /var/panoptes] @@ -55,6 +60,9 @@ usage() { " } +# Better select prompt. +PS3="Select: " + DEVELOPER=${DEVELOPER:-false} PANUSER=${PANUSER:-$USER} PANDIR=${PANDIR:-/var/panoptes} @@ -67,51 +75,61 @@ DOCKER_COMPOSE_VERSION="${DOCKER_COMPOSE_VERSION:-1.26.2}" DOCKER_COMPOSE_INSTALL="https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-${OS}-${ARCH}" DOCKER_BASE=${DOCKER_BASE:-"gcr.io/panoptes-exp"} - while [[ $# -gt 0 ]] do key="$1" case ${key} in --developer) - DEVELOPER=true - shift # past bool argument - ;; + DEVELOPER=true + shift # past bool argument + ;; -u|--user) - PANUSER="$2" - shift # past argument - shift # past value - ;; + PANUSER="$2" + shift # past argument + shift # past value + ;; -d|--pandir) - PANDIR="$2" - shift # past argument - shift # past value - ;; + PANDIR="$2" + shift # past argument + shift # past value + ;; -h|--help) - PANDIR="$2" - usage - exit 1 - ;; + PANDIR="$2" + usage + exit 1 + ;; esac done if ! ${DEVELOPER}; then - echo -n "Are you installing POCS as a developer? (for PANOPTES units, select No)" - select yn in "Yes" "No"; do - case ${yn} in - Yes ) echo "Enabling developer mode. Note that you will need your GitHub username to proceed"; DEVELOPER=true; break;; - No ) echo "Installing POCS in production mode"; break;; + echo "How would you like to install the unit?" + select mode in "Developer" "PANOPTES Unit"; do + case ${mode} in + Developer) + echo "Enabling developer mode. Note that you will need your GitHub username to proceed"; + DEVELOPER=true; + break + ;; + "PANOPTES Unit") + echo "Installing POCS in production mode"; + break + ;; esac done fi if "${DEVELOPER}"; then + echo "To install POCS as a developer make sure you have first forked the following repositories:" + echo " https://github.com/panoptes/POCS" + echo " https://github.com/panoptes/panoptes-utils" + echo " https://github.com/panoptes/panoptes-tutorials" + echo "" + while [[ -z "${GITHUB_USER}" ]]; do - read -p "Github User [NOTE: you must have a fork created already]: " GITHUB_USER + read -p "Github User: " GITHUB_USER done fi -echo "DEVELOPER=${DEVELOPER} PANDIR=${PANDIR} PANUSER=${PANUSER} GITHUB_USER=${GITHUB_USER}" - function command_exists { # https://gist.github.com/gubatron/1eb077a1c5fcf510e8e5 # this should be a very portable way of checking if something is on the path @@ -142,21 +160,19 @@ function make_directories { } function setup_env_vars { - echo "Writing environment variables to ${ENV_FILE}" - if [[ -f "${ENV_FILE}" ]]; then - echo "\n**** Added by install-pocs script ****\n" >> "${ENV_FILE}" - fi - - cat >> "${ENV_FILE}" <> "${ENV_FILE}" <> ~/.bashrc - [[ -f "$HOME/.zshrc" ]] && echo '. /var/panoptes/env' >> ~/.zshrc + # Source the files in the shell. + test -f "$HOME/.bashrc" && echo '. /var/panoptes/env' >> ~/.bashrc + test -f "$HOME/.zshrc" && echo '. /var/panoptes/env' >> ~/.zshrc fi } @@ -253,19 +269,16 @@ function get_or_build_images { function do_install { clear + echo "Installing PANOPTES software." if ${DEVELOPER}; then - echo "" echo "**** Developer Mode ****" - echo "" + echo "GITHUB_USER=${GITHUB_USER}" fi - echo "Installing PANOPTES software." echo "PANUSER: ${PANUSER}" echo "PANDIR: ${PANDIR}" echo "OS: ${OS}" echo "Logfile: ${LOGFILE}" - - echo "Creating directories in ${PANDIR}" make_directories From 6a744af76837ebb22cb8224d78cc4086039f3e59 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Thu, 9 Jul 2020 06:38:20 -1000 Subject: [PATCH 14/22] Update install script (again) (#984) * Update install-pocs.sh Fixing how the environment variables are written to the file. They should only be written if the file does not already exist. Cleanup of the menu prompts and other code. * Update install-pocs.sh A better `select` prompt via `PS3` * Update scripts/install/install-pocs.sh Fixing case select option. Co-authored-by: Joshua Liberman <65671429+jlibermann@users.noreply.github.com> * Update scripts/install/install-pocs.sh Break from select. Co-authored-by: Joshua Liberman <65671429+jlibermann@users.noreply.github.com> * Update scripts/install/install-pocs.sh Break from select. Co-authored-by: Joshua Liberman <65671429+jlibermann@users.noreply.github.com> * Update install-pocs.sh Better one-line if test. * Updating install command. * * Test for ssh access when installing as a developer. * * Changelog * * Sneaking in another change to make dome code consistent in the testing. Co-authored-by: Joshua Liberman <65671429+jlibermann@users.noreply.github.com> --- CHANGELOG.rst | 3 ++- README.md | 6 ++++-- scripts/install/install-pocs.sh | 30 ++++++++++++++++++++-------- src/panoptes/pocs/dome/astrohaven.py | 2 +- 4 files changed, 29 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 27f2d9b9b..f4123c3c0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -19,7 +19,8 @@ Changed * Env var file is sourced for zshrc and bashrc. * Fix the clone of the repos in install script. (#978) * Adding a date version to script. (#979) - * `docker-compose` version bumped to `1.26.2` (#979) + * `docker-compose` version bumped to `1.26.2`. (#979) + * Better testing for ssh access. (#984) * Development Environment (#974) diff --git a/README.md b/README.md index 5983c0cb5..d79ac3841 100644 --- a/README.md +++ b/README.md @@ -46,13 +46,15 @@ The non-developer mode of the script is intended for PANOPTES units. To install POCS via the script, open a terminal and enter: ```bash -curl -fsSL https://install.projectpanoptes.org | bash +curl -fsSL https://install.projectpanoptes.org > install-pocs.sh +bash install-pocs.sh ``` Or using `wget`: ```bash -wget -O - https://install.projectpanoptes.org | bash +wget -qO- https://install.projectpanoptes.org > install-pocs.sh +bash install-pocs.sh ``` diff --git a/scripts/install/install-pocs.sh b/scripts/install/install-pocs.sh index c27d77d52..13db18141 100644 --- a/scripts/install/install-pocs.sh +++ b/scripts/install/install-pocs.sh @@ -5,16 +5,18 @@ usage() { echo -n "################################################## # Install POCS and friends. # -# Script Version: 2020-07-06 +# Script Version: 2020-07-08 # # This script is designed to install the PANOPTES Observatory # Control System (POCS) on a cleanly installed Ubuntu system. # # This script is meant for quick & easy install via: # -# $ curl -fsSL https://install.projectpanoptes.org | bash +# $ curl -fsSL https://install.projectpanoptes.org > install-pocs.sh +# $ bash install-pocs.sh # or -# $ wget -O - https://install.projectpanoptes.org | bash +# $ wget -qO- https://install.projectpanoptes.org > install-pocs.sh +# $ bash install-pocs.sh # # The script will do the following: # @@ -46,6 +48,7 @@ usage() { # Changes: # * 2020-07-05 - Initial release of versioned script. # * 2020-07-06 (wtgee) - Fix the writing of the env file. Cleanup. +# * 2020-07-08 (wtgee) - Better test for ssh access for developer. # ############################################################# $ $(basename $0) [--developer] [--user panoptes] [--pandir /var/panoptes] @@ -200,16 +203,27 @@ function system_deps { function get_repos { PUBLIC_GITHUB_URL="https://github.com/panoptes" + REPOS=("POCS" "panoptes-utils") + if "${DEVELOPER}"; then echo "Using repositories from user: ${GITHUB_USER}" - declare -a repos=("POCS" "panoptes-utils" "panoptes-tutorials") - GITHUB_URL="git@github.com:${GITHUB_USER}" + + # Test for ssh access + if [[ $(ssh -T git@github.com 2>&1) =~ "success" ]]; then + GITHUB_URL="git@github.com:${GITHUB_USER}" + else + echo "No SSH key found, cloning via https. You may want to set up your ssh keys." + GITHUB_URL="https://github.com/${GITHUB_USER}" + fi + + # If a developer, also get the tutorials + REPOS+=("panoptes-tutorials") else - declare -a repos=("POCS" "panoptes-utils") GITHUB_URL="${PUBLIC_GITHUB_URL}" fi - for repo in "${repos[@]}"; do + echo "Cloning ${REPOS}" + for repo in "${REPOS[@]}"; do if [[ ! -d "${PANDIR}/${repo}" ]]; then cd "${PANDIR}" echo "Cloning ${GITHUB_URL}/${repo}" @@ -255,7 +269,7 @@ function get_or_build_images { if ${DEVELOPER}; then echo "Building local PANOPTES docker images." - cd "${POCS}" + cd "${PANDIR}/POCS" ./docker/setup-local-environment.sh else echo "Pulling PANOPTES docker images from Google Cloud Registry (GCR)." diff --git a/src/panoptes/pocs/dome/astrohaven.py b/src/panoptes/pocs/dome/astrohaven.py index 0a2f0723d..d756aac86 100644 --- a/src/panoptes/pocs/dome/astrohaven.py +++ b/src/panoptes/pocs/dome/astrohaven.py @@ -198,7 +198,7 @@ def _full_move(self, send, target_feedback, feedback_countdown=1): # At the start of looping, we may see the previous stable state until # we start seeing the echo of `send`. pass - else: + else: # pragma: no cover self.logger.warning(f'Unexpected value from dome! {send=!r} {target_feedback=!r} {data=!r}') if time.time() < end_by: continue From 1ec67a690b79714d5700d55ada76624ec5ef7ce8 Mon Sep 17 00:00:00 2001 From: Joshua Liberman <65671429+jlibermann@users.noreply.github.com> Date: Thu, 9 Jul 2020 12:39:00 -0400 Subject: [PATCH 15/22] Minor docs update (#985) Co-authored-by: jlibermann --- docker/setup-local-environment.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/setup-local-environment.sh b/docker/setup-local-environment.sh index a1d5af13b..9a102962a 100755 --- a/docker/setup-local-environment.sh +++ b/docker/setup-local-environment.sh @@ -52,9 +52,9 @@ cat < Date: Mon, 13 Jul 2020 17:15:00 -1000 Subject: [PATCH 16/22] Install Script Testing (#986) * Fixing conditional so script can proceed without restart. * Bump version on script. * Use linuxserver.io for docker-compose so we can get it on `arm64`. * Better checks on writing to the shell rc files. * Clean up bash file. * Default to `panoptes` for github user (it was already that way but in a bad bash variable kind of way). * * Changelog. --- CHANGELOG.rst | 5 +++++ scripts/install/install-pocs.sh | 34 +++++++++++++++++++++++---------- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f4123c3c0..34d355c11 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -21,6 +21,11 @@ Changed * Adding a date version to script. (#979) * `docker-compose` version bumped to `1.26.2`. (#979) * Better testing for ssh access. (#984) + * Using [linuxserver.io docker-compose](https://hub.docker.com/r/linuxserver/docker-compose) +so we also have `arm` version without work. (#986) + * Fixing conditional so script can proceed without restart. (#986) + * Generalizing install script in sections. (#986) + * Development Environment (#974) diff --git a/scripts/install/install-pocs.sh b/scripts/install/install-pocs.sh index 13db18141..6d443808c 100644 --- a/scripts/install/install-pocs.sh +++ b/scripts/install/install-pocs.sh @@ -5,7 +5,7 @@ usage() { echo -n "################################################## # Install POCS and friends. # -# Script Version: 2020-07-08 +# Script Version: 2020-07-09 # # This script is designed to install the PANOPTES Observatory # Control System (POCS) on a cleanly installed Ubuntu system. @@ -49,6 +49,8 @@ usage() { # * 2020-07-05 - Initial release of versioned script. # * 2020-07-06 (wtgee) - Fix the writing of the env file. Cleanup. # * 2020-07-08 (wtgee) - Better test for ssh access for developer. +# * 2020-07-09 (wtgee) - Fix conditional for writing shell rc files. Use 3rd +# party docker-compose (linuxserver.io) for arm. # ############################################################# $ $(basename $0) [--developer] [--user panoptes] [--pandir /var/panoptes] @@ -73,9 +75,12 @@ LOGFILE="${PANDIR}/install-pocs.log" OS="$(uname -s)" ARCH="$(uname -m)" ENV_FILE="${PANDIR}/env" +GITHUB_USER="panoptes" -DOCKER_COMPOSE_VERSION="${DOCKER_COMPOSE_VERSION:-1.26.2}" -DOCKER_COMPOSE_INSTALL="https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-${OS}-${ARCH}" +#DOCKER_COMPOSE_VERSION="${DOCKER_COMPOSE_VERSION:-1.26.2}" +# We are currently using this 3rd party source for docker-compose because the +# official version doesn't build for arm64 yet. wtgee 2020-07-09 +DOCKER_COMPOSE_INSTALL="https://raw.githubusercontent.com/linuxserver/docker-docker-compose/master/run.sh" DOCKER_BASE=${DOCKER_BASE:-"gcr.io/panoptes-exp"} while [[ $# -gt 0 ]] @@ -127,9 +132,9 @@ if "${DEVELOPER}"; then echo " https://github.com/panoptes/panoptes-utils" echo " https://github.com/panoptes/panoptes-tutorials" echo "" - + while [[ -z "${GITHUB_USER}" ]]; do - read -p "Github User: " GITHUB_USER + read -p "Github User [panoptes]: " GITHUB_USER done fi @@ -171,11 +176,21 @@ export PANUSER=${PANUSER} export PANDIR=${PANDIR} export POCS=${PANDIR}/POCS export PANLOG=${PANDIR}/logs +**** End install-pocs script **** EOF # Source the files in the shell. - test -f "$HOME/.bashrc" && echo '. /var/panoptes/env' >> ~/.bashrc - test -f "$HOME/.zshrc" && echo '. /var/panoptes/env' >> ~/.zshrc + SHELLS=(".bashrc" ".zshrc") + + for SHELL_RC in "${SHELLS[@]}"; do + SHELL_RC_PATH="$HOME/${SHELL_RC}" + if test -f "${SHELL_RC_PATH}"; then + # Check if we have already added the file. + if grep -xq ". ${PANDIR}/env" "${SHELL_RC_PATH}"; then + printf '\n. ${PANDIR}/env\n' >> "${SHELL_RC_PATH}" + fi + fi + done fi } @@ -202,24 +217,23 @@ function system_deps { function get_repos { PUBLIC_GITHUB_URL="https://github.com/panoptes" + GITHUB_URL="https://github.com/${GITHUB_USER}" REPOS=("POCS" "panoptes-utils") if "${DEVELOPER}"; then echo "Using repositories from user: ${GITHUB_USER}" + echo "Testing for ssh access to github.com" # Test for ssh access if [[ $(ssh -T git@github.com 2>&1) =~ "success" ]]; then GITHUB_URL="git@github.com:${GITHUB_USER}" else echo "No SSH key found, cloning via https. You may want to set up your ssh keys." - GITHUB_URL="https://github.com/${GITHUB_USER}" fi # If a developer, also get the tutorials REPOS+=("panoptes-tutorials") - else - GITHUB_URL="${PUBLIC_GITHUB_URL}" fi echo "Cloning ${REPOS}" From 9626944ae6200f21e065eac654f5542988baf17f Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Mon, 27 Jul 2020 09:14:57 -1000 Subject: [PATCH 17/22] Bump panoptes-utils (#992) * Fixes the usage of the `dir` built-in. --- setup.cfg | 4 ++-- src/panoptes/pocs/core.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 887c22ba0..aa4cfd514 100644 --- a/setup.cfg +++ b/setup.cfg @@ -46,7 +46,7 @@ install_requires = Flask matplotlib numpy - panoptes-utils>=0.2.21 + panoptes-utils>=0.2.22 pyserial>=3.1.1 PyYaml requests @@ -209,4 +209,4 @@ exclude_lines = if 0: if __name__ == .__main__.: -ignore_errors = True \ No newline at end of file +ignore_errors = True diff --git a/src/panoptes/pocs/core.py b/src/panoptes/pocs/core.py index 3776b283d..61e4d07e9 100644 --- a/src/panoptes/pocs/core.py +++ b/src/panoptes/pocs/core.py @@ -435,7 +435,7 @@ def has_free_space(self, directory=None, required_space=0.25 * u.gigabyte, low_s """ directory = directory or os.getenv('PANDIR') req_space = required_space.to(u.gigabyte) - self._free_space = get_free_space(dir=directory) + self._free_space = get_free_space(directory=directory) space_is_low = self._free_space.value <= (req_space.value * low_space_percent) From 1dbc2e4f477ef47ed5c48272a5a157ee1c5f6d5c Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Mon, 27 Jul 2020 09:36:59 -1000 Subject: [PATCH 18/22] Minor docs cleanup * Fix the https in the intersphinx. (#991) * Don't skip the init method. * Minor docs cleanup. --- docs/conf.py | 22 ++++++++++++++++------ docs/requirements.txt | 1 - src/panoptes/pocs/scheduler/observation.py | 4 ++-- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 0476a5711..69f6f58a2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -8,10 +8,10 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import os -import sys import inspect +import os import shutil +import sys __location__ = os.path.join(os.getcwd(), os.path.dirname( inspect.getfile(inspect.currentframe()))) @@ -271,17 +271,27 @@ # -- External mapping ------------------------------------------------------------ python_version = '.'.join(map(str, sys.version_info[0:2])) intersphinx_mapping = { - 'sphinx': ('http://www.sphinx-doc.org/en/stable', None), + 'sphinx': ('https://www.sphinx-doc.org/en/stable', None), 'python': ('https://docs.python.org/' + python_version, None), 'matplotlib': ('https://matplotlib.org', None), 'numpy': ('https://docs.scipy.org/doc/numpy', None), - 'sklearn': ('http://scikit-learn.org/stable', None), - 'pandas': ('http://pandas.pydata.org/pandas-docs/stable', None), + 'sklearn': ('https://scikit-learn.org/stable', None), + 'pandas': ('https://pandas.pydata.org/pandas-docs/stable', None), 'scipy': ('https://docs.scipy.org/doc/scipy/reference', None), - 'astropy': ('http://docs.astropy.org/en/stable/', None), + 'astropy': ('https://docs.astropy.org/en/stable/', None), 'astroplan': ('https://astroplan.readthedocs.io/en/latest/', None), 'panoptes.utils': ('https://panoptes-utils.readthedocs.io/en/latest/', None), } # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = True + + +def skip(app, what, name, obj, would_skip, options): + if name == "__init__": + return False + return would_skip + + +def setup(app): + app.connect("autodoc-skip-member", skip) diff --git a/docs/requirements.txt b/docs/requirements.txt index 0a70e145b..d03d5c662 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,2 @@ -r ../requirements.txt sphinx_rtd_theme - diff --git a/src/panoptes/pocs/scheduler/observation.py b/src/panoptes/pocs/scheduler/observation.py index ddc80f841..0baf68437 100644 --- a/src/panoptes/pocs/scheduler/observation.py +++ b/src/panoptes/pocs/scheduler/observation.py @@ -1,7 +1,7 @@ import os -from astropy import units as u from collections import OrderedDict +from astropy import units as u from panoptes.pocs.base import PanBase from panoptes.pocs.scheduler.field import Field @@ -11,7 +11,7 @@ class Observation(PanBase): @u.quantity_input(exptime=u.second) def __init__(self, field, exptime=120 * u.second, min_nexp=60, exp_set_size=10, priority=100, filter_name=None, *args, **kwargs): - """ An observation of a given `~pocs.scheduler.field.Field`. + """ An observation of a given `panoptes.pocs.scheduler.field.Field`. An observation consists of a minimum number of exposures (`min_nexp`) that must be taken at a set exposure time (`exptime`). These exposures come From c6c23c6fa3e1450bab649e6ed64e2cd3b1ef8bcd Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Mon, 27 Jul 2020 11:25:42 -1000 Subject: [PATCH 19/22] Update install-pocs.sh (#987) Thanks @jlibermann for the work on this! I'm merging this as-is because I've run through the install a number of times on the Pi and other hardware. --- * Update install-pocs.sh * Test for ssh access to given github user and if fail, exit program with instructions on how to set up ssh. * `panoptes-tutorials` always cloned. * Check for clone failures. * Update install-pocs.sh * Move env vars to top of script. * Cleanup old comments. * Update install-pocs.sh Slightly better help message. * Update scripts/install/install-pocs.sh Ah, right! Thanks! Co-authored-by: Joshua Liberman <65671429+jlibermann@users.noreply.github.com> * Update install-pocs.sh * Don't use `exit` because we want to source script instead of run it. * Switch to new docker group before pull. * Update install-pocs.sh Pull images as sudo otherwise it's annoying. :angry: * Update install-pocs.sh Better check for if line has been written to source file. * Update install-pocs.sh Double-quotes so we interpolate the rc sourcing. * Update install-pocs.sh Only clone repositories if installing as a developer * Update install-pocs.sh Fix comments in env file. * Update install-pocs.sh * Add cli option for unit install. * Cleanup. * Tidy the script. Co-authored-by: Joshua Liberman <65671429+jlibermann@users.noreply.github.com> --- scripts/install/install-pocs.sh | 423 ++++++++++++++++---------------- 1 file changed, 214 insertions(+), 209 deletions(-) diff --git a/scripts/install/install-pocs.sh b/scripts/install/install-pocs.sh index 6d443808c..7bb1e4636 100644 --- a/scripts/install/install-pocs.sh +++ b/scripts/install/install-pocs.sh @@ -5,7 +5,7 @@ usage() { echo -n "################################################## # Install POCS and friends. # -# Script Version: 2020-07-09 +# Script Version: 2020-07-27 # # This script is designed to install the PANOPTES Observatory # Control System (POCS) on a cleanly installed Ubuntu system. @@ -37,7 +37,7 @@ usage() { # host system) copies of the files. # # The "developer" mode will ask for a github username and will clone and -# fetch the repos. The `docker/setup-local-enviornment.sh` script will then +# fetch the repos. The $(docker/setup-local-enviornment.sh) script will then # be run to build the docker images locally. # # If not in "developer" mode, the docker images will be pulled from GCR. @@ -51,6 +51,7 @@ usage() { # * 2020-07-08 (wtgee) - Better test for ssh access for developer. # * 2020-07-09 (wtgee) - Fix conditional for writing shell rc files. Use 3rd # party docker-compose (linuxserver.io) for arm. +# * 2020-07-27 (wtgee) - Cleanup and consistency for Unit install. # ############################################################# $ $(basename $0) [--developer] [--user panoptes] [--pandir /var/panoptes] @@ -75,263 +76,267 @@ LOGFILE="${PANDIR}/install-pocs.log" OS="$(uname -s)" ARCH="$(uname -m)" ENV_FILE="${PANDIR}/env" + GITHUB_USER="panoptes" +GITHUB_URL="https://github.com/${GITHUB_USER}" + +PANOPTES_UPSTREAM_URL="https://github.com/panoptes" + +# Repositories to clone. +REPOS=("POCS" "panoptes-utils" "panoptes-tutorials") -#DOCKER_COMPOSE_VERSION="${DOCKER_COMPOSE_VERSION:-1.26.2}" -# We are currently using this 3rd party source for docker-compose because the -# official version doesn't build for arm64 yet. wtgee 2020-07-09 DOCKER_COMPOSE_INSTALL="https://raw.githubusercontent.com/linuxserver/docker-docker-compose/master/run.sh" DOCKER_BASE=${DOCKER_BASE:-"gcr.io/panoptes-exp"} -while [[ $# -gt 0 ]] -do -key="$1" -case ${key} in - --developer) - DEVELOPER=true - shift # past bool argument - ;; - -u|--user) - PANUSER="$2" - shift # past argument - shift # past value - ;; - -d|--pandir) - PANDIR="$2" - shift # past argument - shift # past value - ;; - -h|--help) - PANDIR="$2" - usage - exit 1 - ;; -esac +while [[ $# -gt 0 ]]; do + key="$1" + case ${key} in + --developer) + DEVELOPER=true + shift # past bool argument + ;; + --install-unit) + DEVELOPER=false + shift # past bool argument + ;; + -u | --user) + PANUSER="$2" + shift # past argument + shift # past value + ;; + -d | --pandir) + PANDIR="$2" + shift # past argument + shift # past value + ;; + -h | --help) + PANDIR="$2" + usage + return + ;; + esac done if ! ${DEVELOPER}; then - echo "How would you like to install the unit?" - select mode in "Developer" "PANOPTES Unit"; do - case ${mode} in - Developer) - echo "Enabling developer mode. Note that you will need your GitHub username to proceed"; - DEVELOPER=true; - break - ;; - "PANOPTES Unit") - echo "Installing POCS in production mode"; - break - ;; - esac - done + echo "How would you like to install the unit?" + select mode in "Developer" "PANOPTES Unit"; do + case ${mode} in + Developer) + echo "Enabling developer mode. Note that you will need your GitHub username to proceed." + DEVELOPER=true + break + ;; + "PANOPTES Unit") + echo "Installing POCS for a PANOPTES unit." + break + ;; + esac + done fi if "${DEVELOPER}"; then - echo "To install POCS as a developer make sure you have first forked the following repositories:" - echo " https://github.com/panoptes/POCS" - echo " https://github.com/panoptes/panoptes-utils" - echo " https://github.com/panoptes/panoptes-tutorials" - echo "" - - while [[ -z "${GITHUB_USER}" ]]; do - read -p "Github User [panoptes]: " GITHUB_USER - done + echo "To install POCS as a developer make sure you have first forked the following repositories:" + echo "" + echo " https://github.com/panoptes/POCS" + echo " https://github.com/panoptes/panoptes-utils" + echo " https://github.com/panoptes/panoptes-tutorials" + echo "" + echo "You will also need to have an ssh key set up on github.com." + echo "See https://docs.github.com/en/github/authenticating-to-github/connecting-to-github-with-ssh" + + read -p "Github User [panoptes]: " GITHUB_USER + + # If a different user, make sure we can access github as that user, otherwise exit. + if test "${GITHUB_USER}" != "panoptes"; then + echo "Testing github ssh access for user: ${GITHUB_USER}" + + # Test for ssh access + if [[ $(ssh -T git@github.com 2>&1) =~ "success" ]]; then + GITHUB_URL="git@github.com:${GITHUB_USER}" + else + echo "Can't ssh to github.com. Have you set up your ssh keys?" + echo "See https://docs.github.com/en/github/authenticating-to-github/connecting-to-github-with-ssh" + return + fi + fi fi -function command_exists { - # https://gist.github.com/gubatron/1eb077a1c5fcf510e8e5 - # this should be a very portable way of checking if something is on the path - # usage: "if command_exists foo; then echo it exists; fi" - type "$1" &> /dev/null +function command_exists() { + # https://gist.github.com/gubatron/1eb077a1c5fcf510e8e5 + # this should be a very portable way of checking if something is on the path + # usage: "if command_exists foo; then echo it exists; fi" + type "$1" &>/dev/null } -function make_directories { - if [[ ! -d "${PANDIR}" ]]; then - # Make directories and make PANUSER the owner. - sudo mkdir -p "${PANDIR}" - else - echo "WARNING ${PANDIR} already exists. You can exit and specify an alternate directory with --pandir or continue." - echo "Would you like to continue with the existing directory?" - select yn in "Yes" "No"; do - case ${yn} in - Yes ) echo "Proceeding with existing directory"; break;; - No ) echo "Exiting"; exit 1;; - esac - done - fi +function make_directories() { + if [[ ! -d "${PANDIR}" ]]; then + # Make directories and make PANUSER the owner. + sudo mkdir -p "${PANDIR}" + else + echo "Would you like to continue with the existing directory?" + select yn in "Yes" "No"; do + case ${yn} in + Yes) + echo "Proceeding with existing directory" + break + ;; + No) + echo "Exiting script" + return + ;; + esac + done + fi - sudo mkdir -p "${PANDIR}/logs" - sudo mkdir -p "${PANDIR}/images" - sudo mkdir -p "${PANDIR}/config_files" - sudo mkdir -p "${PANDIR}/.key" - sudo chown -R "${PANUSER}":"${PANUSER}" "${PANDIR}" + sudo mkdir -p "${PANDIR}/logs" + sudo mkdir -p "${PANDIR}/images" + sudo mkdir -p "${PANDIR}/config_files" + sudo mkdir -p "${PANDIR}/.key" + sudo chown -R "${PANUSER}":"${PANUSER}" "${PANDIR}" } -function setup_env_vars { - if [[ ! -f "${ENV_FILE}" ]]; then - echo "Writing environment variables to ${ENV_FILE}" - cat >> "${ENV_FILE}" <>"${ENV_FILE}" <> "${SHELL_RC_PATH}" - fi - fi - done - fi -} - -function system_deps { - if [[ "${OS}" = "Linux" ]]; then - sudo apt-get update >> "${LOGFILE}" 2>&1 - # TODO(wtgee) figure out why we needed openssh-server on the host. - sudo apt-get --yes install \ - wget curl git openssh-server ack jq httpie byobu \ - >> "${LOGFILE}" 2>&1 - elif [[ "${OS}" = "Darwin" ]]; then - sudo brew update | sudo tee -a "${LOGFILE}" - sudo brew install \ - wget curl git jq httpie \ - | sudo tee -a "${LOGFILE}" - fi + # Source the files in the shell. + SHELLS=(".bashrc" ".zshrc") - # Add an SSH key if one doesn't exist. - if [[ ! -f "${HOME}/.ssh/id_rsa" ]]; then - echo "Adding ssh key" - ssh-keygen -t rsa -N "" -f "${HOME}/.ssh/id_rsa"; - fi + for SHELL_RC in "${SHELLS[@]}"; do + SHELL_RC_PATH="$HOME/${SHELL_RC}" + if test -f "${SHELL_RC_PATH}"; then + # Check if we have already added the file. + if ! grep -qm 1 ". ${PANDIR}/env" "${SHELL_RC_PATH}"; then + printf "\n. ${PANDIR}/env\n" >>"${SHELL_RC_PATH}" + fi + fi + done + fi } -function get_repos { - PUBLIC_GITHUB_URL="https://github.com/panoptes" - GITHUB_URL="https://github.com/${GITHUB_USER}" - - REPOS=("POCS" "panoptes-utils") - - if "${DEVELOPER}"; then - echo "Using repositories from user: ${GITHUB_USER}" - echo "Testing for ssh access to github.com" - - # Test for ssh access - if [[ $(ssh -T git@github.com 2>&1) =~ "success" ]]; then - GITHUB_URL="git@github.com:${GITHUB_USER}" - else - echo "No SSH key found, cloning via https. You may want to set up your ssh keys." - fi +function system_deps() { + if [[ "${OS}" == "Linux" ]]; then + sudo apt-get update >>"${LOGFILE}" 2>&1 + sudo apt-get --yes install \ + wget curl git openssh-server ack jq httpie byobu \ + >>"${LOGFILE}" 2>&1 + elif [[ "${OS}" == "Darwin" ]]; then + sudo brew update | sudo tee -a "${LOGFILE}" + sudo brew install \ + wget curl git jq httpie | + sudo tee -a "${LOGFILE}" + fi + + # Add an SSH key if one doesn't exist. + if [[ ! -f "${HOME}/.ssh/id_rsa" ]]; then + echo "Adding ssh key" + ssh-keygen -t rsa -N "" -f "${HOME}/.ssh/id_rsa" + fi +} - # If a developer, also get the tutorials - REPOS+=("panoptes-tutorials") +function get_repos() { + echo "Cloning ${REPOS}" + for repo in "${REPOS[@]}"; do + if [[ ! -d "${PANDIR}/${repo}" ]]; then + cd "${PANDIR}" + echo "Cloning ${GITHUB_URL}/${repo}" + # Just redirect the errors because otherwise looks like it hangs. + git clone --single-branch --quiet "${GITHUB_URL}/${repo}.git" &>>"${LOGFILE}" + + # Set panoptes as upstream if clone succeeded. + if [ $? -eq 0 ]; then + cd "${repo}" + git remote add upstream "${PANOPTES_UPSTREAM_URL}/${repo}" + fi + else + echo "${repo} already exists in ${PANDIR}. No auto-update for now, skipping repo." fi - - echo "Cloning ${REPOS}" - for repo in "${REPOS[@]}"; do - if [[ ! -d "${PANDIR}/${repo}" ]]; then - cd "${PANDIR}" - echo "Cloning ${GITHUB_URL}/${repo}" - # Just redirect the errors because otherwise looks like it hangs. - # TODO handle errors if repo doesn't exist (e.g. bad github name). - git clone "${GITHUB_URL}/${repo}.git" >> "${LOGFILE}" 2>&1 - - # Set panoptes as upstream - cd "${repo}" - git remote add upstream "${PUBLIC_GITHUB_URL}/${repo}" - else - # TODO Figure out how to do updates. - echo "${repo} already exists in ${PANDIR}. No auto-update for now, skipping repo." - fi - done + done } -function get_docker { - # Get Docker - if ! command_exists docker; then - echo "Installing Docker" - if [[ "${OS}" = "Linux" ]]; then - /bin/bash -c "$(wget -qO- https://get.docker.com)" &>> "${LOGFILE}" - - echo "Adding ${PANUSER} to docker group" - sudo usermod -aG docker "${PANUSER}" >> "${LOGFILE}" 2>&1 - elif [[ "${OS}" = "Darwin" ]]; then - brew cask install docker - echo "Adding ${PANUSER} to docker group" - sudo dscl -aG docker "${PANUSER}" - fi +function get_docker() { + if ! command_exists docker; then + echo "Installing Docker" + if [[ "${OS}" == "Linux" ]]; then + /bin/bash -c "$(wget -qO- https://get.docker.com)" &>>"${LOGFILE}" + + echo "Adding ${PANUSER} to docker group" + sudo usermod -aG docker "${PANUSER}" >>"${LOGFILE}" 2>&1 + elif [[ "${OS}" == "Darwin" ]]; then + brew cask install docker + echo "Adding ${PANUSER} to docker group" + sudo dscl -aG docker "${PANUSER}" fi + fi - if ! command_exists docker-compose; then - echo "Installing docker-compose" - # Docker compose as container - https://docs.docker.com/compose/install/#install-compose - sudo wget -q "${DOCKER_COMPOSE_INSTALL}" -O /usr/local/bin/docker-compose - sudo chmod a+x /usr/local/bin/docker-compose - fi + if ! command_exists docker-compose; then + echo "Installing docker-compose" + sudo wget -q "${DOCKER_COMPOSE_INSTALL}" -O /usr/local/bin/docker-compose + sudo chmod a+x /usr/local/bin/docker-compose + fi } -function get_or_build_images { - if ${DEVELOPER}; then - echo "Building local PANOPTES docker images." +function get_or_build_images() { + if ${DEVELOPER}; then + echo "Building local PANOPTES docker images." - cd "${PANDIR}/POCS" - ./docker/setup-local-environment.sh - else - echo "Pulling PANOPTES docker images from Google Cloud Registry (GCR)." + cd "${PANDIR}/POCS" + ./docker/setup-local-environment.sh + else + echo "Pulling PANOPTES docker images from Google Cloud Registry (GCR)." - docker pull "${DOCKER_BASE}/panoptes-pocs:latest" - docker pull "${DOCKER_BASE}/panoptes-utils:latest" - docker pull "${DOCKER_BASE}/aag-weather:latest" - fi + sudo docker pull "${DOCKER_BASE}/panoptes-pocs:latest" + sudo docker pull "${DOCKER_BASE}/panoptes-utils:latest" + sudo docker pull "${DOCKER_BASE}/aag-weather:latest" + fi } -function do_install { - clear +function do_install() { + clear - echo "Installing PANOPTES software." - if ${DEVELOPER}; then - echo "**** Developer Mode ****" - echo "GITHUB_USER=${GITHUB_USER}" - fi - echo "PANUSER: ${PANUSER}" - echo "PANDIR: ${PANDIR}" - echo "OS: ${OS}" - echo "Logfile: ${LOGFILE}" + echo "Installing PANOPTES software." + if ${DEVELOPER}; then + echo "**** Developer Mode ****" + echo "GITHUB_USER=${GITHUB_USER}" + fi + echo "PANUSER: ${PANUSER}" + echo "PANDIR: ${PANDIR}" + echo "OS: ${OS}" + echo "Logfile: ${LOGFILE}" - echo "Creating directories in ${PANDIR}" - make_directories + echo "Creating directories in ${PANDIR}" + make_directories - echo "Setting up environment variables in ${ENV_FILE}" - setup_env_vars + echo "Setting up environment variables in ${ENV_FILE}" + setup_env_vars - echo "Installing system dependencies" - system_deps + echo "Installing system dependencies" + system_deps - echo "Installing docker and docker-compose" - get_docker + echo "Installing docker and docker-compose" + get_docker + if ${DEVELOPER}; then echo "Cloning PANOPTES source code" get_repos + fi - get_or_build_images + get_or_build_images - echo "Please reboot your machine before using POCS." - - read -p "Reboot now? [y/N]: " -r - if [[ $REPLY =~ ^[Yy]$ ]]; then - sudo reboot - fi + echo "Please reboot your machine before using POCS." - exit 0; + read -p "Reboot now? [y/N]: " -r + if [[ $REPLY =~ ^[Yy]$ ]]; then + sudo reboot + fi } do_install From e1bf5cf924418cc2058d6f5005ab6bd3283aa558 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Sat, 15 Aug 2020 16:27:30 -0700 Subject: [PATCH 20/22] panoptes-utils and docker updates (#994) * Docker updates * Setting build style and docker files to match `panoptes-utils`. * Simplifying `developer` image. * Removing AAG docker compose files. * * Ignore wayward coverage files. * * Remove `mocket` * * CI files don't rebuild utils. * * Fixing up the developer environment. * Deverloper Docker environment * Don't install extra plugins. * Jupyter lab installed but not started by default. * Fix github action path * Dont' double-start shell. * * Fix the bash. * * We removed travis? * * Don't deal with yaml in this PR. * * Don't deal with yaml in this PR. * Add `httpie` and `jq` for `developer` * * Dev env only shares POCS. * Cleanup start commands. * * Readme updates. * * Don't run the dev environment in the background. * * Add github actions for creating a release from a proper tag and for creating a package from a release. --- .dockerignore | 4 + .gcloudignore | 3 +- .github/workflows/create-release.yml | 28 ++++++ .github/workflows/python-publish.yml | 31 +++++++ .github/workflows/pythontest.yaml | 6 +- .gitignore | 1 + .readthedocs.yml | 2 +- .travis.yml | 21 ----- bin/panoptes-develop | 24 +----- docker/Dockerfile | 65 ++++++++++++++ docker/README.rst | 82 +++--------------- docker/build-image.sh | 20 +++-- docker/cloudbuild.yaml | 65 ++++++++------ docker/develop.Dockerfile | 52 ----------- docker/developer-env.Dockerfile | 67 --------------- docker/developer.Dockerfile | 59 +++++++++++++ docker/docker-compose-aag.yaml | 40 --------- ...env.yaml => docker-compose-developer.yaml} | 22 ++--- docker/docker-compose-testing.yaml | 25 ------ docker/latest.Dockerfile | 47 ---------- docker/setup-local-environment.sh | 86 +++++++++---------- docs/requirements.txt | 1 - scripts/testing/run-tests.sh | 5 +- scripts/testing/test-software.sh | 28 ++++++ setup.cfg | 5 +- 25 files changed, 340 insertions(+), 449 deletions(-) create mode 100644 .github/workflows/create-release.yml create mode 100644 .github/workflows/python-publish.yml delete mode 100644 .travis.yml create mode 100644 docker/Dockerfile delete mode 100644 docker/develop.Dockerfile delete mode 100644 docker/developer-env.Dockerfile create mode 100644 docker/developer.Dockerfile delete mode 100644 docker/docker-compose-aag.yaml rename docker/{docker-compose-developer-env.yaml => docker-compose-developer.yaml} (50%) delete mode 100644 docker/docker-compose-testing.yaml delete mode 100644 docker/latest.Dockerfile create mode 100755 scripts/testing/test-software.sh diff --git a/.dockerignore b/.dockerignore index f277c06c8..b296ccfa9 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,6 +3,10 @@ docs/* .venv venv +# Pass the git folder to the local build context. +# This is only needed for local builds because of setuptools_scm. +# The cloudbuild version ignores this folder (and creates a smaller image) +# because it installs from pypi rather than local. !.git .github diff --git a/.gcloudignore b/.gcloudignore index d5f5d4964..730ffb944 100644 --- a/.gcloudignore +++ b/.gcloudignore @@ -2,7 +2,8 @@ .venv venv -!.git +# See note in .dockerignore about the git folder. +.git .github *.md diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml new file mode 100644 index 000000000..d4f27d967 --- /dev/null +++ b/.github/workflows/create-release.yml @@ -0,0 +1,28 @@ +# This will create a release for any properly tagged branch. +on: + push: + # Sequence of patterns matched against refs/tags + tags: + - 'v[0-9].[0-9]+.[0-9]+' # Push events to matching vX.Y.Z, but not vX.Y.Zdev + +name: Create Release + +jobs: + build: + name: Create Release + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Create Release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token + with: + tag_name: ${{ github.ref }} + release_name: Release ${{ github.ref }} + body: | + See Changelog for details + draft: false + prerelease: false diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml new file mode 100644 index 000000000..4e1ef42d2 --- /dev/null +++ b/.github/workflows/python-publish.yml @@ -0,0 +1,31 @@ +# This workflows will upload a Python Package using Twine when a release is created +# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries + +name: Upload Python Package + +on: + release: + types: [created] + +jobs: + deploy: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + - name: Build and publish + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + python setup.py sdist bdist_wheel + twine upload dist/* diff --git a/.github/workflows/pythontest.yaml b/.github/workflows/pythontest.yaml index 7fc6d7630..aa5479c52 100644 --- a/.github/workflows/pythontest.yaml +++ b/.github/workflows/pythontest.yaml @@ -31,9 +31,7 @@ jobs: uses: actions/checkout@v2 - name: Build panoptes-pocs image run: | - # Make sure git goes to the build context. - sed -i s'/^\.git$/\!\.git/' .dockerignore - docker build -t panoptes-pocs:develop -f docker/develop.Dockerfile . + PANOPTES_POCS=. INCLUDE_UTILS=false docker/setup-local-environment.sh - name: Test with pytest in panoptes-pocs container run: | mkdir -p coverage_dir && chmod 777 coverage_dir @@ -45,7 +43,7 @@ jobs: -v $PWD/coverage_dir:/var/panoptes/logs \ -v $PWD/coverage_dir:/tmp/coverage \ panoptes-pocs:develop \ - scripts/testing/run-tests.sh + "/var/panoptes/POCS/scripts/testing/run-tests.sh" - name: Upload coverage report to codecov.io uses: codecov/codecov-action@v1 if: success() diff --git a/.gitignore b/.gitignore index 2830c8d2b..7dd8d4802 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,7 @@ htmlcov/* .tox junit.xml coverage.xml +.coverage.* .pytest_cache/ # Build and docs folder/files diff --git a/.readthedocs.yml b/.readthedocs.yml index f4a658576..087a8e040 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -12,7 +12,7 @@ sphinx: formats: all python: - version: 3.7 + version: 3.8 install: - requirements: docs/requirements.txt - method: pip diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 119b62fe6..000000000 --- a/.travis.yml +++ /dev/null @@ -1,21 +0,0 @@ -dist: xenial -sudo: required -language: python -addons: - apt: - packages: - - docker-ce -python: - # Doesn't matter because we run inside the docker container. - - "3.6" -services: - - docker -before_install: - # Make sure git goes to the build context. - - sed -i s'/^\.git$/\!\.git/' .dockerignore - - docker build -t panoptes-pocs:develop -f ${TRAVIS_BUILD_DIR}/docker/develop.Dockerfile ${TRAVIS_BUILD_DIR} -install: true -script: - - docker run -it - panoptes-pocs:develop - scripts/testing/run-tests.sh diff --git a/bin/panoptes-develop b/bin/panoptes-develop index aea51f7e9..8958a938c 100755 --- a/bin/panoptes-develop +++ b/bin/panoptes-develop @@ -6,22 +6,14 @@ PARAMS=${@:2} export PANDIR=${PANDIR:-/var/panoptes} export IMAGE="${IMAGE:-panoptes-pocs}" -export TAG="${TAG:-developer-env}" +export TAG="${TAG:-developer}" cd "${PANDIR}" ## Add the daemon option by default. if [[ "${SUBCMD}" == "up" ]]; then - SUBCMD="up -d" - export CONTAINER_NAME="pocs-developer-env" - export COMPOSE_FILE="${PANDIR}/POCS/docker/docker-compose-developer-env.yaml" -fi - -## Add the deamon option by default. -if [[ "${SUBCMD}" == "test" ]]; then - SUBCMD="up" - export CONTAINER_NAME="pocs-develop-testing" - export COMPOSE_FILE="${PANDIR}/POCS/docker/docker-compose-testing.yaml" + export CONTAINER_NAME="pocs-developer" + export COMPOSE_FILE="${PANDIR}/POCS/docker/docker-compose-developer.yaml" fi # Pass any other cli args to the containers as an env var named CLI_ARGS @@ -35,17 +27,7 @@ export DOCKER_RUN_OPTIONS="${DOCKER_RUN_OPTIONS:--e IMAGE=${IMAGE} -e TAG=${TAG} eval "DOCKER_RUN_OPTIONS=\"${DOCKER_RUN_OPTIONS}\" \ docker-compose \ --project-directory ${PANDIR} \ - --env-file ${PANDIR}/env \ -f ${COMPOSE_FILE} \ -p panoptes \ ${SUBCMD} \ ${PARAMS}" - -# If we just started the environment, try to open the browser for the user. -if [[ "${SUBCMD}" == "up -d" ]]; then - # Prompt for password - "${PANDIR}/POCS/bin/wait-for-it.sh" \ - localhost:8888 \ - -- \ - docker exec -it -u panoptes "${CONTAINER_NAME}" jupyter notebook list | grep http | cut -d ' ' -f 1 | xargs xdg-open -fi diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 000000000..8dd92988c --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,65 @@ +ARG image_url=gcr.io/panoptes-exp/panoptes-utils:latest +FROM ${image_url} AS pocs-base + +LABEL description="Installs the panoptes-pocs module from GitHub. \ +Used as a production image, i.e. for running on PANOPTES units." +LABEL maintainers="developers@projectpanoptes.org" +LABEL repo="github.com/panoptes/POCS" + +ARG panuser=panoptes +ARG userid=1000 +ARG pan_dir=/var/panoptes +ARG pocs_dir="${pan_dir}/POCS" +ARG conda_env_name="panoptes" + +ARG arduino_url="https://raw.githubusercontent.com/arduino/arduino-cli/master/install.sh" +ARG gphoto2_url="https://raw.githubusercontent.com/gonzalo/gphoto2-updater/master/gphoto2-updater.sh" + +# Default install latest from pip. +ARG github_branch="${BRANCH_NAME:-master}" +ARG pip_extras="[testing,google]" +ARG pip_install="git+https://github.com/panoptes/POCS@${github_branch}#egg=panoptes-pocs" + +ENV DEBIAN_FRONTEND=noninteractive +ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 + +ENV PANDIR $pan_dir +ENV PANLOG "$pan_dir/logs" +ENV PANUSER $panuser +ENV POCS $pocs_dir + +# Install system dependencies. +USER root +RUN apt-get update && apt-get install --no-install-recommends --yes \ + gcc \ + libncurses5-dev \ + udev + +# Install program dependencies. +USER ${PANUSER} +WORKDIR ${POCS} +RUN mkdir -p "${PANDIR}/scripts" && \ + cd "${PANDIR}/scripts" && \ + # Install gphoto2 auto-updater + wget $gphoto2_url -O gphoto2-updater.sh && \ + chmod +x gphoto2-updater.sh && \ + sudo /bin/bash gphoto2-updater.sh --stable && \ + # Install arduino-cli. + wget -q "${arduino_url}" -O install-arduino-cli.sh && \ + sudo BINDIR="/usr/local/bin" /bin/sh install-arduino-cli.sh && \ + sudo chown -R "${PANUSER}":"${PANUSER}" "${PANDIR}" + +USER ${PANUSER} +# Install the module. +# Can't seem to get around the hard-coding here. +COPY --chown=panoptes:panoptes . . +RUN echo "Installing ${pip_install}" && \ + "${PANDIR}/conda/envs/${conda_env_name}/bin/pip" install -e "${pip_install}${pip_extras}" && \ + # Cleanup + sudo apt-get autoremove --purge --yes && \ + sudo apt-get autoclean --yes && \ + sudo apt-get --yes clean && \ + sudo rm -rf /var/lib/apt/lists/* && \ + "${PANDIR}/conda/bin/conda" clean -tipsy + +USER root diff --git a/docker/README.rst b/docker/README.rst index b92b7b10b..9475a5cb8 100644 --- a/docker/README.rst +++ b/docker/README.rst @@ -5,8 +5,6 @@ POCS is available as a docker image hosted on Google Cloud Registry (GCR): Image name: ``gcr.io/panoptes-exp/panoptes-pocs`` -Tags: ``latest``, ``develop``, and ``developer-env``. - Setup ~~~~~ @@ -14,83 +12,29 @@ To build the images locally: .. code:: bash - docker/setup-local-environment.sh + INCLUDE_UTILS=true docker/setup-local-environment.sh + +This will build all required images locally and is suitable for testing and development. Then, to run the test suite locally: .. code:: bash - panoptes-develop test - -This will build all three images locally and is suitable for testing and development. - -Description -~~~~~~~~~~~ - -The ``panoptes-pocs`` image comes in three separate flavors, or tags, -that serve different purposes. - -latest -^^^^^^ + scripts/testing/test-software.sh -The ``latest`` image is the "production" version of ``panoptes-pocs``. +developer +^^^^^^^^^ -PANOPTES units should be running this flavor. +The ``developer`` image is meant to be be used by developers or anyone wishing to +explore the code. It is the same as the local ``develop``, but also installs additional +plotting libraries and the ``jupyter`` environment. -When running the install script, this will be the default install option unless the "developer" is selected. - -develop -^^^^^^^ - -The ``develop`` image is used for running the automated tests. These are -run automatically on both GitHub and Travis for all code pushes but can -also be run locally while doing development. - -developer-env -^^^^^^^^^^^^^ - -The ``developer-env`` image is meant to be be used by developers or anyone wishing to -explore the code. The image should be built locally using the ``docker/setup-local-environment.sh`` -script (or, ideally, just use the ``install-pocs`` script). +The image should be built locally using the ``docker/setup-local-environment.sh`` +script (see above). The ``bin/panoptes-develop up`` can then be used to start a docker container -instance that will launch ``jupyter-lab`` from ``$PANDIR`` automatically. - -There are a few ways to get the development version. - -1) If you have ``git`` and are comfortable using the command line: +instance that will launch ``jupyter lab`` from ``$PANDIR`` automatically. .. code-block:: bash - cd $PANDIR - - # Get the repository. - git clone https://github.com/panoptes/panoptes-pocs.git - cd panoptes-pocs - - # Run environment. - bin/panoptes-develop up - -2) If you would like to build your own local docker image: - -.. code-block:: bash - - cd $PANDIR/panoptes-pocs - # First build the 'latest' image locally. - docker build -t panoptes-pocs:latest -f docker/latest.Dockerfile . - - # Then build the develop image locally. - docker build \ - --build-arg base_image=panoptes-pocs:latest \ - -t panoptes-pocs:develop \ - -f docker/develop.Dockerfile . - - # Wait for build to finish... - - # Run with new image. - IMAGE=panoptes-pocs bin/panoptes-develop up - -3) If you are using a new system: - - TODO: Document this section. - + bin/panoptes-develop up \ No newline at end of file diff --git a/docker/build-image.sh b/docker/build-image.sh index bb0009b59..a9dcd1063 100755 --- a/docker/build-image.sh +++ b/docker/build-image.sh @@ -1,14 +1,16 @@ -#!/bin/bash -e +#!/usr/bin/env bash +set -e -SOURCE_DIR="${PANDIR}/POCS" -BASE_CLOUD_FILE="cloudbuild.yaml" -TAG="${1:-develop}" +TAG="${1:-latest}" + +IMAGE_NAME="panoptes-pocs" +CLOUD_FILE="cloudbuild.yaml" +SOURCE_DIR="${PANDIR}/${IMAGE_NAME}" cd "${SOURCE_DIR}" -echo "Building gcr.io/panoptes-exp/panoptes-pocs:${TAG}" +echo "Building gcr.io/panoptes-exp/${IMAGE_NAME}:${TAG} in ${SOURCE_DIR}" gcloud builds submit \ - --timeout="5h" \ - --substitutions="_TAG=${TAG}" \ - --config "${SOURCE_DIR}/docker/${BASE_CLOUD_FILE}" \ - "${SOURCE_DIR}" + --substitutions="_TAG=${TAG}" \ + --config "${SOURCE_DIR}/docker/${CLOUD_FILE}" \ + "${SOURCE_DIR}" diff --git a/docker/cloudbuild.yaml b/docker/cloudbuild.yaml index b8ecf6d4f..83bd5688e 100644 --- a/docker/cloudbuild.yaml +++ b/docker/cloudbuild.yaml @@ -1,40 +1,49 @@ +options: + machineType: "N1_HIGHCPU_8" + substitutionOption: "ALLOW_LOOSE" +timeout: 18000s # 5 hours + +substitutions: + _PLATFORMS: linux/amd64,linux/arm64 + _TAG: latest + steps: # Set up multiarch support - - name: 'gcr.io/cloud-builders/docker' - id: 'setup-buildx' + - name: "gcr.io/cloud-builders/docker" + id: "setup-buildx" env: - - 'DOCKER_CLI_EXPERIMENTAL=enabled' + - "DOCKER_CLI_EXPERIMENTAL=enabled" args: - - 'run' - - '--privileged' - - '--rm' - - 'docker/binfmt:a7996909642ee92942dcd6cff44b9b95f08dad64' - waitFor: ['-'] + - "run" + - "--privileged" + - "--rm" + - "docker/binfmt:a7996909642ee92942dcd6cff44b9b95f08dad64" + waitFor: ["-"] # Build builder - - name: 'gcr.io/cloud-builders/docker' - id: 'build-builder' + - name: "gcr.io/cloud-builders/docker" + id: "build-builder" env: - - 'DOCKER_CLI_EXPERIMENTAL=enabled' + - "DOCKER_CLI_EXPERIMENTAL=enabled" args: - - 'buildx' - - 'create' - - '--use' - - '--driver=docker-container' - waitFor: ['setup-buildx'] + - "buildx" + - "create" + - "--use" + - "--driver=docker-container" + waitFor: ["setup-buildx"] # Build - - name: 'gcr.io/cloud-builders/docker' - id: 'build-images' + - name: "gcr.io/cloud-builders/docker" + id: "build-images" env: - - 'DOCKER_CLI_EXPERIMENTAL=enabled' + - "DOCKER_CLI_EXPERIMENTAL=enabled" args: - - 'buildx' - - 'build' - - '--push' - - '--platform=linux/amd64,linux/arm64' - - '-f=docker/latest.Dockerfile' - - '--tag=gcr.io/${PROJECT_ID}/panoptes-pocs:latest' - - '--cache-from=gcr.io/${PROJECT_ID}/panoptes-pocs:latest' - - '.' - waitFor: ['build-builder'] + - "buildx" + - "build" + - "--push" + - "--platform=linux/amd64,linux/arm64" + - "-f=docker/Dockerfile" + - "--tag=gcr.io/${PROJECT_ID}/panoptes-pocs:${_TAG}" + - "--cache-from=gcr.io/${PROJECT_ID}/panoptes-pocs:${_TAG}" + - "." + waitFor: ["build-base"] diff --git a/docker/develop.Dockerfile b/docker/develop.Dockerfile deleted file mode 100644 index c1057919a..000000000 --- a/docker/develop.Dockerfile +++ /dev/null @@ -1,52 +0,0 @@ -ARG IMAGE_URL=gcr.io/panoptes-exp/panoptes-utils:latest -FROM ${IMAGE_URL} AS pocs-base - -LABEL description="Installs the panoptes-pocs module in local editable \ -mode. This requires the entire git history to be present. Used for testing." -LABEL maintainers="developers@projectpanoptes.org" -LABEL repo="github.com/panoptes/POCS" - -ARG pandir=/var/panoptes -ARG arduino_url="https://raw.githubusercontent.com/arduino/arduino-cli/master/install.sh" -ARG gphoto2_url="https://raw.githubusercontent.com/gonzalo/gphoto2-updater/master/gphoto2-updater.sh" - -ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 -ENV SHELL /bin/zsh -ENV PANDIR $pandir -ENV POCS ${PANDIR}/POCS -ENV SOLVE_FIELD /usr/bin/solve-field - -RUN apt-get update \ - && apt-get install --no-install-recommends --yes \ - gcc libncurses5-dev udev \ - # GPhoto2 - && wget $gphoto2_url \ - && chmod +x gphoto2-updater.sh \ - && /bin/bash gphoto2-updater.sh --stable \ - && rm gphoto2-updater.sh \ - # arduino-cli - && curl -fsSL $arduino_url | BINDIR="/usr/local/bin" sh - -# panoptes-utils -USER ${PANUSER} -COPY --chown=panoptes:panoptes . "${PANDIR}/POCS/" -RUN cd "${PANDIR}/POCS" && \ - pip3 install -U -e ".[testing,google]" - -# Cleanup apt. -USER root -RUN apt-get autoremove --purge -y \ - autoconf \ - automake \ - autopoint \ - build-essential \ - gcc \ - gettext \ - libtool \ - pkg-config && \ - apt-get autoremove --purge -y && \ - apt-get -y clean && \ - rm -rf /var/lib/apt/lists/* - -WORKDIR ${POCS} -CMD ["/bin/zsh"] diff --git a/docker/developer-env.Dockerfile b/docker/developer-env.Dockerfile deleted file mode 100644 index b3af9afef..000000000 --- a/docker/developer-env.Dockerfile +++ /dev/null @@ -1,67 +0,0 @@ -ARG BASE_IMAGE=panoptes-pocs:develop -FROM ${BASE_IMAGE} - -LABEL description="Installs the local folder in develop mode (i.e. pip install .e). \ -and installs a number of developer tools. Runs jupyter lab instance. This assumes the \ -`panoptes-pocs:develop` has already been built locally." -LABEL maintainers="developers@projectpanoptes.org" -LABEL repo="github.com/panoptes/POCS" - -ARG panuser=panoptes -ARG userid=1000 -ARG pan_dir=/var/panoptes -ARG pocs_dir="${pan_dir}/POCS" - -ENV DEBIAN_FRONTEND=noninteractive -ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 -ENV SHELL /bin/zsh - -ENV USERID $userid -ENV PANDIR $pan_dir -ENV PANLOG "$pan_dir/logs" -ENV PANUSER $panuser -ENV POCS $pocs_dir -ENV PATH "/home/${PANUSER}/.local/bin:$PATH" - -RUN apt-get update && \ - # Node for jupyterlab. - curl -sL https://deb.nodesource.com/setup_12.x | bash - && \ - # Make a developer's life easier. - apt-get install -y --no-install-recommends \ - wget curl bzip2 ca-certificates nano neovim \ - gcc git pkg-config ncdu sudo nodejs && \ - echo "$PANUSER ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers - - -USER $PANUSER -# Can't seem to get around the hard-coding -COPY --chown=panoptes:panoptes . ${PANDIR}/POCS/ -RUN cd ${PANDIR}/POCS && \ - # Install everything! - pip3 install -e ".[google,developer,plotting,testing]" && \ - # Set some jupyterlab defaults. - mkdir -p /home/panoptes/.jupyter && \ - jupyter-lab --generate-config && \ - # Jupyterlab extensions. - echo "c.JupyterApp.answer_yesBool = True" >> \ - "/home/panoptes/.jupyter/jupyter_notebook_config.py" && \ - echo "c.JupyterApp.open_browserBool = False" >> \ - "/home/panoptes/.jupyter/jupyter_notebook_config.py" && \ - echo "c.JupyterAppy.notebook_dir = '${PANDIR}'" >> \ - "/home/panoptes/.jupyter/jupyter_notebook_config.py" && \ - jupyter labextension install @pyviz/jupyterlab_pyviz \ - jupyterlab-drawio \ - @aquirdturtle/collapsible_headings \ - @telamonian/theme-darcula - -USER root - -# Cleanup apt. -RUN apt-get autoremove --purge -y && \ - apt-get -y clean && \ - rm -rf /var/lib/apt/lists/* - -WORKDIR ${PANDIR} - -# Start a jupyterlab instance. -CMD ["/home/panoptes/.local/bin/jupyter-lab"] diff --git a/docker/developer.Dockerfile b/docker/developer.Dockerfile new file mode 100644 index 000000000..4e0fcd78e --- /dev/null +++ b/docker/developer.Dockerfile @@ -0,0 +1,59 @@ +ARG BASE_IMAGE=panoptes-pocs:develop +FROM ${BASE_IMAGE} + +LABEL description="Installs a number of developer tools. Runs jupyter lab instance. \ +This assumes the `panoptes-pocs:develop` has already been built locally." +LABEL maintainers="developers@projectpanoptes.org" +LABEL repo="github.com/panoptes/POCS" + +ARG panuser=panoptes +ARG userid=1000 +ARG pan_dir=/var/panoptes +ARG pocs_dir="${pan_dir}/POCS" +ARG conda_env_name="panoptes" + +ENV DEBIAN_FRONTEND=noninteractive +ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 +ENV SHELL /bin/zsh + +ENV USERID $userid +ENV PANDIR $pan_dir +ENV PANLOG "$pan_dir/logs" +ENV PANUSER $panuser +ENV POCS $pocs_dir + +RUN apt-get update && \ + # Make a developer's life easier. + apt-get install --yes --no-install-recommends \ + bzip2 ca-certificates nano neovim \ + ncdu htop + +USER $PANUSER +RUN echo "Installing developer tools" && \ + "${PANDIR}/conda/bin/conda" install --name "${conda_env_name}" \ + altair \ + bokeh \ + httpie \ + jupyterlab \ + jq \ + holoviews \ + hvplot \ + nodejs \ + seaborn && \ + # Set some jupyterlab defaults. + mkdir -p /home/panoptes/.jupyter && \ + /usr/bin/env zsh -c "${PANDIR}/conda/envs/${conda_env_name}/bin/jupyter-lab --no-browser --generate-config" && \ + # Jupyterlab extensions. + echo "c.JupyterApp.answer_yes = True" >> \ + "/home/panoptes/.jupyter/jupyter_notebook_config.py" && \ + echo "c.JupyterApp.open_browser = False" >> \ + "/home/panoptes/.jupyter/jupyter_notebook_config.py" && \ + echo "c.JupyterApp.notebook_dir = '${PANDIR}'" >> \ + "/home/panoptes/.jupyter/jupyter_notebook_config.py" && \ + # Cleanup + sudo apt-get autoremove --purge --yes && \ + sudo apt-get autoclean --yes && \ + sudo rm -rf /var/lib/apt/lists/* + +USER root +WORKDIR ${PANDIR} \ No newline at end of file diff --git a/docker/docker-compose-aag.yaml b/docker/docker-compose-aag.yaml deleted file mode 100644 index fdcdb94f2..000000000 --- a/docker/docker-compose-aag.yaml +++ /dev/null @@ -1,40 +0,0 @@ -version: '3.7' -services: - aag-weather-reader: - image: gcr.io/panoptes-exp/aag-weather:latest - init: true - container_name: aag-weather-reader - privileged: true - network_mode: host - restart: on-failure - volumes: - - pandir:/var/panoptes - command: - - "python" - - "/app/scripts/read-aag.py" - - "--config-file" - - "/var/panoptes/conf_files/pocs_local.yaml" - - "--db-file" - - "/var/panoptes/json_store/panoptes/weather.db" - - "--store-result" - - "--verbose" - aag-weather-server: - image: gcr.io/panoptes-exp/aag-weather:latest - init: true - container_name: aag-weather-server - privileged: true - network_mode: host - environment: - - DB_NAME=/var/panoptes/json_store/panoptes/weather.db - command: ["flask", "run"] - restart: on-failure - volumes: - - pandir:/var/panoptes -volumes: - pandir: - driver: local - driver_opts: - type: none - device: /var/panoptes - o: bind - diff --git a/docker/docker-compose-developer-env.yaml b/docker/docker-compose-developer.yaml similarity index 50% rename from docker/docker-compose-developer-env.yaml rename to docker/docker-compose-developer.yaml index 9abaa45a8..c96ea10d6 100644 --- a/docker/docker-compose-developer-env.yaml +++ b/docker/docker-compose-developer.yaml @@ -6,30 +6,26 @@ services: container_name: config-server privileged: true network_mode: host - env_file: $PANDIR/env restart: on-failure volumes: - - pandir:/var/panoptes - command: - - "panoptes-config-server" - - "run" - - "/var/panoptes/POCS/conf_files/pocs.yaml" - develop-env: - image: "${IMAGE:-panoptes-pocs}:${TAG:-developer-env}" + - pocsdir:/var/panoptes/POCS + command: ["panoptes-config-server run /var/panoptes/POCS/conf_files/pocs.yaml"] + developer: + image: "${IMAGE:-panoptes-pocs}:${TAG:-developer}" init: true - container_name: "${CONTAINER_NAME:-pocs-developer-env}" + container_name: "${CONTAINER_NAME:-pocs-developer}" privileged: true network_mode: host - env_file: $PANDIR/env depends_on: - "config-server" volumes: - - pandir:/var/panoptes + - pocsdir:/var/panoptes/POCS + command: ["jupyter lab", "--ip=0.0.0.0"] volumes: - pandir: + pocsdir: driver: local driver_opts: type: none - device: /var/panoptes + device: /var/panoptes/POCS o: bind diff --git a/docker/docker-compose-testing.yaml b/docker/docker-compose-testing.yaml deleted file mode 100644 index 98cbbc998..000000000 --- a/docker/docker-compose-testing.yaml +++ /dev/null @@ -1,25 +0,0 @@ -version: '3.7' -services: - develop-env: - image: "${IMAGE:-panoptes-pocs}:${TAG:-develop}" - init: true - container_name: "${CONTAINER_NAME:-panoptes-pocs:develop}" - privileged: true - network_mode: host - env_file: $PANDIR/env - tty: true - environment: - - CLI_ARGS="${CLI_ARGS}" - command: - - "${POCS}/scripts/testing/run-tests.sh" - volumes: - - pandir:/var/panoptes - - /var/panoptes/logs:/var/panoptes/logs - - /var/panoptes/POCS:/var/panoptes/POCS -volumes: - pandir: - driver: local - driver_opts: - type: none - device: /var/panoptes - o: bind diff --git a/docker/latest.Dockerfile b/docker/latest.Dockerfile deleted file mode 100644 index 0d9575bfa..000000000 --- a/docker/latest.Dockerfile +++ /dev/null @@ -1,47 +0,0 @@ -ARG IMAGE_URL=gcr.io/panoptes-exp/panoptes-utils:latest -FROM ${IMAGE_URL} AS pocs-base - -LABEL description="Installs the panoptes-pocs module from pip. \ -Used as a production image, i.e. for running on PANOPTES units." -LABEL maintainers="developers@projectpanoptes.org" -LABEL repo="github.com/panoptes/POCS" - -ARG pandir=/var/panoptes -ARG arduino_url="https://raw.githubusercontent.com/arduino/arduino-cli/master/install.sh" -ARG gphoto2_url="https://raw.githubusercontent.com/gonzalo/gphoto2-updater/master/gphoto2-updater.sh" - -ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 -ENV SHELL /bin/zsh -ENV PANDIR $pandir -ENV POCS ${PANDIR}/POCS -ENV SOLVE_FIELD /usr/bin/solve-field - -RUN apt-get update \ - && apt-get install --no-install-recommends --yes \ - gcc libncurses5-dev udev \ - # GPhoto2 - && wget $gphoto2_url \ - && chmod +x gphoto2-updater.sh \ - && /bin/bash gphoto2-updater.sh --stable \ - && rm gphoto2-updater.sh \ - # arduino-cli - && curl -fsSL $arduino_url | BINDIR="/usr/local/bin" sh \ - # Install the module. - && pip3 install -U "panoptes-pocs[google]" - -# Cleanup apt. -RUN apt-get autoremove --purge -y \ - autoconf \ - automake \ - autopoint \ - build-essential \ - gcc \ - gettext \ - libtool \ - pkg-config && \ - apt-get autoremove --purge -y && \ - apt-get -y clean && \ - rm -rf /var/lib/apt/lists/* - -WORKDIR ${POCS} -CMD ["/bin/zsh"] diff --git a/docker/setup-local-environment.sh b/docker/setup-local-environment.sh index 9a102962a..975dd8a40 100755 --- a/docker/setup-local-environment.sh +++ b/docker/setup-local-environment.sh @@ -1,60 +1,56 @@ #!/usr/bin/env bash -set -e - -export PANDIR=${PANDIR:-/var/panoptes} -export POCS=${POCS:-/var/panoptes/POCS} - -echo "Setting up local environment." - -echo "Removing stale docker images to make space" -docker system prune --force - -echo "Building local panoptes-utils" -. "${PANDIR}/panoptes-utils/docker/setup-local-environment.sh" -cd "${POCS}" +set -e -# In the local develop we need to pass git to the docker build context. -#sed -i s'/^\.git$/\!\.git/' .dockerignore +INCLUDE_BASE=${INCLUDE_BASE:-true} # INCLUDE_UTILS must be true to work. +INCLUDE_UTILS=${INCLUDE_UTILS:-false} +INCLUDE_DEVELOPER=${INCLUDE_DEVELOPER:-false} -echo "Building local panoptes-pocs:latest from panoptes-utils:develop" -docker build \ - --quiet --force-rm \ - --build-arg IMAGE_URL="panoptes-utils:develop" \ - -t "panoptes-pocs:latest" \ - -f "${POCS}/docker/latest.Dockerfile" \ - "${POCS}" +PANOPTES_UTILS=${PANOPTES_UTILS:-$PANDIR/panoptes-utils} +PANOPTES_POCS=${PANOPTES_POCS:-$PANDIR/POCS} +_IMAGE_URL="gcr.io/panoptes-exp/panoptes-utils:latest" -echo "Building local panoptes-pocs:develop" -docker build \ - --quiet --force-rm \ - --build-arg IMAGE_URL="panoptes-pocs:latest" \ +echo "Setting up local environment." +cd "${PANOPTES_POCS}" + +build_utils() { + /bin/bash "${PANOPTES_UTILS}/docker/setup-local-environment.sh" + # Use our local image for build below. + _IMAGE_URL="panoptes-utils:develop" +} + +build_develop() { + echo "Building local panoptes-pocs:develop from ${_IMAGE_URL} in ${PANOPTES_POCS}" + docker build \ + --build-arg="image_url=${_IMAGE_URL}" \ + --build-arg="pip_install=." \ -t "panoptes-pocs:develop" \ - -f "${POCS}/docker/develop.Dockerfile" \ - "${POCS}" - -echo "Building local panoptes-pocs:developer-env" -docker build \ - --quiet --force-rm \ - --build-arg IMAGE_URL="panoptes-pocs:develop" \ - -t "panoptes-pocs:developer-env" \ - -f "${POCS}/docker/developer-env.Dockerfile" \ - "${POCS}" + -f "${PANOPTES_POCS}/docker/Dockerfile" \ + "${PANOPTES_POCS}" +} -# Revert our .dockerignore changes. -#sed -i s'/^!\.git$/\.git/' .dockerignore +build_developer() { + echo "Building local panoptes-pocs:developer from ${_IMAGE_URL} in ${PANOPTES_POCS}" + docker build \ + -t "panoptes-pocs:developer" \ + -f "${PANOPTES_POCS}/docker/developer.Dockerfile" \ + "${PANOPTES_POCS}" +} -docker system prune --force - -cat <=0.2.22 + panoptes-utils>=0.2.23 pyserial>=3.1.1 PyYaml requests - responses scalpl scipy transitions @@ -89,10 +88,10 @@ testing = pycodestyle pytest pytest-cov + pytest-doctestplus pytest-remotedata>=0.3.1' responses - [options.entry_points] # Add here console scripts like: # console_scripts = From b98b0d5511e4b1b33e5ad3436df80d5d8ca29ea2 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Fri, 21 Aug 2020 11:47:08 -0700 Subject: [PATCH 21/22] Minor dockerfile updates 02 (#995) * * Removing script for cloudbuild - it should always be triggered from github for consistent environments. * Moving `developer` files into subfolder. * Dockerfile: * `readline`, `ncurses`, `gphoto2` installed from apt. * Not using `gphoto2` updater for now, although it is saved in scripts folder. * Always install local folder in editable mode (as per script removal above). * Convert README back to markdown. Most news docs should go in gitbook. * Removed some `google` dependencies. * * Fix GH actions test. * * Clone the repo in cloudbuild. * Pull existing image for cache * * More generic base image. * Pull base image of same tag. * Pass base image as arg to dockerfile. * * Bump `panoptes-utils` version * Fetch cached image earlier (not sure this is really helping). * * Fix cloudbuild directory. * * Bump panoptes-utils again. * * Changelog * * Automated releases and packages to pypi. --- .github/workflows/create-release.yml | 22 +++++++- .github/workflows/python-publish.yml | 31 ----------- .github/workflows/pythontest.yaml | 2 +- CHANGELOG.rst | 53 +++++++++++-------- docker/Dockerfile | 18 +++---- docker/README.md | 38 +++++++++++++ docker/README.rst | 40 -------------- docker/build-image.sh | 16 ------ docker/cloudbuild.yaml | 29 +++++++--- .../Dockerfile} | 0 .../docker-compose.yaml} | 0 .../setup-local-environment.sh | 19 +++---- setup.cfg | 5 +- 13 files changed, 134 insertions(+), 139 deletions(-) delete mode 100644 .github/workflows/python-publish.yml create mode 100644 docker/README.md delete mode 100644 docker/README.rst delete mode 100755 docker/build-image.sh rename docker/{developer.Dockerfile => developer/Dockerfile} (100%) rename docker/{docker-compose-developer.yaml => developer/docker-compose.yaml} (100%) rename {docker => scripts}/setup-local-environment.sh (60%) diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index d4f27d967..75cb5888e 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -5,7 +5,7 @@ on: tags: - 'v[0-9].[0-9]+.[0-9]+' # Push events to matching vX.Y.Z, but not vX.Y.Zdev -name: Create Release +name: Create GitHub Release jobs: build: @@ -26,3 +26,23 @@ jobs: See Changelog for details draft: false prerelease: false + deploy: + name: Push Release to PyPi + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + - name: Build and publish + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + python setup.py sdist bdist_wheel + twine upload dist/* diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml deleted file mode 100644 index 4e1ef42d2..000000000 --- a/.github/workflows/python-publish.yml +++ /dev/null @@ -1,31 +0,0 @@ -# This workflows will upload a Python Package using Twine when a release is created -# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries - -name: Upload Python Package - -on: - release: - types: [created] - -jobs: - deploy: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.x' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine - - name: Build and publish - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: | - python setup.py sdist bdist_wheel - twine upload dist/* diff --git a/.github/workflows/pythontest.yaml b/.github/workflows/pythontest.yaml index aa5479c52..ef806a429 100644 --- a/.github/workflows/pythontest.yaml +++ b/.github/workflows/pythontest.yaml @@ -31,7 +31,7 @@ jobs: uses: actions/checkout@v2 - name: Build panoptes-pocs image run: | - PANOPTES_POCS=. INCLUDE_UTILS=false docker/setup-local-environment.sh + PANOPTES_POCS=. INCLUDE_UTILS=false scripts/setup-local-environment.sh - name: Test with pytest in panoptes-pocs container run: | mkdir -p coverage_dir && chmod 777 coverage_dir diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 34d355c11..2c6c00916 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -12,8 +12,12 @@ adheres to `Semantic Versioning `__. Changed ~~~~~~~ -* `panoptes-utils` to `0.2.21`. (#979) -* `panoptes-utils` to `0.2.20`. (#974) +* Dependency updates: + + * `panoptes-utils` to `0.2.26`. (#995) + * `panoptes-utils` to `0.2.21`. (#979) + * `panoptes-utils` to `0.2.20`. (#974) + * Install script. (#974) * Env var file is sourced for zshrc and bashrc. @@ -21,16 +25,15 @@ Changed * Adding a date version to script. (#979) * `docker-compose` version bumped to `1.26.2`. (#979) * Better testing for ssh access. (#984) - * Using [linuxserver.io docker-compose](https://hub.docker.com/r/linuxserver/docker-compose) -so we also have `arm` version without work. (#986) + * Using `linuxserver.io docker-compose `_ so we also have `arm` version without work. (#986) * Fixing conditional so script can proceed without restart. (#986) * Generalizing install script in sections. (#986) - * Development Environment (#974) * Many cleanups to environment and launch. See docs. * Config server started along with development environment. + * Docker images and python packages are now automated via GitHub Actions and Google Cloud Build. (#995) * Docker image updates (#972) @@ -44,6 +47,7 @@ so we also have `arm` version without work. (#986) * Python moved to 3.8. (#974) * Docker images are now built with buildx to get an arm version running. (#978) * Removing readline and pendulum dependencies. (#978) + * Fully automated build and release of packages with GitHub Actions. (#995) * Testing (#974) @@ -69,10 +73,11 @@ so we also have `arm` version without work. (#986) * Camera simulator cleanup. (#974) * Scheduler (#974) + * The `fields_file` is read when scheduler is created. [0.7.4] - 2020-05-31 ----------- +-------------------- Note that we skipped ``0.7.2`` and ``0.7.3``. @@ -111,7 +116,9 @@ Added * Storing an explicit ``safety`` collection in the database. * Configuration file specific for testing rather than relying on ``pocs.yaml``. * Convenience scripts for running tests inside docker container: - ``scripts/testing/test-software.sh`` + + ``scripts/testing/test-software.sh`` + * GitHub Actions for testing and coverage upload. Changed @@ -119,19 +126,25 @@ Changed * Docker as default. (#951). * Weather items have moved to `aag-weather `__. - * Two docker containers run from the ``aag-weather`` image and have a ``docker/docker-compose-aag.yaml`` file to start. + + * Two docker containers run from the ``aag-weather`` image and have a ``docker/docker-compose-aag.yaml`` file to start. + * Config items related to the configuration system have been moved to the `Config Server `__ in ``panoptes-utils`` repo. - * The main interface for POCS related items is through ``self.get_config``, which can take a key and a default, e.g. ``self.get_config('mount.horizon', default='30 deg')``. - * Test writing is affected and is currently more difficult than would be ideal. An updated test writing document will be following this release. + + * The main interface for POCS related items is through ``self.get_config``, which can take a key and a default, e.g. ``self.get_config('mount.horizon', default='30 deg')``. + * Test writing is affected and is currently more difficult than would be ideal. An updated test writing document will be following this release. * Logging has changed to `loguru `__ and has been greatly simplified: - * ``get_root_logger`` has been replaced by ``get_logger``. + + * ``get_root_logger`` has been replaced by ``get_logger``. + * The ``per-run`` logs have been removed and have been replaced by two logs files: - * ``$PANDIR/logs/panoptes.log``: Log file meant for watching on the + + * ``$PANDIR/logs/panoptes.log``: Log file meant for watching on the command line (via ``tail``) or for otherwise human-readable logs. Rotated daily at 11:30 am. Only the previous days' log is retained. - * ``$PANDIR/logs/panoptes_YYYYMMDD.log``: Log file meant for archive + * ``$PANDIR/logs/panoptes_YYYYMMDD.log``: Log file meant for archive or information gathering. Stored in JSON format for ingestion into log analysis service. Rotated daily at 11:30 and stored in a compressed file for 7 days. Future updates will add option to @@ -223,11 +236,9 @@ Removed [0.6.1] - 2018-09-20 -------------------- -| Lots of changes in this release. In particular we've pushed through a -lot of changes -| (especially with the help of @jamessynge) to make the development -process a lot -| smoother. This has in turn contribute to the quality of the codebase. +* Lots of changes in this release. In particular we've pushed through a lot of changes +* (especially with the help of @jamessynge) to make the development process a lot +* smoother. This has in turn contribute to the quality of the codebase. Too long between releases but even more exciting improvements to come! Next up is tackling the events notification system, which will let us @@ -256,11 +267,9 @@ Changed ~~~~~~~ * Mount -* POCS Shell: Hitting ``Ctrl-c`` will complete movement through states - [#590]. +* POCS Shell: Hitting ``Ctrl-c`` will complete movement through states [#590]. * Pointing updates, including ``auto_correct`` [#580]. -* Tracking mode updates (**fixes for Northern Hemisphere only!**) - [#549]. +* Tracking mode updates (**fixes for Northern Hemisphere only!**) [#549]. * Serial interaction improvements [#388, #403]. * Shutdown improvements [#407, #421]. * Dome diff --git a/docker/Dockerfile b/docker/Dockerfile index 8dd92988c..110ddb99e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -14,11 +14,7 @@ ARG conda_env_name="panoptes" ARG arduino_url="https://raw.githubusercontent.com/arduino/arduino-cli/master/install.sh" ARG gphoto2_url="https://raw.githubusercontent.com/gonzalo/gphoto2-updater/master/gphoto2-updater.sh" - -# Default install latest from pip. -ARG github_branch="${BRANCH_NAME:-master}" ARG pip_extras="[testing,google]" -ARG pip_install="git+https://github.com/panoptes/POCS@${github_branch}#egg=panoptes-pocs" ENV DEBIAN_FRONTEND=noninteractive ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 @@ -32,7 +28,9 @@ ENV POCS $pocs_dir USER root RUN apt-get update && apt-get install --no-install-recommends --yes \ gcc \ - libncurses5-dev \ + gphoto2 \ + ncurses-dev \ + readline-common \ udev # Install program dependencies. @@ -43,23 +41,23 @@ RUN mkdir -p "${PANDIR}/scripts" && \ # Install gphoto2 auto-updater wget $gphoto2_url -O gphoto2-updater.sh && \ chmod +x gphoto2-updater.sh && \ - sudo /bin/bash gphoto2-updater.sh --stable && \ + # Don't actually update gphoto2 as of right now. + # sudo /bin/bash gphoto2-updater.sh --stable && \ # Install arduino-cli. wget -q "${arduino_url}" -O install-arduino-cli.sh && \ sudo BINDIR="/usr/local/bin" /bin/sh install-arduino-cli.sh && \ sudo chown -R "${PANUSER}":"${PANUSER}" "${PANDIR}" -USER ${PANUSER} # Install the module. +USER ${PANUSER} # Can't seem to get around the hard-coding here. COPY --chown=panoptes:panoptes . . -RUN echo "Installing ${pip_install}" && \ - "${PANDIR}/conda/envs/${conda_env_name}/bin/pip" install -e "${pip_install}${pip_extras}" && \ +RUN "${PANDIR}/conda/envs/${conda_env_name}/bin/pip" install -e ".${pip_extras}" && \ # Cleanup sudo apt-get autoremove --purge --yes && \ sudo apt-get autoclean --yes && \ sudo apt-get --yes clean && \ sudo rm -rf /var/lib/apt/lists/* && \ - "${PANDIR}/conda/bin/conda" clean -tipsy + "${PANDIR}/conda/bin/conda" clean -tipy USER root diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 000000000..dce637c34 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,38 @@ +Docker Images +============= + +POCS is available as a docker image hosted on Google Cloud Registry (GCR): + +Image name: `gcr.io/panoptes-exp/panoptes-pocs:latest` + +### `develop` image + +To build the images locally: + +```bash +scripts/setup-local-environment.sh +``` + +This will build all required images locally and is suitable for testing and development. + +Then, to run the test suite locally: + +```bash +scripts/testing/test-software.sh +```` + +### `developer` image + +The `developer` image is meant to be be used by developers or anyone wishing to +explore the code. It is the same as the local `develop`, but also installs additional +plotting libraries and the `jupyter` environment. + +The image should be built locally using the `docker/setup-local-environment.sh` +script (see above). + +The `bin/panoptes-develop up` can then be used to start a docker container +instance that will launch `jupyter lab` from `$PANDIR` automatically. + +```bash +bin/panoptes-develop up +``` diff --git a/docker/README.rst b/docker/README.rst deleted file mode 100644 index 9475a5cb8..000000000 --- a/docker/README.rst +++ /dev/null @@ -1,40 +0,0 @@ -Docker Images -============= - -POCS is available as a docker image hosted on Google Cloud Registry (GCR): - -Image name: ``gcr.io/panoptes-exp/panoptes-pocs`` - -Setup -~~~~~ - -To build the images locally: - -.. code:: bash - - INCLUDE_UTILS=true docker/setup-local-environment.sh - -This will build all required images locally and is suitable for testing and development. - -Then, to run the test suite locally: - -.. code:: bash - - scripts/testing/test-software.sh - -developer -^^^^^^^^^ - -The ``developer`` image is meant to be be used by developers or anyone wishing to -explore the code. It is the same as the local ``develop``, but also installs additional -plotting libraries and the ``jupyter`` environment. - -The image should be built locally using the ``docker/setup-local-environment.sh`` -script (see above). - -The ``bin/panoptes-develop up`` can then be used to start a docker container -instance that will launch ``jupyter lab`` from ``$PANDIR`` automatically. - -.. code-block:: bash - - bin/panoptes-develop up \ No newline at end of file diff --git a/docker/build-image.sh b/docker/build-image.sh deleted file mode 100755 index a9dcd1063..000000000 --- a/docker/build-image.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash -set -e - -TAG="${1:-latest}" - -IMAGE_NAME="panoptes-pocs" -CLOUD_FILE="cloudbuild.yaml" -SOURCE_DIR="${PANDIR}/${IMAGE_NAME}" - -cd "${SOURCE_DIR}" - -echo "Building gcr.io/panoptes-exp/${IMAGE_NAME}:${TAG} in ${SOURCE_DIR}" -gcloud builds submit \ - --substitutions="_TAG=${TAG}" \ - --config "${SOURCE_DIR}/docker/${CLOUD_FILE}" \ - "${SOURCE_DIR}" diff --git a/docker/cloudbuild.yaml b/docker/cloudbuild.yaml index 83bd5688e..90c69bfcc 100644 --- a/docker/cloudbuild.yaml +++ b/docker/cloudbuild.yaml @@ -5,9 +5,25 @@ timeout: 18000s # 5 hours substitutions: _PLATFORMS: linux/amd64,linux/arm64 + _BASE_IMAGE: panoptes-utils:latest + _IMAGE_NAME: panoptes-pocs + _REPO_URL: https://github.com/panoptes/POCS _TAG: latest steps: + # Fetch the repo from github + - name: gcr.io/cloud-builders/git + id: "clone-repo" + args: ["clone", "${_REPO_URL}"] + waitFor: ["-"] + + # Pull the cached image. + - name: 'gcr.io/cloud-builders/docker' + id: "pull-cached-image" + entrypoint: 'bash' + args: ['-c', 'docker pull gcr.io/${PROJECT_ID}/${_IMAGE_NAME}:${_TAG} || exit 0'] + waitFor: ["-"] + # Set up multiarch support - name: "gcr.io/cloud-builders/docker" id: "setup-buildx" @@ -32,7 +48,7 @@ steps: - "--driver=docker-container" waitFor: ["setup-buildx"] - # Build + # Build with cloned panoptes-utils as source directory - name: "gcr.io/cloud-builders/docker" id: "build-images" env: @@ -41,9 +57,10 @@ steps: - "buildx" - "build" - "--push" - - "--platform=linux/amd64,linux/arm64" + - "--platform=${_PLATFORMS}" - "-f=docker/Dockerfile" - - "--tag=gcr.io/${PROJECT_ID}/panoptes-pocs:${_TAG}" - - "--cache-from=gcr.io/${PROJECT_ID}/panoptes-pocs:${_TAG}" - - "." - waitFor: ["build-base"] + - "--build-arg=image_url=gcr.io/${PROJECT_ID}/${_BASE_IMAGE}" + - "--tag=gcr.io/${PROJECT_ID}/${_IMAGE_NAME}:${_TAG}" + - "--cache-from=gcr.io/${PROJECT_ID}/${_IMAGE_NAME}:${_TAG}" + - "POCS" + waitFor: ["build-builder", "clone-repo"] diff --git a/docker/developer.Dockerfile b/docker/developer/Dockerfile similarity index 100% rename from docker/developer.Dockerfile rename to docker/developer/Dockerfile diff --git a/docker/docker-compose-developer.yaml b/docker/developer/docker-compose.yaml similarity index 100% rename from docker/docker-compose-developer.yaml rename to docker/developer/docker-compose.yaml diff --git a/docker/setup-local-environment.sh b/scripts/setup-local-environment.sh similarity index 60% rename from docker/setup-local-environment.sh rename to scripts/setup-local-environment.sh index 975dd8a40..a043c4fde 100755 --- a/docker/setup-local-environment.sh +++ b/scripts/setup-local-environment.sh @@ -1,5 +1,4 @@ #!/usr/bin/env bash - set -e INCLUDE_BASE=${INCLUDE_BASE:-true} # INCLUDE_UTILS must be true to work. @@ -8,35 +7,37 @@ INCLUDE_DEVELOPER=${INCLUDE_DEVELOPER:-false} PANOPTES_UTILS=${PANOPTES_UTILS:-$PANDIR/panoptes-utils} PANOPTES_POCS=${PANOPTES_POCS:-$PANDIR/POCS} -_IMAGE_URL="gcr.io/panoptes-exp/panoptes-utils:latest" +_UTILS_IMAGE_URL="gcr.io/panoptes-exp/panoptes-utils:latest" echo "Setting up local environment." cd "${PANOPTES_POCS}" build_utils() { /bin/bash "${PANOPTES_UTILS}/docker/setup-local-environment.sh" - # Use our local image for build below. - _IMAGE_URL="panoptes-utils:develop" + # Use our local image for build below instead of gcr.io image. + _UTILS_IMAGE_URL="panoptes-utils:develop" } build_develop() { - echo "Building local panoptes-pocs:develop from ${_IMAGE_URL} in ${PANOPTES_POCS}" + echo "Building local panoptes-pocs:develop from ${_UTILS_IMAGE_URL} in ${PANOPTES_POCS}" docker build \ - --build-arg="image_url=${_IMAGE_URL}" \ - --build-arg="pip_install=." \ -t "panoptes-pocs:develop" \ -f "${PANOPTES_POCS}/docker/Dockerfile" \ "${PANOPTES_POCS}" } build_developer() { - echo "Building local panoptes-pocs:developer from ${_IMAGE_URL} in ${PANOPTES_POCS}" + echo "Building local panoptes-pocs:developer from panoptes-pocs:develop in ${PANOPTES_POCS}" docker build \ -t "panoptes-pocs:developer" \ - -f "${PANOPTES_POCS}/docker/developer.Dockerfile" \ + -f "${PANOPTES_POCS}/docker/developer/Dockerfile" \ "${PANOPTES_POCS}" } +#################################################################################### +# Script logic below +#################################################################################### + if [ "${INCLUDE_UTILS}" = true ]; then build_utils fi diff --git a/setup.cfg b/setup.cfg index d0c087b35..6a4ea83b5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -46,7 +46,7 @@ install_requires = Flask matplotlib numpy - panoptes-utils>=0.2.23 + panoptes-utils>=0.2.26 pyserial>=3.1.1 PyYaml requests @@ -67,10 +67,9 @@ exclude = # Add here additional requirements for extra features, to install with: # `pip install POCS[PDF]` like: developer = - gcloud + google-cloud-sdk google-cloud-bigquery[pandas,pyarrow] google-cloud-firestore - google-cloud-pubsub google-cloud-storage jupyterlab pandas From ee2b35fbfa066271c2c2f093439588ffeb497c68 Mon Sep 17 00:00:00 2001 From: Wilfred Gee Date: Fri, 21 Aug 2020 09:37:42 -1000 Subject: [PATCH 22/22] Changelog for v0.7.5 --- CHANGELOG.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 2c6c00916..4952f7f30 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,8 +6,8 @@ All notable changes to this project will be documented in this file. The format is based on `Keep a Changelog `__, and this project adheres to `Semantic Versioning `__. -[0.7.5dev] ----------- +[0.7.5] - 2020-08-21 +-------------------- Changed ~~~~~~~