diff --git a/.dockerignore b/.dockerignore index bcb02874b..b296ccfa9 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,16 +1,34 @@ -!.git docs/* -.eggs .idea .venv venv -*.egg-info + +# Pass the git folder to the local build context. +# This is only needed for local builds because of setuptools_scm. +# The cloudbuild version ignores this folder (and creates a smaller image) +# because it installs from pypi rather than local. +!.git .github *.md !README*.md -*.log -*.pdf +logs/ +**/.ipynb_checkpoints +**/.pytest_cache +**/.eggs +**/*.pdf +**/*.log +**/*.egg-info **/*.pyc -**/__pycache__ \ No newline at end of file +**/__pycache__ +notebooks +examples +docs +build +coverage.xml + +# Build and docs folder/files +build/* +dist/* +sdist/* diff --git a/.gcloudignore b/.gcloudignore new file mode 100644 index 000000000..730ffb944 --- /dev/null +++ b/.gcloudignore @@ -0,0 +1,25 @@ +.idea +.venv +venv + +# See note in .dockerignore about the git folder. +.git +.github + +*.md +!README*.md + +logs/ +**/.ipynb_checkpoints +**/.pytest_cache +**/.eggs +**/*.pdf +**/*.log +**/*.egg-info +**/*.pyc +**/__pycache__ +notebooks +examples +docs +build +coverage.xml diff --git a/codecov.yml b/.github/codecov.yml similarity index 96% rename from codecov.yml rename to .github/codecov.yml index 3360aac18..00297b4ae 100644 --- a/codecov.yml +++ b/.github/codecov.yml @@ -7,7 +7,6 @@ ignore: - "pocs/camera/libfliconstants.py" - "pocs/camera/zwo.py" - "pocs/camera/libasi.py" - - "pocs/tests/*" - "pocs/tests/bisque/*" - "pocs/dome/bisque.py" - "pocs/mount/bisque.py" diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml new file mode 100644 index 000000000..75cb5888e --- /dev/null +++ b/.github/workflows/create-release.yml @@ -0,0 +1,48 @@ +# This will create a release for any properly tagged branch. +on: + push: + # Sequence of patterns matched against refs/tags + tags: + - 'v[0-9].[0-9]+.[0-9]+' # Push events to matching vX.Y.Z, but not vX.Y.Zdev + +name: Create GitHub Release + +jobs: + build: + name: Create Release + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Create Release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token + with: + tag_name: ${{ github.ref }} + release_name: Release ${{ github.ref }} + body: | + See Changelog for details + draft: false + prerelease: false + deploy: + name: Push Release to PyPi + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + - name: Build and publish + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + python setup.py sdist bdist_wheel + twine upload dist/* diff --git a/.github/workflows/pythontest.yaml b/.github/workflows/pythontest.yaml index e257653d5..ef806a429 100644 --- a/.github/workflows/pythontest.yaml +++ b/.github/workflows/pythontest.yaml @@ -6,7 +6,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7] + python-version: [3.8] steps: - name: Checkout code uses: actions/checkout@v2 @@ -25,16 +25,14 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7] + python-version: [3.8] steps: - name: Checkout code uses: actions/checkout@v2 - - name: Fetch all history for all tags and branches - run: git fetch --prune --unshallow - - name: Build pocs image + - name: Build panoptes-pocs image run: | - docker build -t pocs:testing -f docker/latest.Dockerfile . - - name: Test with pytest in pocs container + PANOPTES_POCS=. INCLUDE_UTILS=false scripts/setup-local-environment.sh + - name: Test with pytest in panoptes-pocs container run: | mkdir -p coverage_dir && chmod 777 coverage_dir ci_env=`bash <(curl -s https://codecov.io/env)` @@ -42,10 +40,10 @@ jobs: $ci_env \ -e REPORT_FILE="/tmp/coverage/coverage.xml" \ --network "host" \ - -v $PWD:/var/panoptes/logs \ + -v $PWD/coverage_dir:/var/panoptes/logs \ -v $PWD/coverage_dir:/tmp/coverage \ - pocs:testing \ - scripts/testing/run-tests.sh + panoptes-pocs:develop \ + "/var/panoptes/POCS/scripts/testing/run-tests.sh" - name: Upload coverage report to codecov.io uses: codecov/codecov-action@v1 if: success() @@ -58,4 +56,4 @@ jobs: if: always() with: name: log-files - path: panoptes-testing.log + path: coverage_dir/panoptes-testing.log diff --git a/.gitignore b/.gitignore index bcbbc904e..7dd8d4802 100644 --- a/.gitignore +++ b/.gitignore @@ -1,14 +1,14 @@ # Temporary and binary files *~ -*.py[cod] *.so *.cfg !.isort.cfg !setup.cfg *.orig -*.log *.pot -__pycache__/* +**/*.py[cod] +**/*.log +**/__pycache__/* .cache/* .*.swp */.ipynb_checkpoints/* @@ -34,6 +34,7 @@ htmlcov/* .tox junit.xml coverage.xml +.coverage.* .pytest_cache/ # Build and docs folder/files diff --git a/.readthedocs.yml b/.readthedocs.yml index f4a658576..087a8e040 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -12,7 +12,7 @@ sphinx: formats: all python: - version: 3.7 + version: 3.8 install: - requirements: docs/requirements.txt - method: pip diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 1f010372e..000000000 --- a/.travis.yml +++ /dev/null @@ -1,18 +0,0 @@ -dist: xenial -sudo: required -language: python -python: - - "3.6" -services: -- docker -before_install: -- docker pull gcr.io/panoptes-exp/pocs:latest -- ci_env=`bash <(curl -s https://codecov.io/env)` -install: true -script: -- docker run -it - $ci_env - -e LOCAL_USER_ID=0 - -v $TRAVIS_BUILD_DIR:/var/panoptes/POCS - gcr.io/panoptes-exp/pocs:latest - scripts/testing/run-tests.sh diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a048c3f94..4952f7f30 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,17 +6,86 @@ All notable changes to this project will be documented in this file. The format is based on `Keep a Changelog `__, and this project adheres to `Semantic Versioning `__. +[0.7.5] - 2020-08-21 +-------------------- + +Changed +~~~~~~~ + +* Dependency updates: + + * `panoptes-utils` to `0.2.26`. (#995) + * `panoptes-utils` to `0.2.21`. (#979) + * `panoptes-utils` to `0.2.20`. (#974) + +* Install script. (#974) + + * Env var file is sourced for zshrc and bashrc. + * Fix the clone of the repos in install script. (#978) + * Adding a date version to script. (#979) + * `docker-compose` version bumped to `1.26.2`. (#979) + * Better testing for ssh access. (#984) + * Using `linuxserver.io docker-compose `_ so we also have `arm` version without work. (#986) + * Fixing conditional so script can proceed without restart. (#986) + * Generalizing install script in sections. (#986) + +* Development Environment (#974) + + * Many cleanups to environment and launch. See docs. + * Config server started along with development environment. + * Docker images and python packages are now automated via GitHub Actions and Google Cloud Build. (#995) + +* Docker image updates (#972) + + * Updated `install-pocs.sh` script. + * ``latest`` installs the ``panoptes-pocs`` module from pip + * ``develop`` installs via ``pip install -e[google.testing]`` and is used for running the CI tests. + * ``developer-env`` installs locally but with all options, i.e. ``pip install -e[google,testing,plotting,developer]``. Also builds ``jupyterlab`` and other developer tools. Starts a ``jupyterlab`` instance by default. + * Use new ``arduino-cli`` installer. + * Add ``bin/panoptes-develop`` and ``bin/wait-for-it.sh`` to installed scripts. + * Add ``docker/setup-local-environment.sh``, a convenience script for building local images. + * Python moved to 3.8. (#974) + * Docker images are now built with buildx to get an arm version running. (#978) + * Removing readline and pendulum dependencies. (#978) + * Fully automated build and release of packages with GitHub Actions. (#995) + +* Testing (#974) + + * Removing all the dynamic config server info, making things a lot simpler. + * `docker-compose` files for running tests. + * Misc documentation updates. + * Code coverage no longer ignores test. + * Testing is run via `panoptes-develop test`. + * Log files are rotated during each run. + +* POCS (#974) + + * POCS instance cannot `initialize` unless it's `observatory.can_observe`. + * Set `simulator` config item at start of `POCS` init method if `simulators` (note plural) is passed. + * Simplification of the `run` method and the various predicates used to control it. Now just use the computed `keep_running`. + * Adding some action flags to the `pocs.yaml` file. + * Remove `POCS.check_environment` class method. + * Add a `console_log_level` and `stderr_log_level`. The former is written to the log file in `$PANLOG` and is meant to be tailed in the console. The `stderr_log_level` is what would be displayed, e.g. in a jupyter notebook. (#977) + * Mount simulator better name and stringify. (#977) + * Global db object for `PanBase` (#977) + * Allow for custom folder for metadata. (#979) + * Default changed to `metadata`. + +* Camera simulator cleanup. (#974) +* Scheduler (#974) + + * The `fields_file` is read when scheduler is created. [0.7.4] - 2020-05-31 ----------- +-------------------- -Note that we skipped `0.7.2` and `0.7.3`. +Note that we skipped ``0.7.2`` and ``0.7.3``. Bug fixes ~~~~~~~~~ -* Package name is `panoptes-pocs` for namespace consistency. (#971) +* Package name is ``panoptes-pocs`` for namespace consistency. (#971) * README changed to rst. (#971) @@ -47,7 +116,9 @@ Added * Storing an explicit ``safety`` collection in the database. * Configuration file specific for testing rather than relying on ``pocs.yaml``. * Convenience scripts for running tests inside docker container: - ``scripts/testing/test-software.sh`` + + ``scripts/testing/test-software.sh`` + * GitHub Actions for testing and coverage upload. Changed @@ -55,19 +126,25 @@ Changed * Docker as default. (#951). * Weather items have moved to `aag-weather `__. - * Two docker containers run from the ``aag-weather`` image and have a ``docker/docker-compose-aag.yaml`` file to start. + + * Two docker containers run from the ``aag-weather`` image and have a ``docker/docker-compose-aag.yaml`` file to start. + * Config items related to the configuration system have been moved to the `Config Server `__ in ``panoptes-utils`` repo. - * The main interface for POCS related items is through ``self.get_config``, which can take a key and a default, e.g. ``self.get_config('mount.horizon', default='30 deg')``. - * Test writing is affected and is currently more difficult than would be ideal. An updated test writing document will be following this release. + + * The main interface for POCS related items is through ``self.get_config``, which can take a key and a default, e.g. ``self.get_config('mount.horizon', default='30 deg')``. + * Test writing is affected and is currently more difficult than would be ideal. An updated test writing document will be following this release. * Logging has changed to `loguru `__ and has been greatly simplified: - * ``get_root_logger`` has been replaced by ``get_logger``. + + * ``get_root_logger`` has been replaced by ``get_logger``. + * The ``per-run`` logs have been removed and have been replaced by two logs files: - * ``$PANDIR/logs/panoptes.log``: Log file meant for watching on the + + * ``$PANDIR/logs/panoptes.log``: Log file meant for watching on the command line (via ``tail``) or for otherwise human-readable logs. Rotated daily at 11:30 am. Only the previous days' log is retained. - * ``$PANDIR/logs/panoptes_YYYYMMDD.log``: Log file meant for archive + * ``$PANDIR/logs/panoptes_YYYYMMDD.log``: Log file meant for archive or information gathering. Stored in JSON format for ingestion into log analysis service. Rotated daily at 11:30 and stored in a compressed file for 7 days. Future updates will add option to @@ -159,11 +236,9 @@ Removed [0.6.1] - 2018-09-20 -------------------- -| Lots of changes in this release. In particular we've pushed through a -lot of changes -| (especially with the help of @jamessynge) to make the development -process a lot -| smoother. This has in turn contribute to the quality of the codebase. +* Lots of changes in this release. In particular we've pushed through a lot of changes +* (especially with the help of @jamessynge) to make the development process a lot +* smoother. This has in turn contribute to the quality of the codebase. Too long between releases but even more exciting improvements to come! Next up is tackling the events notification system, which will let us @@ -192,11 +267,9 @@ Changed ~~~~~~~ * Mount -* POCS Shell: Hitting ``Ctrl-c`` will complete movement through states - [#590]. +* POCS Shell: Hitting ``Ctrl-c`` will complete movement through states [#590]. * Pointing updates, including ``auto_correct`` [#580]. -* Tracking mode updates (**fixes for Northern Hemisphere only!**) - [#549]. +* Tracking mode updates (**fixes for Northern Hemisphere only!**) [#549]. * Serial interaction improvements [#388, #403]. * Shutdown improvements [#407, #421]. * Dome diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index e95bb6ca8..1cf8cafb5 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -182,12 +182,12 @@ In order to test your installation you should have followed all of the steps abo for getting your unit ready. To run the test suite, you will need to open a terminal and navigate to the ``$POCS`` directory. -.. code:: bash +.. code-block:: bash cd $POCS # Run the software testing - scripts/testing/test-software.sh + panoptes-develop test .. note:: @@ -197,7 +197,7 @@ and navigate to the ``$POCS`` directory. It is often helpful to view the log output in another terminal window while the test suite is running: -.. code:: bash +.. code-block:: bash # Follow the log file tail -F $PANDIR/logs/panoptes.log @@ -215,7 +215,7 @@ to github. This can be done either by running the entire test suite as above or by running an individual test related to the code you are changing. For instance, to test the code related to the cameras one can run: -.. code:: bash +.. code-block:: bash pytest -xv pocs/tests/test_camera.py @@ -256,7 +256,7 @@ and ``weather``. Optionally you can use ``all`` to test a fully connected unit. connected but does not test the safety conditions. It is assumed that hardware testing is always done with direct supervision. -.. code:: bash +.. code-block:: bash # Test an attached camera pytest --with-hardware=camera diff --git a/README.md b/README.md new file mode 100644 index 000000000..d79ac3841 --- /dev/null +++ b/README.md @@ -0,0 +1,90 @@ +Welcome to POCS documentation! +============================== + +

+PAN001 +

+
+ +[![GHA Status](https://img.shields.io/endpoint.svg?url=https%3A%2F%2Factions-badge.atrox.dev%2Fpanoptes%2FPOCS%2Fbadge%3Fref%3Ddevelop&style=flat)](https://actions-badge.atrox.dev/panoptes/POCS/goto?ref=develop) [![Travis Status](https://travis-ci.com/panoptes/POCS.svg?branch=develop)](https://travis-ci.com/panoptes/POCS) [![codecov](https://codecov.io/gh/panoptes/POCS/branch/develop/graph/badge.svg)](https://codecov.io/gh/panoptes/POCS) [![Documentation Status](https://readthedocs.org/projects/pocs/badge/?version=latest)](https://pocs.readthedocs.io/en/latest/?badge=latest) [![PyPI version](https://badge.fury.io/py/panoptes-pocs.svg)](https://badge.fury.io/py/panoptes-pocs) + +# Project PANOPTES + +[PANOPTES](https://www.projectpanoptes.org) is an open source citizen science project +designed to find [transiting exoplanets](https://spaceplace.nasa.gov/transits/en/) with +digital cameras. The goal of PANOPTES is to establish a global network of of robotic +cameras run by amateur astronomers and schools (or anyone!) in order to monitor, +as continuously as possible, a very large number of stars. For more general information +about the project, including the science case and resources for interested individuals, see the +[project overview](https://projectpanoptes.org/articles/). + +# POCS + +POCS (PANOPTES Observatory Control System) is the main software driver for a +PANOPTES unit, responsible for high-level control of the unit. + +For more information, see the full documentation at: https://pocs.readthedocs.io. + +[`panoptes-utils`](https://www.github.com/panoptes/panoptes-utils) is a related repository and POCS +relies on most of the tools within `panoptes-utils`. + +## Install + +### POCS Environment + +If you are running a PANOPTES unit then you will most likely want an entire PANOPTES environment. + +There is a bash shell script that will install an entire working POCS system on your computer. Some +folks even report that it works on a Mac. + +The script will ask if you want to install in "developer" mode or not. If so, you should fork this repo, [panoptes-utils](https://github.com/panoptes/panoptes-utils), and [panoptes-tutorials](https://github.com/panoptes/panoptes-tutorials), +and then give your github username when prompted. + +The non-developer mode of the script is intended for PANOPTES units. + + +To install POCS via the script, open a terminal and enter: + +```bash +curl -fsSL https://install.projectpanoptes.org > install-pocs.sh +bash install-pocs.sh +``` + +Or using `wget`: + +```bash +wget -qO- https://install.projectpanoptes.org > install-pocs.sh +bash install-pocs.sh +``` + + +### POCS Module + +If you want just the POCS module, for instance if you want to override it in +your own OCS (see [Huntsman-POCS](https://github.com/AstroHuntsman/huntsman-pocs) +for an example), then install via `pip`: + +```bash +pip install panoptes-pocs +``` + +If you want the extra features, such as Google Cloud Platform connectivity, then +use the extras options: + +```bash +pip install "panoptes-pocs[google,testing]" +``` + +See the full documentation at: https://pocs.readthedocs.io + +### For helping develop POCS software + +See [Coding in PANOPTES](https://github.com/panoptes/POCS/wiki/Coding-in-PANOPTES) + +Links +----- + +- PANOPTES Homepage: https://www.projectpanoptes.org +- Forum: https://forum.projectpanoptes.org +- Documentation: https://pocs.readthedocs.io +- Source Code: https://github.com/panoptes/POCS diff --git a/bin/panoptes-develop b/bin/panoptes-develop new file mode 100755 index 000000000..8958a938c --- /dev/null +++ b/bin/panoptes-develop @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +set -e + +SUBCMD=$1 +PARAMS=${@:2} + +export PANDIR=${PANDIR:-/var/panoptes} +export IMAGE="${IMAGE:-panoptes-pocs}" +export TAG="${TAG:-developer}" + +cd "${PANDIR}" + +## Add the daemon option by default. +if [[ "${SUBCMD}" == "up" ]]; then + export CONTAINER_NAME="pocs-developer" + export COMPOSE_FILE="${PANDIR}/POCS/docker/docker-compose-developer.yaml" +fi + +# Pass any other cli args to the containers as an env var named CLI_ARGS +CLI_ARGS=("${@:2}") + +# We use a docker container for docker-compose, so we need to pass the env vars to +# that container so it can properly place them in the docker-compose file. +export DOCKER_RUN_OPTIONS="${DOCKER_RUN_OPTIONS:--e IMAGE=${IMAGE} -e TAG=${TAG} -e CONTAINER_NAME=${CONTAINER_NAME} -e CLI_ARGS=\"${CLI_ARGS}\"}" + +# Run the docker-compose command with user params. +eval "DOCKER_RUN_OPTIONS=\"${DOCKER_RUN_OPTIONS}\" \ + docker-compose \ + --project-directory ${PANDIR} \ + -f ${COMPOSE_FILE} \ + -p panoptes \ + ${SUBCMD} \ + ${PARAMS}" diff --git a/bin/wait-for-it.sh b/bin/wait-for-it.sh new file mode 100755 index 000000000..761bc5cee --- /dev/null +++ b/bin/wait-for-it.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash +# https://github.com/vishnubob/wait-for-it +# Use this script to test if a given TCP host/port are available + +WAITFORIT_cmdname=${0##*/} + +echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } + +usage() +{ + cat << USAGE >&2 +Usage: + $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args] + -h HOST | --host=HOST Host or IP under test + -p PORT | --port=PORT TCP port under test + Alternatively, you specify the host and port as host:port + -s | --strict Only execute subcommand if the test succeeds + -q | --quiet Don't output any status messages + -t TIMEOUT | --timeout=TIMEOUT + Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit 1 +} + +wait_for() +{ + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + else + echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout" + fi + WAITFORIT_start_ts=$(date +%s) + while : + do + if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then + nc -z $WAITFORIT_HOST $WAITFORIT_PORT + WAITFORIT_result=$? + else + (echo > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1 + WAITFORIT_result=$? + fi + if [[ $WAITFORIT_result -eq 0 ]]; then + WAITFORIT_end_ts=$(date +%s) + echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds" + break + fi + sleep 1 + done + return $WAITFORIT_result +} + +wait_for_wrapper() +{ + # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 + if [[ $WAITFORIT_QUIET -eq 1 ]]; then + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + else + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + fi + WAITFORIT_PID=$! + trap "kill -INT -$WAITFORIT_PID" INT + wait $WAITFORIT_PID + WAITFORIT_RESULT=$? + if [[ $WAITFORIT_RESULT -ne 0 ]]; then + echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + fi + return $WAITFORIT_RESULT +} + +# process arguments +while [[ $# -gt 0 ]] +do + case "$1" in + *:* ) + WAITFORIT_hostport=(${1//:/ }) + WAITFORIT_HOST=${WAITFORIT_hostport[0]} + WAITFORIT_PORT=${WAITFORIT_hostport[1]} + shift 1 + ;; + --child) + WAITFORIT_CHILD=1 + shift 1 + ;; + -q | --quiet) + WAITFORIT_QUIET=1 + shift 1 + ;; + -s | --strict) + WAITFORIT_STRICT=1 + shift 1 + ;; + -h) + WAITFORIT_HOST="$2" + if [[ $WAITFORIT_HOST == "" ]]; then break; fi + shift 2 + ;; + --host=*) + WAITFORIT_HOST="${1#*=}" + shift 1 + ;; + -p) + WAITFORIT_PORT="$2" + if [[ $WAITFORIT_PORT == "" ]]; then break; fi + shift 2 + ;; + --port=*) + WAITFORIT_PORT="${1#*=}" + shift 1 + ;; + -t) + WAITFORIT_TIMEOUT="$2" + if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi + shift 2 + ;; + --timeout=*) + WAITFORIT_TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + WAITFORIT_CLI=("$@") + break + ;; + --help) + usage + ;; + *) + echoerr "Unknown argument: $1" + usage + ;; + esac +done + +if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then + echoerr "Error: you need to provide a host and port to test." + usage +fi + +WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15} +WAITFORIT_STRICT=${WAITFORIT_STRICT:-0} +WAITFORIT_CHILD=${WAITFORIT_CHILD:-0} +WAITFORIT_QUIET=${WAITFORIT_QUIET:-0} + +# check to see if timeout is from busybox? +WAITFORIT_TIMEOUT_PATH=$(type -p timeout) +WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH) +if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then + WAITFORIT_ISBUSY=1 + WAITFORIT_BUSYTIMEFLAG="-t" + +else + WAITFORIT_ISBUSY=0 + WAITFORIT_BUSYTIMEFLAG="" +fi + +if [[ $WAITFORIT_CHILD -gt 0 ]]; then + wait_for + WAITFORIT_RESULT=$? + exit $WAITFORIT_RESULT +else + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + wait_for_wrapper + WAITFORIT_RESULT=$? + else + wait_for + WAITFORIT_RESULT=$? + fi +fi + +if [[ $WAITFORIT_CLI != "" ]]; then + if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then + echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess" + exit $WAITFORIT_RESULT + fi + exec "${WAITFORIT_CLI[@]}" +else + exit $WAITFORIT_RESULT +fi diff --git a/conf_files/pocs.yaml b/conf_files/pocs.yaml index 5be6c3e56..e062a5ec4 100644 --- a/conf_files/pocs.yaml +++ b/conf_files/pocs.yaml @@ -8,68 +8,72 @@ # files and communicate with the Google Cloud network. # # Leave the pan_id at `PAN000` for testing until you have been assigned -# an official id. Update pocs_local.yaml with offical name once received. +# an official id. Update pocs_local.yaml with official name once received. ################################################################################ name: Generic PANOPTES Unit pan_id: PAN000 location: - name: Mauna Loa Observatory - latitude: 19.54 deg - longitude: -155.58 deg - elevation: 3400.0 m - horizon: 30 deg # targets must be above this to be considered valid. - flat_horizon: -6 deg # Flats when sun between this and focus horizon. - focus_horizon: -12 deg # Dark enough to focus on stars. - observe_horizon: -18 deg # Sun below this limit to observe. - obsctructions: [] - timezone: US/Hawaii - gmt_offset: -600 # Offset in minutes from GMT during. - # standard time (not daylight saving). + name: Mauna Loa Observatory + latitude: 19.54 deg + longitude: -155.58 deg + elevation: 3400.0 m + horizon: 30 deg # targets must be above this to be considered valid. + flat_horizon: -6 deg # Flats when sun between this and focus horizon. + focus_horizon: -12 deg # Dark enough to focus on stars. + observe_horizon: -18 deg # Sun below this limit to observe. + obsctructions: [] + timezone: US/Hawaii + gmt_offset: -600 # Offset in minutes from GMT during. + directories: - base: /var/panoptes - images: images - data: data - resources: POCS/resources/ - targets: POCS/resources/targets - mounts: POCS/resources/mounts + base: /var/panoptes + images: images + data: data + resources: POCS/resources/ + targets: POCS/resources/targets + mounts: POCS/resources/mounts + db: - name: panoptes - type: file + name: panoptes + type: file + folder: metadata + +wait_delay: 180 # time in seconds before checking safety/etc while waiting. +max_transition_attempts: 5 # number of transitions attempts. +status_check_interval: 60 # periodic status check. state_machine: simple_state_table scheduler: - type: dispatch - fields_file: simple.yaml - check_file: False + type: dispatch + fields_file: simple.yaml + check_file: False mount: - brand: ioptron - model: 30 - driver: ioptron - serial: - port: /dev/ttyUSB0 - timeout: 0. - baudrate: 9600 - non_sidereal_available: True - min_tracking_threshold: 100 # ms - max_tracking_threshold: 99999 # ms + brand: ioptron + model: 30 + driver: ioptron + serial: + port: /dev/ttyUSB0 + timeout: 0. + baudrate: 9600 + non_sidereal_available: True + min_tracking_threshold: 100 # ms + max_tracking_threshold: 99999 # ms pointing: - auto_correct: True - threshold: 100 # arcseconds ~ 10 pixels - exptime: 30 # seconds - max_iterations: 5 + auto_correct: True + threshold: 100 # arcseconds ~ 10 pixels + exptime: 30 # seconds + max_iterations: 5 cameras: - auto_detect: True - primary: 14d3bd - devices: - - - model: canon_gphoto2 - - - model: canon_gphoto2 + auto_detect: True + primary: 14d3bd + devices: + - model: canon_gphoto2 + - model: canon_gphoto2 ######################### Environmental Sensors ################################ # Configure the environmental sensors that are attached. @@ -82,13 +86,13 @@ cameras: # serial_port: /dev/ttyACM1 ################################################################################ environment: - auto_detect: False - camera_board: - serial_port: /dev/ttyACM0 - control_board: - serial_port: /dev/ttyACM1 - weather: - url: http://localhost:5000/latest.json + auto_detect: False + camera_board: + serial_port: /dev/ttyACM0 + control_board: + serial_port: /dev/ttyACM1 + weather: + url: http://localhost:5000/latest.json ########################## Observations ######################################## @@ -107,8 +111,8 @@ environment: # TODO: Add options for cleaning up old data (e.g. >30 days) ################################################################################ observations: - make_timelapse: True - keep_jpgs: True + make_timelapse: True + keep_jpgs: True ######################## Google Network ######################################## # By default all images are stored on googlecloud servers and we also @@ -121,8 +125,19 @@ observations: # service_account_key: Location of the JSON service account key. ################################################################################ panoptes_network: - image_storage: False - service_account_key: # Location of JSON account key - project_id: panoptes-survey - buckets: - images: panoptes-survey + image_storage: False + service_account_key: # Location of JSON account key + project_id: panoptes-survey + buckets: + images: panoptes-survey + +############################### pocs ################################## +# POCS status flags. The values below represent initial values but +# they can be switched at run-time if needed. +####################################################################### +pocs: + INITIALIZED: false + INTERRUPTED: false + KEEP_RUNNING: true + DO_STATES: true + RUN_ONCE: false \ No newline at end of file diff --git a/conftest.py b/conftest.py index 7ebaf58ac..73df248e6 100644 --- a/conftest.py +++ b/conftest.py @@ -2,21 +2,17 @@ import os import stat import pytest -from _pytest.logging import caplog as _caplog -import subprocess import time import tempfile import shutil - from contextlib import suppress -from multiprocessing import Process -from scalpl import Cut +from _pytest.logging import caplog as _caplog from panoptes.pocs import hardware from panoptes.utils.database import PanDB -from panoptes.utils.config import load_config +from panoptes.utils.config.client import get_config from panoptes.utils.config.client import set_config -from panoptes.utils.config.server import app as config_server_app +from panoptes.utils.config.server import config_server from panoptes.pocs.utils.logger import get_logger, PanLogger @@ -33,13 +29,17 @@ os.getenv('PANLOG', '/var/panoptes/logs'), 'panoptes-testing.log' ) +startup_message = ' STARTING NEW PYTEST RUN ' logger.add(log_file_path, - format=LOGGER_INFO.format, - colorize=True, enqueue=True, # multiprocessing + colorize=True, backtrace=True, diagnose=True, + catch=True, + # Start new log file for each testing run. + rotation=lambda msg, _: startup_message in msg, level='TRACE') +logger.log('testing', '*' * 25 + startup_message + '*' * 25) # Make the log file world readable. os.chmod(log_file_path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) @@ -58,16 +58,12 @@ def pytest_addoption(parser): nargs='+', default=[], help=f"A comma separated list of hardware to NOT test. List items can include: {hw_names}") - group.addoption( - "--solve", - action="store_true", - default=False, - help="If tests that require solving should be run") group.addoption( "--test-databases", nargs="+", default=['file'], - help=f"Test databases in the list. List items can include: {db_names}. Note that travis-ci will test all of " + help=f"Test databases in the list. List items can include: {db_names}. Note that " + f"travis-ci will test all of " f"them by default.") @@ -96,11 +92,11 @@ def pytest_collection_modifyitems(config, items): # User does not want to run tests that interact with hardware called name, # whether it is marked as with_name or without_name. if name in with_hardware: - print('Warning: {!r} in both --with-hardware and --without-hardware'.format(name)) + print(f'Warning: {name} in both --with-hardware and --without-hardware') with_hardware.remove(name) - skip = pytest.mark.skip(reason="--without-hardware={} specified".format(name)) - with_keyword = 'with_' + name - without_keyword = 'without_' + name + skip = pytest.mark.skip(reason=f"--without-hardware={name} specified") + with_keyword = f'with_{name}' + without_keyword = f'without_{name}' for item in items: if with_keyword in item.keywords or without_keyword in item.keywords: item.add_marker(skip) @@ -108,7 +104,7 @@ def pytest_collection_modifyitems(config, items): for name in hardware.get_all_names(without=with_hardware): # We don't have hardware called name, so find all tests that need that # hardware and mark it to be skipped. - skip = pytest.mark.skip(reason="Test needs --with-hardware={} option to run".format(name)) + skip = pytest.mark.skip(reason=f"Test needs --with-hardware={name} option to run") keyword = 'with_' + name for item in items: if keyword in item.keywords: @@ -147,137 +143,48 @@ def pytest_runtest_logreport(report): """Adds the failure info that pytest prints to stdout into the log.""" if report.skipped or report.outcome != 'failed': return - try: + with suppress(Exception): logger.log('testing', '') - logger.log('testing', f' TEST {report.nodeid} FAILED during {report.when} {report.longreprtext} ') + logger.log('testing', + f' TEST {report.nodeid} FAILED during {report.when} {report.longreprtext} ') if report.capstdout: - logger.log('testing', f'============ Captured stdout during {report.when} {report.capstdout} ============') + logger.log('testing', + f'============ Captured stdout during {report.when} {report.capstdout} ' + f'============') if report.capstderr: - logger.log('testing', f'============ Captured stdout during {report.when} {report.capstderr} ============') - except Exception: - pass - - -@pytest.fixture(scope='session') -def db_name(): - return 'panoptes_testing' - - -@pytest.fixture(scope='session') -def images_dir(tmpdir_factory): - directory = tmpdir_factory.mktemp('images') - return str(directory) - - -@pytest.fixture(scope='session') -def config_host(): - return 'localhost' - - -@pytest.fixture(scope='session') -def static_config_port(): - """Used for the session-scoped config_server where no config values - are expected to change during testing. - """ - return '6563' - - -@pytest.fixture(scope='module') -def config_port(): - """Used for the function-scoped config_server when it is required to change - config values during testing. See `dynamic_config_server` docs below. - """ - return '4861' + logger.log('testing', + f'============ Captured stdout during {report.when} {report.capstderr} ' + f'============') @pytest.fixture(scope='session') def config_path(): - return os.path.join(os.getenv('POCS'), 'tests', 'pocs_testing.yaml') + return os.path.expandvars('${POCS}/tests/pocs_testing.yaml') -@pytest.fixture(scope='session') -def config_server_args(config_path): - loaded_config = load_config(config_files=config_path, ignore_local=True) - return { - 'config_file': config_path, - 'auto_save': False, - 'ignore_local': True, - 'POCS': loaded_config, - 'POCS_cut': Cut(loaded_config) - } - - -def make_config_server(config_host, config_port, config_server_args, images_dir, db_name): - def start_config_server(): - # Load the config items into the app config. - for k, v in config_server_args.items(): - config_server_app.config[k] = v - - # Start the actual flask server. - config_server_app.run(host=config_host, port=config_port) +@pytest.fixture(scope='module', autouse=True) +def static_config_server(config_path, images_dir, db_name): + logger.log('testing', f'Starting static_config_server for testing session') - proc = Process(target=start_config_server) - proc.start() + proc = config_server( + config_path, + ignore_local=True, + auto_save=False + ) - logger.log('testing', f'config_server started with PID={proc.pid}') + logger.log('testing', f'static_config_server started with {proc.pid=}') # Give server time to start - time.sleep(1) - - # Adjust various config items for testing - unit_name = 'Generic PANOPTES Unit' - unit_id = 'PAN000' - logger.log('testing', f'Setting testing name and unit_id to {unit_id}') - set_config('name', unit_name, port=config_port) - set_config('pan_id', unit_id, port=config_port) - - logger.log('testing', f'Setting testing database to {db_name}') - set_config('db.name', db_name, port=config_port) - - fields_file = 'simulator.yaml' - logger.log('testing', f'Setting testing scheduler fields_file to {fields_file}') - set_config('scheduler.fields_file', fields_file, port=config_port) - - # TODO(wtgee): determine if we need separate directories for each module. - logger.log('testing', f'Setting temporary image directory for testing') - set_config('directories.images', images_dir, port=config_port) - - # Make everything a simulator - simulators = hardware.get_simulator_names(simulator=['all']) - logger.log('testing', f'Setting all hardware to use simulators: {simulators}') - set_config('simulator', simulators, port=config_port) - - return proc - - -@pytest.fixture(scope='session', autouse=True) -def static_config_server(config_host, static_config_port, config_server_args, images_dir, db_name): - logger.log('testing', f'Starting config_server for testing session') - proc = make_config_server(config_host, static_config_port, config_server_args, images_dir, db_name) - yield proc - pid = proc.pid - proc.terminate() - time.sleep(0.1) - logger.log('testing', f'Killed config_server started with PID={pid}') - + while get_config('name') is None: # pragma: no cover + logger.log('testing', f'Waiting for static_config_server {proc.pid=}, sleeping 1 second.') + time.sleep(1) -@pytest.fixture(scope='function') -def dynamic_config_server(config_host, config_port, config_server_args, images_dir, db_name): - """If a test requires changing the configuration we use a function-scoped testing - server. We only do this on tests that require it so we are not constantly starting and stopping - the config server unless necessary. To use this, each test that requires it must use the - `dynamic_config_server` and `config_port` fixtures and must pass the `config_port` to all - instances that are created (propagated through PanBase). - """ + set_config('directories.images', images_dir) - logger.log('testing', f'Starting config_server for testing function') - proc = make_config_server(config_host, config_port, config_server_args, images_dir, db_name) - - yield proc - pid = proc.pid + logger.log('testing', f'Startup config_server name=[{get_config("name")}]') + yield + logger.log('testing', f'Killing static_config_server started with PID={proc.pid}') proc.terminate() - time.sleep(0.1) - logger.log('testing', f'Killed config_server started with PID={pid}') @pytest.fixture @@ -289,6 +196,17 @@ def temp_file(tmp_path): f.unlink(missing_ok=True) +@pytest.fixture(scope='session') +def db_name(): + return 'panoptes_testing' + + +@pytest.fixture(scope='session') +def images_dir(tmpdir_factory): + directory = tmpdir_factory.mktemp('images') + return str(directory) + + @pytest.fixture(scope='function', params=_all_databases) def db_type(request, db_name): db_list = request.config.option.test_databases @@ -312,7 +230,7 @@ def memory_db(db_name): @pytest.fixture(scope='session') def data_dir(): - return '/var/panoptes/panoptes-utils/tests/data' + return os.path.expandvars('${POCS}/tests/data') @pytest.fixture(scope='function') diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 000000000..110ddb99e --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,63 @@ +ARG image_url=gcr.io/panoptes-exp/panoptes-utils:latest +FROM ${image_url} AS pocs-base + +LABEL description="Installs the panoptes-pocs module from GitHub. \ +Used as a production image, i.e. for running on PANOPTES units." +LABEL maintainers="developers@projectpanoptes.org" +LABEL repo="github.com/panoptes/POCS" + +ARG panuser=panoptes +ARG userid=1000 +ARG pan_dir=/var/panoptes +ARG pocs_dir="${pan_dir}/POCS" +ARG conda_env_name="panoptes" + +ARG arduino_url="https://raw.githubusercontent.com/arduino/arduino-cli/master/install.sh" +ARG gphoto2_url="https://raw.githubusercontent.com/gonzalo/gphoto2-updater/master/gphoto2-updater.sh" +ARG pip_extras="[testing,google]" + +ENV DEBIAN_FRONTEND=noninteractive +ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 + +ENV PANDIR $pan_dir +ENV PANLOG "$pan_dir/logs" +ENV PANUSER $panuser +ENV POCS $pocs_dir + +# Install system dependencies. +USER root +RUN apt-get update && apt-get install --no-install-recommends --yes \ + gcc \ + gphoto2 \ + ncurses-dev \ + readline-common \ + udev + +# Install program dependencies. +USER ${PANUSER} +WORKDIR ${POCS} +RUN mkdir -p "${PANDIR}/scripts" && \ + cd "${PANDIR}/scripts" && \ + # Install gphoto2 auto-updater + wget $gphoto2_url -O gphoto2-updater.sh && \ + chmod +x gphoto2-updater.sh && \ + # Don't actually update gphoto2 as of right now. + # sudo /bin/bash gphoto2-updater.sh --stable && \ + # Install arduino-cli. + wget -q "${arduino_url}" -O install-arduino-cli.sh && \ + sudo BINDIR="/usr/local/bin" /bin/sh install-arduino-cli.sh && \ + sudo chown -R "${PANUSER}":"${PANUSER}" "${PANDIR}" + +# Install the module. +USER ${PANUSER} +# Can't seem to get around the hard-coding here. +COPY --chown=panoptes:panoptes . . +RUN "${PANDIR}/conda/envs/${conda_env_name}/bin/pip" install -e ".${pip_extras}" && \ + # Cleanup + sudo apt-get autoremove --purge --yes && \ + sudo apt-get autoclean --yes && \ + sudo apt-get --yes clean && \ + sudo rm -rf /var/lib/apt/lists/* && \ + "${PANDIR}/conda/bin/conda" clean -tipy + +USER root diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 000000000..dce637c34 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,38 @@ +Docker Images +============= + +POCS is available as a docker image hosted on Google Cloud Registry (GCR): + +Image name: `gcr.io/panoptes-exp/panoptes-pocs:latest` + +### `develop` image + +To build the images locally: + +```bash +scripts/setup-local-environment.sh +``` + +This will build all required images locally and is suitable for testing and development. + +Then, to run the test suite locally: + +```bash +scripts/testing/test-software.sh +```` + +### `developer` image + +The `developer` image is meant to be be used by developers or anyone wishing to +explore the code. It is the same as the local `develop`, but also installs additional +plotting libraries and the `jupyter` environment. + +The image should be built locally using the `docker/setup-local-environment.sh` +script (see above). + +The `bin/panoptes-develop up` can then be used to start a docker container +instance that will launch `jupyter lab` from `$PANDIR` automatically. + +```bash +bin/panoptes-develop up +``` diff --git a/docker/build-image.sh b/docker/build-image.sh deleted file mode 100755 index ee8510ab2..000000000 --- a/docker/build-image.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -e - -SOURCE_DIR="${PANDIR}/POCS" -BASE_CLOUD_FILE="cloudbuild.yaml" -TAG="${1:-develop}" - -cd "${SOURCE_DIR}" - -echo "Building gcr.io/panoptes-exp/pocs:${TAG}" -gcloud builds submit \ - --timeout="1h" \ - --substitutions="_TAG=${TAG}" \ - --config "${SOURCE_DIR}/docker/${BASE_CLOUD_FILE}" \ - "${SOURCE_DIR}" diff --git a/docker/cloudbuild.yaml b/docker/cloudbuild.yaml index 882d41ebf..90c69bfcc 100644 --- a/docker/cloudbuild.yaml +++ b/docker/cloudbuild.yaml @@ -1,19 +1,66 @@ +options: + machineType: "N1_HIGHCPU_8" + substitutionOption: "ALLOW_LOOSE" +timeout: 18000s # 5 hours + +substitutions: + _PLATFORMS: linux/amd64,linux/arm64 + _BASE_IMAGE: panoptes-utils:latest + _IMAGE_NAME: panoptes-pocs + _REPO_URL: https://github.com/panoptes/POCS + _TAG: latest + steps: -- name: 'docker' - id: 'amd64-build' - args: - - 'build' - - '--build-arg image_url=gcr.io/panoptes-exp/panoptes-utils:${_TAG}' - - '-f=docker/${_TAG}.Dockerfile' - - '--tag=gcr.io/${PROJECT_ID}/panoptes-pocs:${_TAG}' - - '.' + # Fetch the repo from github + - name: gcr.io/cloud-builders/git + id: "clone-repo" + args: ["clone", "${_REPO_URL}"] + waitFor: ["-"] + + # Pull the cached image. + - name: 'gcr.io/cloud-builders/docker' + id: "pull-cached-image" + entrypoint: 'bash' + args: ['-c', 'docker pull gcr.io/${PROJECT_ID}/${_IMAGE_NAME}:${_TAG} || exit 0'] + waitFor: ["-"] + + # Set up multiarch support + - name: "gcr.io/cloud-builders/docker" + id: "setup-buildx" + env: + - "DOCKER_CLI_EXPERIMENTAL=enabled" + args: + - "run" + - "--privileged" + - "--rm" + - "docker/binfmt:a7996909642ee92942dcd6cff44b9b95f08dad64" + waitFor: ["-"] -- name: 'docker' - id: 'amd64-push' - args: - - 'push' - - 'gcr.io/${PROJECT_ID}/panoptes-pocs:${_TAG}' - waitFor: ['amd64-build'] + # Build builder + - name: "gcr.io/cloud-builders/docker" + id: "build-builder" + env: + - "DOCKER_CLI_EXPERIMENTAL=enabled" + args: + - "buildx" + - "create" + - "--use" + - "--driver=docker-container" + waitFor: ["setup-buildx"] -images: - - 'gcr.io/${PROJECT_ID}/panoptes-pocs:${_TAG}' + # Build with cloned panoptes-utils as source directory + - name: "gcr.io/cloud-builders/docker" + id: "build-images" + env: + - "DOCKER_CLI_EXPERIMENTAL=enabled" + args: + - "buildx" + - "build" + - "--push" + - "--platform=${_PLATFORMS}" + - "-f=docker/Dockerfile" + - "--build-arg=image_url=gcr.io/${PROJECT_ID}/${_BASE_IMAGE}" + - "--tag=gcr.io/${PROJECT_ID}/${_IMAGE_NAME}:${_TAG}" + - "--cache-from=gcr.io/${PROJECT_ID}/${_IMAGE_NAME}:${_TAG}" + - "POCS" + waitFor: ["build-builder", "clone-repo"] diff --git a/docker/developer/Dockerfile b/docker/developer/Dockerfile new file mode 100644 index 000000000..4e0fcd78e --- /dev/null +++ b/docker/developer/Dockerfile @@ -0,0 +1,59 @@ +ARG BASE_IMAGE=panoptes-pocs:develop +FROM ${BASE_IMAGE} + +LABEL description="Installs a number of developer tools. Runs jupyter lab instance. \ +This assumes the `panoptes-pocs:develop` has already been built locally." +LABEL maintainers="developers@projectpanoptes.org" +LABEL repo="github.com/panoptes/POCS" + +ARG panuser=panoptes +ARG userid=1000 +ARG pan_dir=/var/panoptes +ARG pocs_dir="${pan_dir}/POCS" +ARG conda_env_name="panoptes" + +ENV DEBIAN_FRONTEND=noninteractive +ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 +ENV SHELL /bin/zsh + +ENV USERID $userid +ENV PANDIR $pan_dir +ENV PANLOG "$pan_dir/logs" +ENV PANUSER $panuser +ENV POCS $pocs_dir + +RUN apt-get update && \ + # Make a developer's life easier. + apt-get install --yes --no-install-recommends \ + bzip2 ca-certificates nano neovim \ + ncdu htop + +USER $PANUSER +RUN echo "Installing developer tools" && \ + "${PANDIR}/conda/bin/conda" install --name "${conda_env_name}" \ + altair \ + bokeh \ + httpie \ + jupyterlab \ + jq \ + holoviews \ + hvplot \ + nodejs \ + seaborn && \ + # Set some jupyterlab defaults. + mkdir -p /home/panoptes/.jupyter && \ + /usr/bin/env zsh -c "${PANDIR}/conda/envs/${conda_env_name}/bin/jupyter-lab --no-browser --generate-config" && \ + # Jupyterlab extensions. + echo "c.JupyterApp.answer_yes = True" >> \ + "/home/panoptes/.jupyter/jupyter_notebook_config.py" && \ + echo "c.JupyterApp.open_browser = False" >> \ + "/home/panoptes/.jupyter/jupyter_notebook_config.py" && \ + echo "c.JupyterApp.notebook_dir = '${PANDIR}'" >> \ + "/home/panoptes/.jupyter/jupyter_notebook_config.py" && \ + # Cleanup + sudo apt-get autoremove --purge --yes && \ + sudo apt-get autoclean --yes && \ + sudo rm -rf /var/lib/apt/lists/* + +USER root +WORKDIR ${PANDIR} \ No newline at end of file diff --git a/docker/developer/docker-compose.yaml b/docker/developer/docker-compose.yaml new file mode 100644 index 000000000..c96ea10d6 --- /dev/null +++ b/docker/developer/docker-compose.yaml @@ -0,0 +1,31 @@ +version: '3.7' +services: + config-server: + image: "panoptes-utils:develop" + init: true + container_name: config-server + privileged: true + network_mode: host + restart: on-failure + volumes: + - pocsdir:/var/panoptes/POCS + command: ["panoptes-config-server run /var/panoptes/POCS/conf_files/pocs.yaml"] + developer: + image: "${IMAGE:-panoptes-pocs}:${TAG:-developer}" + init: true + container_name: "${CONTAINER_NAME:-pocs-developer}" + privileged: true + network_mode: host + depends_on: + - "config-server" + volumes: + - pocsdir:/var/panoptes/POCS + command: ["jupyter lab", "--ip=0.0.0.0"] +volumes: + pocsdir: + driver: local + driver_opts: + type: none + device: /var/panoptes/POCS + o: bind + diff --git a/docker/docker-compose-aag.yaml b/docker/docker-compose-aag.yaml deleted file mode 100644 index fdcdb94f2..000000000 --- a/docker/docker-compose-aag.yaml +++ /dev/null @@ -1,40 +0,0 @@ -version: '3.7' -services: - aag-weather-reader: - image: gcr.io/panoptes-exp/aag-weather:latest - init: true - container_name: aag-weather-reader - privileged: true - network_mode: host - restart: on-failure - volumes: - - pandir:/var/panoptes - command: - - "python" - - "/app/scripts/read-aag.py" - - "--config-file" - - "/var/panoptes/conf_files/pocs_local.yaml" - - "--db-file" - - "/var/panoptes/json_store/panoptes/weather.db" - - "--store-result" - - "--verbose" - aag-weather-server: - image: gcr.io/panoptes-exp/aag-weather:latest - init: true - container_name: aag-weather-server - privileged: true - network_mode: host - environment: - - DB_NAME=/var/panoptes/json_store/panoptes/weather.db - command: ["flask", "run"] - restart: on-failure - volumes: - - pandir:/var/panoptes -volumes: - pandir: - driver: local - driver_opts: - type: none - device: /var/panoptes - o: bind - diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 6acf52339..5cb77c273 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -1,7 +1,7 @@ version: '3.7' services: peas-shell: - image: gcr.io/panoptes-exp/pocs:latest + image: gcr.io/panoptes-exp/panoptes-pocs:latest init: true container_name: peas-shell hostname: peas-shell @@ -19,7 +19,7 @@ services: - "-f" - "/dev/null" pocs-shell: - image: gcr.io/panoptes-exp/pocs:latest + image: gcr.io/panoptes-exp/panoptes-pocs:latest init: true container_name: pocs-shell hostname: pocs-shell diff --git a/docker/latest.Dockerfile b/docker/latest.Dockerfile deleted file mode 100644 index 202d82b3b..000000000 --- a/docker/latest.Dockerfile +++ /dev/null @@ -1,55 +0,0 @@ -ARG image_url=gcr.io/panoptes-exp/panoptes-utils:testing - -FROM $image_url AS pocs-base -LABEL maintainer="developers@projectpanoptes.org" - -ARG pandir=/var/panoptes -ARG arduino_url="https://downloads.arduino.cc/arduino-cli/arduino-cli_latest_Linux_64bit.tar.gz" - -ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 -ENV SHELL /bin/zsh -ENV PANDIR $pandir -ENV POCS ${PANDIR}/POCS -ENV USER panoptes - -RUN apt-get update \ - && apt-get install --no-install-recommends --yes \ - gcc libncurses5-dev udev \ - # GPhoto2 - && wget https://raw.githubusercontent.com/gonzalo/gphoto2-updater/master/gphoto2-updater.sh \ - && chmod +x gphoto2-updater.sh \ - && /bin/bash gphoto2-updater.sh --stable \ - && rm gphoto2-updater.sh \ - # arduino-cli - && wget -q $arduino_url -O arduino-cli.tar.gz \ - # Untar and capture output name (NOTE: assumes only one file). - && tar xvfz arduino-cli.tar.gz \ - && mv arduino-cli /usr/local/bin/arduino-cli \ - && chmod +x /usr/local/bin/arduino-cli - -COPY ./requirements.txt /tmp/requirements.txt -# First deal with pip and PyYAML - see https://github.com/pypa/pip/issues/5247 -RUN pip install --no-cache-dir --no-deps --ignore-installed pip PyYAML && \ - pip install --no-cache-dir -r /tmp/requirements.txt - -# Install module -COPY . ${POCS}/ -RUN cd ${POCS} && pip install -e ".[google]" - -# Cleanup apt. -USER root -RUN apt-get autoremove --purge -y \ - autoconf \ - automake \ - autopoint \ - build-essential \ - gcc \ - gettext \ - libtool \ - pkg-config && \ - apt-get autoremove --purge -y && \ - apt-get -y clean && \ - rm -rf /var/lib/apt/lists/* - -WORKDIR ${POCS} -CMD ["/bin/zsh"] diff --git a/README.rst b/docs/README.rst similarity index 75% rename from README.rst rename to docs/README.rst index fc84cfc28..ed3a61efa 100644 --- a/README.rst +++ b/docs/README.rst @@ -14,7 +14,7 @@ PANOPTES Observatory Control System .. warning:: - The recent `v0.7.0` release of POCS is not backwards compatible. If you + The recent `v0.7.0` (May 2020) release of POCS is not backwards compatible. If you are one of the folks running that software, please either do a reinstall of your system using the instructions below or see our `forum `__ for advice. @@ -23,16 +23,28 @@ PANOPTES Observatory Control System Overview -------- -`PANOPTES `__ is an open source citizen science project -that is designed to find transiting exoplanets with digital cameras. The goal of -PANOPTES is to establish a global network of of robotic cameras run by amateur -astronomers schools in order to monitor, as continuously as possible, a very large -number of stars. For more general information about the project, including the -science case and resources for interested individuals, see the `about page `__. +Project PANOPTES +^^^^^^^^^^^^^^^^ + +`PANOPTES `_ is an open source citizen science project +designed to find `transiting exoplanets `_ with +digital cameras. The goal of PANOPTES is to establish a global network of of robotic +cameras run by amateur astronomers and schools (or anyone!) in order to monitor, +as continuously as possible, a very large number of stars. For more general information +about the project, including the science case and resources for interested individuals, see the +`project overview `_. + +POCS +^^^^ + +POCS (PANOPTES Observatory Control System) is the main software driver for a +PANOPTES unit, responsible for high-level control of the unit. + +For more information, see the full documentation at: https://pocs.readthedocs.io. -POCS (PANOPTES Observatory Control System) is the main software driver for the -PANOPTES unit, responsible for high-level control of the unit. This repository -also contains a number of scripts for running a full instance of POCS. +`panoptes-utils `_ is a related repository and POCS +relies on most of the tools within `panoptes-utils`. See https://panoptes-pocs.readthedocs.io for +more information. Getting Started --------------- diff --git a/docs/conf.py b/docs/conf.py index 0476a5711..69f6f58a2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -8,10 +8,10 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import os -import sys import inspect +import os import shutil +import sys __location__ = os.path.join(os.getcwd(), os.path.dirname( inspect.getfile(inspect.currentframe()))) @@ -271,17 +271,27 @@ # -- External mapping ------------------------------------------------------------ python_version = '.'.join(map(str, sys.version_info[0:2])) intersphinx_mapping = { - 'sphinx': ('http://www.sphinx-doc.org/en/stable', None), + 'sphinx': ('https://www.sphinx-doc.org/en/stable', None), 'python': ('https://docs.python.org/' + python_version, None), 'matplotlib': ('https://matplotlib.org', None), 'numpy': ('https://docs.scipy.org/doc/numpy', None), - 'sklearn': ('http://scikit-learn.org/stable', None), - 'pandas': ('http://pandas.pydata.org/pandas-docs/stable', None), + 'sklearn': ('https://scikit-learn.org/stable', None), + 'pandas': ('https://pandas.pydata.org/pandas-docs/stable', None), 'scipy': ('https://docs.scipy.org/doc/scipy/reference', None), - 'astropy': ('http://docs.astropy.org/en/stable/', None), + 'astropy': ('https://docs.astropy.org/en/stable/', None), 'astroplan': ('https://astroplan.readthedocs.io/en/latest/', None), 'panoptes.utils': ('https://panoptes-utils.readthedocs.io/en/latest/', None), } # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = True + + +def skip(app, what, name, obj, would_skip, options): + if name == "__init__": + return False + return would_skip + + +def setup(app): + app.connect("autodoc-skip-member", skip) diff --git a/docs/docker.rst b/docs/docker.rst new file mode 100644 index 000000000..d7c26ae46 --- /dev/null +++ b/docs/docker.rst @@ -0,0 +1,2 @@ +.. _docker: +.. include:: ../docker/README.rst diff --git a/docs/index.rst b/docs/index.rst index c164ede9a..f459c12b9 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2,7 +2,7 @@ POCS ==== -.. include:: ../README.rst +.. include:: ./README.rst Contents ======== @@ -10,11 +10,12 @@ Contents .. toctree:: :maxdepth: 2 - License - Authors + Docker Guider + Contributing Guide Changelog Module Reference - Contributing Guide + Authors + License Indices and tables diff --git a/docs/requirements.txt b/docs/requirements.txt index 0a70e145b..483a4e960 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1 @@ --r ../requirements.txt sphinx_rtd_theme - diff --git a/scripts/install/install-pocs.sh b/scripts/install/install-pocs.sh old mode 100755 new mode 100644 index 17bee90d8..7bb1e4636 --- a/scripts/install/install-pocs.sh +++ b/scripts/install/install-pocs.sh @@ -5,202 +5,338 @@ usage() { echo -n "################################################## # Install POCS and friends. # +# Script Version: 2020-07-27 +# # This script is designed to install the PANOPTES Observatory # Control System (POCS) on a cleanly installed Ubuntu system. # # This script is meant for quick & easy install via: # -# $ curl -L https://install.projectpanoptes.org | bash +# $ curl -fsSL https://install.projectpanoptes.org > install-pocs.sh +# $ bash install-pocs.sh # or -# $ wget -O - https://install.projectpanoptes.org | bash +# $ wget -qO- https://install.projectpanoptes.org > install-pocs.sh +# $ bash install-pocs.sh +# +# The script will do the following: # -# The script will insure that Docker is installed, download the -# latest Docker images (see list below) and clone a copy of the -# relevant PANOPTES repositories. +# * Create the needed directory structure. +# * Ensure that docker and docker-compose are installed. +# * Fetch and/or build the docker images needed to run. +# * If in "developer" mode, clone user's fork and set panoptes upstream. +# * Write the environment variables to ${PANDIR}/env # # Docker Images: # # ${DOCKER_BASE}/panoptes-utils # ${DOCKER_BASE}/pocs # -# The script will ask for a github user name. If you are a developer -# you can enter your github username to work from your fork. Otherwise -# the default user (panoptes) is okay for running the unit. +# The script will ask if it should be installed in "developer" mode or not. +# +# The regular install is for running units and will not create local (to the +# host system) copies of the files. +# +# The "developer" mode will ask for a github username and will clone and +# fetch the repos. The $(docker/setup-local-enviornment.sh) script will then +# be run to build the docker images locally. # -# The script has been tested with a fresh install of Ubuntu 19.04 +# If not in "developer" mode, the docker images will be pulled from GCR. +# +# The script has been tested with a fresh install of Ubuntu 20.04 # but may work on other linux systems. +# +# Changes: +# * 2020-07-05 - Initial release of versioned script. +# * 2020-07-06 (wtgee) - Fix the writing of the env file. Cleanup. +# * 2020-07-08 (wtgee) - Better test for ssh access for developer. +# * 2020-07-09 (wtgee) - Fix conditional for writing shell rc files. Use 3rd +# party docker-compose (linuxserver.io) for arm. +# * 2020-07-27 (wtgee) - Cleanup and consistency for Unit install. +# ############################################################# - $ $(basename $0) [--user panoptes] [--pandir /var/panoptes] + $ $(basename $0) [--developer] [--user panoptes] [--pandir /var/panoptes] Options: - USER The PANUSER environment variable, defaults to current user (i.e. USER=`$USER`). - PANDIR Default install directory, defaults to /var/panoptes. Saved as PANDIR + DEVELOPER Install POCS in developer mode, default False. + + If in DEVELOPER mode, the following options are also available: + USER The PANUSER environment variable, defaults to current user (i.e. PANUSER=$USER). + PANDIR Default install directory, defaults to PANDIR=${PANDIR}. Saved as PANDIR environment variable. " } -DOCKER_BASE="gcr.io/panoptes-exp" +# Better select prompt. +PS3="Select: " -if [ -z "${PANUSER}" ]; then - export PANUSER=$USER -fi -if [ -z "${PANDIR}" ]; then - export PANDIR='/var/panoptes' -fi +DEVELOPER=${DEVELOPER:-false} +PANUSER=${PANUSER:-$USER} +PANDIR=${PANDIR:-/var/panoptes} +LOGFILE="${PANDIR}/install-pocs.log" +OS="$(uname -s)" +ARCH="$(uname -m)" +ENV_FILE="${PANDIR}/env" + +GITHUB_USER="panoptes" +GITHUB_URL="https://github.com/${GITHUB_USER}" + +PANOPTES_UPSTREAM_URL="https://github.com/panoptes" + +# Repositories to clone. +REPOS=("POCS" "panoptes-utils" "panoptes-tutorials") -while [[ $# -gt 0 ]] -do -key="$1" -case $key in - -u|--user) +DOCKER_COMPOSE_INSTALL="https://raw.githubusercontent.com/linuxserver/docker-docker-compose/master/run.sh" +DOCKER_BASE=${DOCKER_BASE:-"gcr.io/panoptes-exp"} + +while [[ $# -gt 0 ]]; do + key="$1" + case ${key} in + --developer) + DEVELOPER=true + shift # past bool argument + ;; + --install-unit) + DEVELOPER=false + shift # past bool argument + ;; + -u | --user) PANUSER="$2" shift # past argument shift # past value ;; - -d|--pandir) + -d | --pandir) PANDIR="$2" shift # past argument shift # past value ;; - -h|--help) + -h | --help) PANDIR="$2" usage - exit 1 + return ;; -esac + esac done -function command_exists { - # https://gist.github.com/gubatron/1eb077a1c5fcf510e8e5 - # this should be a very portable way of checking if something is on the path - # usage: "if command_exists foo; then echo it exists; fi" - type "$1" &> /dev/null -} +if ! ${DEVELOPER}; then + echo "How would you like to install the unit?" + select mode in "Developer" "PANOPTES Unit"; do + case ${mode} in + Developer) + echo "Enabling developer mode. Note that you will need your GitHub username to proceed." + DEVELOPER=true + break + ;; + "PANOPTES Unit") + echo "Installing POCS for a PANOPTES unit." + break + ;; + esac + done +fi -do_install() { - clear +if "${DEVELOPER}"; then + echo "To install POCS as a developer make sure you have first forked the following repositories:" + echo "" + echo " https://github.com/panoptes/POCS" + echo " https://github.com/panoptes/panoptes-utils" + echo " https://github.com/panoptes/panoptes-tutorials" + echo "" + echo "You will also need to have an ssh key set up on github.com." + echo "See https://docs.github.com/en/github/authenticating-to-github/connecting-to-github-with-ssh" - OS="$(uname -s)" - case "${OS}" in - Linux*) machine=Linux;; - Darwin*) machine=Mac;; - *) machine="UNKNOWN:${unameOut}" - esac - echo ${machine} - - # Install directory - read -p "PANOPTES base directory [${PANDIR:-/var/panoptes}]: " PANDIR - PANDIR=${PANDIR:-/var/panoptes} - - LOGFILE="${PANDIR}/logs/install-pocs.log" - - echo "Installing PANOPTES software." - echo "USER: ${PANUSER}" - echo "OS: ${OS}" - echo "Base dir: ${PANDIR}" - echo "Logfile: ${LOGFILE}" - - # Directories - if [[ ! -d "${PANDIR}" ]]; then - echo "Creating directories in ${PANDIR}" - # Make directories - sudo mkdir -p "${PANDIR}" - sudo chown -R "${PANUSER}":"${PANUSER}" "${PANDIR}" - - mkdir -p "${PANDIR}/logs" - mkdir -p "${PANDIR}/images" - mkdir -p "${PANDIR}/conf_files" - mkdir -p "${PANDIR}/.key" + read -p "Github User [panoptes]: " GITHUB_USER + + # If a different user, make sure we can access github as that user, otherwise exit. + if test "${GITHUB_USER}" != "panoptes"; then + echo "Testing github ssh access for user: ${GITHUB_USER}" + + # Test for ssh access + if [[ $(ssh -T git@github.com 2>&1) =~ "success" ]]; then + GITHUB_URL="git@github.com:${GITHUB_USER}" else - echo "WARNING ${PANDIR} already exists. You can exit and specify an alternate directory with --pandir or continue." - select yn in "Yes" "No"; do - case $yn in - Yes ) echo "Proceeding with existing directory"; break;; - No ) echo "Exiting"; exit 1;; - esac - done + echo "Can't ssh to github.com. Have you set up your ssh keys?" + echo "See https://docs.github.com/en/github/authenticating-to-github/connecting-to-github-with-ssh" + return fi + fi +fi - # apt: git, wget - echo "Installing system dependencies" +function command_exists() { + # https://gist.github.com/gubatron/1eb077a1c5fcf510e8e5 + # this should be a very portable way of checking if something is on the path + # usage: "if command_exists foo; then echo it exists; fi" + type "$1" &>/dev/null +} - if [[ "${OS}" = "Linux" ]]; then - sudo apt-get update >> "${LOGFILE}" 2>&1 - sudo apt-get --yes install wget curl git openssh-server ack jq httpie byobu >> "${LOGFILE}" 2>&1 - elif [[ "${OS}" = "Darwin" ]]; then - sudo brew update | sudo tee -a "${LOGFILE}" - sudo brew install wget curl git jq httpie | sudo tee -a "${LOGFILE}" - fi +function make_directories() { + if [[ ! -d "${PANDIR}" ]]; then + # Make directories and make PANUSER the owner. + sudo mkdir -p "${PANDIR}" + else + echo "Would you like to continue with the existing directory?" + select yn in "Yes" "No"; do + case ${yn} in + Yes) + echo "Proceeding with existing directory" + break + ;; + No) + echo "Exiting script" + return + ;; + esac + done + fi + + sudo mkdir -p "${PANDIR}/logs" + sudo mkdir -p "${PANDIR}/images" + sudo mkdir -p "${PANDIR}/config_files" + sudo mkdir -p "${PANDIR}/.key" + sudo chown -R "${PANUSER}":"${PANUSER}" "${PANDIR}" +} + +function setup_env_vars() { + if [[ ! -f "${ENV_FILE}" ]]; then + echo "Writing environment variables to ${ENV_FILE}" + cat >>"${ENV_FILE}" <> "${LOGFILE}" 2>&1 - else - # TODO Do an update here. - echo "" + # Source the files in the shell. + SHELLS=(".bashrc" ".zshrc") + + for SHELL_RC in "${SHELLS[@]}"; do + SHELL_RC_PATH="$HOME/${SHELL_RC}" + if test -f "${SHELL_RC_PATH}"; then + # Check if we have already added the file. + if ! grep -qm 1 ". ${PANDIR}/env" "${SHELL_RC_PATH}"; then + printf "\n. ${PANDIR}/env\n" >>"${SHELL_RC_PATH}" fi + fi done + fi +} - # Get Docker - if ! command_exists docker; then - echo "Installing Docker" - if [[ "${OS}" = "Linux" ]]; then - /bin/bash -c "$(wget -qO- https://get.docker.com)" &>> ${PANDIR}/logs/install-pocs.log - - echo "Adding ${PANUSER} to docker group" - sudo usermod -aG docker "${PANUSER}" >> "${LOGFILE}" 2>&1 - elif [[ "${OS}" = "Darwin" ]]; then - brew cask install docker - echo "Adding ${PANUSER} to docker group" - sudo dscl -aG docker "${PANUSER}" - fi +function system_deps() { + if [[ "${OS}" == "Linux" ]]; then + sudo apt-get update >>"${LOGFILE}" 2>&1 + sudo apt-get --yes install \ + wget curl git openssh-server ack jq httpie byobu \ + >>"${LOGFILE}" 2>&1 + elif [[ "${OS}" == "Darwin" ]]; then + sudo brew update | sudo tee -a "${LOGFILE}" + sudo brew install \ + wget curl git jq httpie | + sudo tee -a "${LOGFILE}" + fi + + # Add an SSH key if one doesn't exist. + if [[ ! -f "${HOME}/.ssh/id_rsa" ]]; then + echo "Adding ssh key" + ssh-keygen -t rsa -N "" -f "${HOME}/.ssh/id_rsa" + fi +} + +function get_repos() { + echo "Cloning ${REPOS}" + for repo in "${REPOS[@]}"; do + if [[ ! -d "${PANDIR}/${repo}" ]]; then + cd "${PANDIR}" + echo "Cloning ${GITHUB_URL}/${repo}" + # Just redirect the errors because otherwise looks like it hangs. + git clone --single-branch --quiet "${GITHUB_URL}/${repo}.git" &>>"${LOGFILE}" + + # Set panoptes as upstream if clone succeeded. + if [ $? -eq 0 ]; then + cd "${repo}" + git remote add upstream "${PANOPTES_UPSTREAM_URL}/${repo}" + fi else - echo "WARNING: Docker images not installed/downloaded." + echo "${repo} already exists in ${PANDIR}. No auto-update for now, skipping repo." fi + done +} - if ! command_exists docker-compose; then - echo "Installing docker-compose" - # Docker compose as container - https://docs.docker.com/compose/install/#install-compose - sudo wget -q https://github.com/docker/compose/releases/download/1.25.4/docker-compose-`uname -s`-`uname -m` -O /usr/local/bin/docker-compose - sudo chmod a+x /usr/local/bin/docker-compose +function get_docker() { + if ! command_exists docker; then + echo "Installing Docker" + if [[ "${OS}" == "Linux" ]]; then + /bin/bash -c "$(wget -qO- https://get.docker.com)" &>>"${LOGFILE}" - docker pull docker/compose + echo "Adding ${PANUSER} to docker group" + sudo usermod -aG docker "${PANUSER}" >>"${LOGFILE}" 2>&1 + elif [[ "${OS}" == "Darwin" ]]; then + brew cask install docker + echo "Adding ${PANUSER} to docker group" + sudo dscl -aG docker "${PANUSER}" fi + fi - echo "Pulling PANOPTES docker images" - docker pull "${DOCKER_BASE}/panoptes-utils:latest" - docker pull "${DOCKER_BASE}/aag-weather:latest" - docker pull "${DOCKER_BASE}/pocs:latest" + if ! command_exists docker-compose; then + echo "Installing docker-compose" + sudo wget -q "${DOCKER_COMPOSE_INSTALL}" -O /usr/local/bin/docker-compose + sudo chmod a+x /usr/local/bin/docker-compose + fi +} - # Add an SSH key if one doesn't exists - if [[ ! -f "${HOME}/.ssh/id_rsa" ]]; then - echo "Looks like you don't have an SSH key set up yet, adding one now." - ssh-keygen -t rsa -N "" -f "${HOME}/.ssh/id_rsa"; - fi +function get_or_build_images() { + if ${DEVELOPER}; then + echo "Building local PANOPTES docker images." - echo "Please reboot your machine before using POCS." + cd "${PANDIR}/POCS" + ./docker/setup-local-environment.sh + else + echo "Pulling PANOPTES docker images from Google Cloud Registry (GCR)." - read -p "Reboot now? [y/N]: " -r - if [[ $REPLY =~ ^[Yy]$ ]]; then - sudo reboot - fi + sudo docker pull "${DOCKER_BASE}/panoptes-pocs:latest" + sudo docker pull "${DOCKER_BASE}/panoptes-utils:latest" + sudo docker pull "${DOCKER_BASE}/aag-weather:latest" + fi +} + +function do_install() { + clear + + echo "Installing PANOPTES software." + if ${DEVELOPER}; then + echo "**** Developer Mode ****" + echo "GITHUB_USER=${GITHUB_USER}" + fi + echo "PANUSER: ${PANUSER}" + echo "PANDIR: ${PANDIR}" + echo "OS: ${OS}" + echo "Logfile: ${LOGFILE}" + + echo "Creating directories in ${PANDIR}" + make_directories + + echo "Setting up environment variables in ${ENV_FILE}" + setup_env_vars + + echo "Installing system dependencies" + system_deps + + echo "Installing docker and docker-compose" + get_docker + + if ${DEVELOPER}; then + echo "Cloning PANOPTES source code" + get_repos + fi + + get_or_build_images + + echo "Please reboot your machine before using POCS." + read -p "Reboot now? [y/N]: " -r + if [[ $REPLY =~ ^[Yy]$ ]]; then + sudo reboot + fi } -# wrapped up in a function so that we have some protection against only getting -# half the file during "curl | sh" - copied from get.docker.com do_install diff --git a/scripts/setup-local-environment.sh b/scripts/setup-local-environment.sh new file mode 100755 index 000000000..a043c4fde --- /dev/null +++ b/scripts/setup-local-environment.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +set -e + +INCLUDE_BASE=${INCLUDE_BASE:-true} # INCLUDE_UTILS must be true to work. +INCLUDE_UTILS=${INCLUDE_UTILS:-false} +INCLUDE_DEVELOPER=${INCLUDE_DEVELOPER:-false} + +PANOPTES_UTILS=${PANOPTES_UTILS:-$PANDIR/panoptes-utils} +PANOPTES_POCS=${PANOPTES_POCS:-$PANDIR/POCS} +_UTILS_IMAGE_URL="gcr.io/panoptes-exp/panoptes-utils:latest" + +echo "Setting up local environment." +cd "${PANOPTES_POCS}" + +build_utils() { + /bin/bash "${PANOPTES_UTILS}/docker/setup-local-environment.sh" + # Use our local image for build below instead of gcr.io image. + _UTILS_IMAGE_URL="panoptes-utils:develop" +} + +build_develop() { + echo "Building local panoptes-pocs:develop from ${_UTILS_IMAGE_URL} in ${PANOPTES_POCS}" + docker build \ + -t "panoptes-pocs:develop" \ + -f "${PANOPTES_POCS}/docker/Dockerfile" \ + "${PANOPTES_POCS}" +} + +build_developer() { + echo "Building local panoptes-pocs:developer from panoptes-pocs:develop in ${PANOPTES_POCS}" + docker build \ + -t "panoptes-pocs:developer" \ + -f "${PANOPTES_POCS}/docker/developer/Dockerfile" \ + "${PANOPTES_POCS}" +} + +#################################################################################### +# Script logic below +#################################################################################### + +if [ "${INCLUDE_UTILS}" = true ]; then + build_utils +fi + +build_develop + +if [ "${INCLUDE_DEVELOPER}" = true ]; then + build_developer +fi + +cat <=0.2.17 + panoptes-utils>=0.2.26 pyserial>=3.1.1 PyYaml - pendulum - readline requests - responses scalpl scipy transitions # The usage of test_requires is discouraged, see `Dependency Management` docs # tests_require = pytest; pytest-cov # Require a specific Python version, e.g. Python 2.7 or >= 3.4 -python_requires = >=3.7 +python_requires = >=3.8 [options.packages.find] where = src @@ -68,18 +66,30 @@ exclude = [options.extras_require] # Add here additional requirements for extra features, to install with: # `pip install POCS[PDF]` like: -# PDF = ReportLab; RXP -# Add here test requirements (semicolon/line-separated) +developer = + google-cloud-sdk + google-cloud-bigquery[pandas,pyarrow] + google-cloud-firestore + google-cloud-storage + jupyterlab + pandas + tabulate +google = + google-cloud-storage +plotting = + altair + bokeh + holoviews + hvplot + seaborn testing = + coverage + pycodestyle pytest pytest-cov - pycodestyle - responses - coverage + pytest-doctestplus pytest-remotedata>=0.3.1' -google = - gcloud - google-cloud-storage + responses [options.entry_points] # Add here console scripts like: @@ -95,8 +105,14 @@ google = [test] # py.test options when running `python setup.py test` addopts = - --cov panoptes.pocs --cov panoptes.peas --cov-report term-missing + --cov panoptes.pocs + --cov panoptes.peas + --cov-branch + --cov-report term-missing:skip-covered + --no-cov-on-fail --doctest-modules + --doctest-ignore-import-errors + --failed-first -x --verbose extras = True @@ -138,12 +154,6 @@ universal = 1 source_dir = docs build_dir = build/sphinx -[devpi:upload] -# Options for the devpi: PyPI server and packaging tool -# VCS export must be deactivated since we are using setuptools-scm -no-vcs = 1 -formats = bdist_wheel - [flake8] # Some sane defaults for the code style checker flake8 exclude = @@ -197,4 +207,4 @@ exclude_lines = if 0: if __name__ == .__main__.: -ignore_errors = True \ No newline at end of file +ignore_errors = True diff --git a/scripts/coverage/sitecustomize.py b/sitecustomize.py similarity index 99% rename from scripts/coverage/sitecustomize.py rename to sitecustomize.py index 1dc959e54..6c70e25d1 100644 --- a/scripts/coverage/sitecustomize.py +++ b/sitecustomize.py @@ -1,5 +1,6 @@ # Ensure coverage starts for all Python processes so that test coverage is calculated # properly when using subprocesses (see https://coverage.readthedocs.io/en/latest/subprocess.html) import coverage + print("Starting coverage from sitecustomize") coverage.process_startup() diff --git a/src/panoptes/peas/sensors.py b/src/panoptes/peas/sensors.py index 85b6e27b4..fe505b257 100644 --- a/src/panoptes/peas/sensors.py +++ b/src/panoptes/peas/sensors.py @@ -6,13 +6,14 @@ from panoptes.utils.config.client import get_config from panoptes.utils.database import PanDB -from panoptes.pocs.utils.logger import get_logger from panoptes.utils.rs232 import SerialData from panoptes.utils import error +from panoptes.pocs.utils.logger import get_logger + class ArduinoSerialMonitor(object): - """Monitors the serial lines and tries to parse any data recevied as JSON. + """Monitors the serial lines and tries to parse any data received as JSON. Checks for the `camera_box` and `computer_box` entries in the config and tries to connect. Values are updated in the database. @@ -129,7 +130,7 @@ def capture(self, store_result=True): self.db.insert_current('power', data['power']) except Exception as e: - self.logger.warning('Exception while reading from sensor {}: {}', sensor_name, e) + self.logger.warning(f'Exception while reading from {sensor_name=}: {e!r}') return sensor_data @@ -164,14 +165,14 @@ def detect_board_on_port(port): Else returns None. """ logger = get_logger() - logger.debug('Attempting to connect to serial port: {}'.format(port)) + logger.debug(f'Attempting to connect to serial {port=}') try: serial_reader = SerialData(port=port, baudrate=9600, retry_limit=1, retry_delay=0) if not serial_reader.is_connected: serial_reader.connect() - logger.debug('Connected to {}', port) + logger.debug(f'Connected to {port=}') except Exception: - logger.warning('Could not connect to port: {}'.format(port)) + logger.warning(f'Could not connect to {port=}') return None try: reading = serial_reader.get_and_parse_reading(retry_limit=3) @@ -182,10 +183,10 @@ def detect_board_on_port(port): result = (data['name'], serial_reader) serial_reader = None return result - logger.warning('Unable to find board name in reading: {!r}', reading) + logger.warning(f'Unable to find board name in {reading=!r}') return None except Exception as e: - logger.error('Exception while auto-detecting port {!r}: {!r}'.format(port, e)) + logger.error(f'Exception while auto-detecting port {port=!r}: {e!r}') finally: if serial_reader: serial_reader.disconnect() @@ -195,11 +196,11 @@ def detect_board_on_port(port): if __name__ == '__main__': devices = find_arduino_devices() if devices: - print("Arduino devices: {}".format(", ".join(devices))) + print(f'Arduino devices: {",".join(devices)}') else: print("No Arduino devices found.") sys.exit(1) names_and_readers = auto_detect_arduino_devices() for (name, serial_reader) in names_and_readers: - print('Device {} has name {}'.format(serial_reader.name, name)) + print(f'Device {serial_reader.name} has {name=}') serial_reader.disconnect() diff --git a/src/panoptes/pocs/base.py b/src/panoptes/pocs/base.py index 40de909dc..84d7b6109 100644 --- a/src/panoptes/pocs/base.py +++ b/src/panoptes/pocs/base.py @@ -1,79 +1,79 @@ -import sys +from requests.exceptions import ConnectionError -from pocs import hardware -from pocs import __version__ -from pocs.utils import config -from pocs.utils.database import PanMongo -from pocs.utils.logger import get_root_logger +from panoptes.pocs import __version__ +from panoptes.utils.database import PanDB +from panoptes.utils.config import client +from panoptes.pocs.utils.logger import get_logger +from panoptes.pocs import hardware -# Global vars -_config = None - - -def reset_global_config(): - """Reset the global _config to None. - - Globals such as _config make tests non-hermetic. Enable conftest.py to clear _config - in an explicit fashion. - """ - global _config - _config = None +# Global database. +PAN_DB_OBJ = None class PanBase(object): - """ Base class for other classes within the PANOPTES ecosystem - Defines common properties for each class (e.g. logger, config). + Defines common properties for each class (e.g. logger, config, db). """ - def __init__(self, *args, **kwargs): - # Load the default and local config files - global _config - if _config is None: - ignore_local_config = kwargs.get('ignore_local_config', False) - _config = config.load_config(ignore_local=ignore_local_config) - + def __init__(self, config_port='6563', *args, **kwargs): self.__version__ = __version__ - # Update with run-time config - if 'config' in kwargs: - _config.update(kwargs['config']) + self._config_port = config_port + + self.logger = get_logger() - self._check_config(_config) - self.config = _config + # If the user requests a db_type then update runtime config + db_type = kwargs.get('db_type', self.get_config('db.type', default='file')) + db_name = kwargs.get('db_name', self.get_config('db.name', default='panoptes')) + db_folder = kwargs.get('db_folder', self.get_config('db.folder', default='json_store')) - self.logger = kwargs.get('logger') - if not self.logger: - self.logger = get_root_logger() + global PAN_DB_OBJ + if PAN_DB_OBJ is None: + PAN_DB_OBJ = PanDB(db_type=db_type, db_name=db_name, storage_dir=db_folder) - self.config['simulator'] = hardware.get_simulator_names(config=self.config, kwargs=kwargs) + self.db = PAN_DB_OBJ - # Set up connection to database - db = kwargs.get('db', self.config['db']['name']) - _db = PanMongo(db=db) + def get_config(self, *args, **kwargs): + """Thin-wrapper around client based get_config that sets default port. - self.db = _db + See `panoptes.utils.config.client.get_config` for more information. - def _check_config(self, temp_config): - """ Checks the config file for mandatory items """ + Args: + *args: Passed to get_config + **kwargs: Passed to get_config + """ + config_value = None + try: + config_value = client.get_config(port=self._config_port, *args, **kwargs) + except ConnectionError as e: # pragma: no cover + self.logger.critical(f'Cannot connect to config_server from {self.__class__}: {e!r}') - if 'directories' not in temp_config: - sys.exit('directories must be specified in config') + return config_value - if 'mount' not in temp_config: - sys.exit('Mount must be specified in config') + def set_config(self, key, new_value, *args, **kwargs): + """Thin-wrapper around client based set_config that sets default port. - if 'state_machine' not in temp_config: - sys.exit('State Table must be specified in config') + See `panoptes.utils.config.client.set_config` for more information. - def __getstate__(self): # pragma: no cover - d = dict(self.__dict__) + Args: + key (str): The key name to use, can be namespaced with dots. + new_value (any): The value to store. + *args: Passed to set_config + **kwargs: Passed to set_config + """ + config_value = None - if 'logger' in d: - del d['logger'] + if key == 'simulator' and new_value == 'all': + # Don't use hardware.get_simulator_names because it checks config. + new_value = hardware.ALL_NAMES - if 'db' in d: - del d['db'] + try: + self.logger.trace(f'Setting config {key=} {new_value=}') + config_value = client.set_config(key, new_value, port=self._config_port, *args, + **kwargs) + self.logger.trace(f'Config set {config_value=}') + except ConnectionError as e: # pragma: no cover + self.logger.critical(f'Cannot connect to config_server from {self.__class__}: {e!r}') - return d + return config_value diff --git a/src/panoptes/pocs/camera/__init__.py b/src/panoptes/pocs/camera/__init__.py index 80b68e18e..90ecce299 100644 --- a/src/panoptes/pocs/camera/__init__.py +++ b/src/panoptes/pocs/camera/__init__.py @@ -5,14 +5,16 @@ import random from astropy import units as u -from panoptes.pocs.camera.camera import AbstractCamera # pragma: no flakes -from panoptes.pocs.camera.camera import AbstractGPhotoCamera # pragma: no flakes +from panoptes.pocs.camera.camera import AbstractCamera # noqa +from panoptes.pocs.camera.camera import AbstractGPhotoCamera # noqa from panoptes.pocs.utils.logger import get_logger from panoptes.utils import error from panoptes.utils.config.client import get_config from panoptes.utils.library import load_module +logger = get_logger() + def list_connected_cameras(): """Detect connected cameras. @@ -43,17 +45,17 @@ def list_connected_cameras(): return ports -def create_cameras_from_config(config_port='6563', **kwargs): +def create_cameras_from_config(*args, **kwargs): """Create camera object(s) based on the config. Creates a camera for each camera item listed in the config. Ensures the appropriate camera module is loaded. Args: - config_port (str, optional): config_server port, default '6563'. + *args (list): Passed to `get_config`. **kwargs (dict): Can pass a `cameras` object that overrides the info in the configuration file. Can also pass `auto_detect`(bool) to try and - automatically discover the ports. + automatically discover the ports. Any other items as passed to `get_config`. Returns: OrderedDict: An ordered dictionary of created camera objects, with the @@ -65,9 +67,8 @@ def create_cameras_from_config(config_port='6563', **kwargs): auto_detect=True and no cameras are found. error.PanError: Description """ - logger = get_logger() - config = get_config(port=config_port) + config = get_config(*args, **kwargs) # Helper method to first check kwargs then config def kwargs_or_config(item, default=None): @@ -141,7 +142,7 @@ def kwargs_or_config(item, default=None): module = load_module(f'panoptes.pocs.camera.{model}') logger.debug(f'Camera module: {module}') # Create the camera object - cam = module.Camera(config_port=config_port, **device_config) + cam = module.Camera(**device_config) except error.NotFound: logger.error(f"Cannot find camera module with config: {device_config}") except Exception as e: @@ -171,15 +172,11 @@ def kwargs_or_config(item, default=None): return cameras -def create_camera_simulator(num_cameras=2, config_port='6563', **kwargs): +def create_camera_simulator(num_cameras=2): """Create simulator camera object(s). Args: num_cameras (int): The number of simulated cameras to create, default 2. - config_port (int): The port to use to connect to the config server, default 6563. - **kwargs (dict): Can pass a `cameras` object that overrides the info in - the configuration file. Can also pass `auto_detect`(bool) to try and - automatically discover the ports. Returns: OrderedDict: An ordered dictionary of created camera objects, with the @@ -189,70 +186,55 @@ def create_camera_simulator(num_cameras=2, config_port='6563', **kwargs): Raises: error.CameraNotFound: Raised if camera cannot be found at specified port or if auto_detect=True and no cameras are found. - error.PanError: Description """ - logger = get_logger() + if num_cameras == 0: + raise error.CameraNotFound(msg="No cameras available") cameras = OrderedDict() - # Create a minimal dummy camera config to get a simulated camera - camera_info = {'autodetect': False, - 'devices': [ - {'model': 'simulator'}, ]} - - logger.debug(f"Camera config: {camera_info}") + # Set up a simulated camera with fully configured simulated focuser. + device_config = { + 'model': 'simulator', + 'port': '/dev/camera/simulator', + 'focuser': {'model': 'simulator', + 'focus_port': '/dev/ttyFAKE', + 'initial_position': 20000, + 'autofocus_range': (40, 80), + 'autofocus_step': (10, 20), + 'autofocus_seconds': 0.1, + 'autofocus_size': 500}, + 'filterwheel': {'model': 'simulator', + 'filter_names': ['one', 'deux', 'drei', 'quattro'], + 'move_time': 0.1 * u.second, + 'timeout': 0.5 * u.second}, + 'readout_time': 0.5, + } + logger.debug(f"SimulatorCamera config: {device_config=}") primary_camera = None - for cam_num in range(num_cameras): cam_name = f'SimCam{cam_num:02d}' logger.debug(f'Using camera simulator {cam_name}') - # Set up a simulated camera with fully configured simulated focuser - device_config = { - 'model': 'simulator', - 'port': '/dev/camera/simulator', - 'focuser': {'model': 'simulator', - 'focus_port': '/dev/ttyFAKE', - 'initial_position': 20000, - 'autofocus_range': (40, 80), - 'autofocus_step': (10, 20), - 'autofocus_seconds': 0.1, - 'autofocus_size': 500}, - 'filterwheel': {'model': 'simulator', - 'filter_names': ['one', 'deux', 'drei', 'quattro'], - 'move_time': 0.1 * u.second, - 'timeout': 0.5 * u.second}, - 'readout_time': 0.5, - # Simulator config should always ignore local settings. - 'ignore_local_config': True - } camera_model = device_config['model'] logger.debug(f'Creating camera: {camera_model}') - try: - module = load_module(f'panoptes.pocs.camera.{camera_model}') - logger.debug(f'Camera module: {module}') - # Create the camera object - cam = module.Camera(name=cam_name, config_port=config_port, **device_config) - except error.NotFound: # pragma: no cover - logger.error(f"Cannot find camera module: {camera_model}") - except Exception as e: # pragma: no cover - logger.error(f"Cannot create camera type: {camera_model} {e!r}") - else: - is_primary = '' - if cam_num == 0: - cam.is_primary = True - primary_camera = cam - is_primary = ' [Primary]' + module = load_module(f'panoptes.pocs.camera.{camera_model}') + logger.debug(f'Camera module: {module}') - logger.debug(f"Camera created: {cam.name} {cam.uid}{is_primary}") + # Create the camera object + cam = module.Camera(name=cam_name, **device_config) - cameras[cam_name] = cam + is_primary = '' + if cam_num == 0: + cam.is_primary = True + primary_camera = cam + is_primary = ' [Primary]' - if len(cameras) == 0: - raise error.CameraNotFound(msg="No cameras available") + logger.debug(f"Camera created: {cam.name} {cam.uid}{is_primary}") + + cameras[cam_name] = cam logger.debug(f"Primary camera: {primary_camera}") logger.debug(f"{len(cameras)} cameras created") diff --git a/src/panoptes/pocs/camera/camera.py b/src/panoptes/pocs/camera/camera.py index 09e3e86a1..5d418f5df 100644 --- a/src/panoptes/pocs/camera/camera.py +++ b/src/panoptes/pocs/camera/camera.py @@ -25,7 +25,7 @@ from panoptes.utils.serializers import from_yaml -def parse_config(lines): +def parse_config(lines): # pragma: no cover yaml_string = '' for line in lines: IsID = len(line.split('/')) > 1 @@ -69,8 +69,10 @@ class AbstractCamera(PanBase, metaclass=ABCMeta): Attributes: filter_type (str): Type of filter attached to camera, default RGGB. - focuser (`panoptes.pocs.focuser.AbstractFocuser`|None): Focuser for the camera, default None. - filter_wheel (`panoptes.pocs.filterwheel.AbstractFilterWheel`|None): Filter wheel for the camera, + focuser (`panoptes.pocs.focuser.AbstractFocuser`|None): Focuser for the camera, + default None. + filter_wheel (`panoptes.pocs.filterwheel.AbstractFilterWheel`|None): Filter wheel for the + camera, default None. is_primary (bool): If this camera is the primary camera for the system, default False. model (str): The model of camera, such as 'gphoto2', 'sbig', etc. Default 'simulator'. @@ -94,7 +96,8 @@ class AbstractCamera(PanBase, metaclass=ABCMeta): For these cameras serial_number should be passed to the constructor instead. For SBIG and FLI this should simply be the serial number engraved on the camera case, whereas for ZWO cameras this should be the 8 character ID string previously saved to the camera - firmware. This can be done using ASICAP, or `panoptes.pocs.camera.libasi.ASIDriver.set_ID()`. + firmware. This can be done using ASICAP, + or `panoptes.pocs.camera.libasi.ASIDriver.set_ID()`. """ _subcomponent_classes = {'Focuser', 'FilterWheel'} @@ -634,7 +637,8 @@ def get_thumbnail(self, seconds, file_path, thumbnail_size, keep_file=False, *ar return thumbnail @abstractmethod - def _start_exposure(self, seconds=None, filename=None, dark=False, header=None, *args, **kwargs): + def _start_exposure(self, seconds=None, filename=None, dark=False, header=None, *args, + **kwargs): """Responsible for the camera-specific process that start an exposure. This method is called from the `take_exposure` method and is used to handle @@ -734,7 +738,9 @@ def _setup_observation(self, observation, headers, filename, **kwargs): if observation.filter_name is not None: try: # Move the filterwheel - self.logger.debug(f'Moving filterwheel={self.filterwheel} to filter_name={observation.filter_name}') + self.logger.debug( + f'Moving filterwheel={self.filterwheel} to filter_name=' + f'{observation.filter_name}') self.filterwheel.move_to(observation.filter_name, blocking=True) except Exception as e: self.logger.error(f'Error moving filterwheel on {self} to' @@ -743,7 +749,8 @@ def _setup_observation(self, observation, headers, filename, **kwargs): else: self.logger.info(f'Filter {observation.filter_name} requested by' - f' observation but {self.filterwheel} is missing that filter, using' + f' observation but {self.filterwheel} is missing that filter, ' + f'using' f' {self.filter_type}.') if headers is None: @@ -756,7 +763,8 @@ def _setup_observation(self, observation, headers, filename, **kwargs): observation.seq_time = start_time # Get the filename - self.logger.debug(f'Setting image_dir={observation.directory}/{self.uid}/{observation.seq_time}') + self.logger.debug( + f'Setting image_dir={observation.directory}/{self.uid}/{observation.seq_time}') image_dir = os.path.join( observation.directory, self.uid, @@ -813,7 +821,8 @@ def _setup_observation(self, observation, headers, filename, **kwargs): metadata.update(headers) self.logger.debug( - f'Observation setup: exptime={exptime} file_path={file_path} image_id={image_id} metadata={metadata}') + f'Observation setup: exptime={exptime} file_path={file_path} image_id={image_id} ' + f'metadata={metadata}') return exptime, file_path, image_id, metadata def _process_fits(self, file_path, info): @@ -845,7 +854,8 @@ def _create_subcomponent(self, subcomponent, class_name): try: base_module = load_module(base_module_name) except error.NotFound as err: - self.logger.critical(f"Couldn't import {class_name} base class module {base_module_name}!") + self.logger.critical( + f"Couldn't import {class_name} base class module {base_module_name}!") raise err base_class = getattr(base_module, f"Abstract{class_name}") @@ -861,7 +871,7 @@ def _create_subcomponent(self, subcomponent, class_name): self.logger.critical(f"Couldn't import {class_name} module {module_name}!") raise err subcomponent_kwargs = copy.deepcopy(subcomponent) - subcomponent_kwargs.update({'camera': self, 'config_port': self._config_port}) + subcomponent_kwargs.update({'camera': self}) setattr(self, class_name_lower, getattr(module, class_name)(**subcomponent_kwargs)) @@ -891,7 +901,8 @@ def __str__(self): else: s += f" & {subcomponent.name}" sub_count += 1 - except Exception: + except Exception as e: + self.logger.warning(f'Unable to stringify camera: {e=}') s = str(self.__class__) return s diff --git a/src/panoptes/pocs/core.py b/src/panoptes/pocs/core.py index f00d3f26d..61e4d07e9 100644 --- a/src/panoptes/pocs/core.py +++ b/src/panoptes/pocs/core.py @@ -31,7 +31,7 @@ class POCS(PanStateMachine, PanBase): class. POCS will call the `initialize` method of the observatory. state_machine_file(str): Filename of the state machine to use, defaults to 'simple_state_table'. - simulator(list): A list of the different modules that can run in simulator mode. Possible + simulators(list): A list of the different modules that can run in simulator mode. Possible modules include: all, mount, camera, weather, night. Defaults to an empty list. Attributes: @@ -44,11 +44,16 @@ def __init__( self, observatory, state_machine_file=None, + simulators=None, *args, **kwargs): - # Explicitly call the base classes in the order we want + # Explicitly call the base classes in the order we want. PanBase.__init__(self, *args, **kwargs) + if simulators: + self.logger.warning(f'Using {simulators=}') + self.set_config('simulator', simulators) + assert isinstance(observatory, Observatory) self.name = self.get_config('name', default='Generic PANOPTES Unit') @@ -61,26 +66,28 @@ def __init__( self.logger.info(f'Making a POCS state machine from {state_machine_file}') PanStateMachine.__init__(self, state_machine_file, **kwargs) - # Add observatory object, which does the bulk of the work + # Add observatory object, which does the bulk of the work. self.observatory = observatory self.is_initialized = False self._free_space = None + self.run_once = kwargs.get('run_once', False) self._obs_run_retries = self.get_config('pocs.RETRY_ATTEMPTS', default=3) + self.connected = True + self.interrupted = False # We want to call and record the status on a periodic interval. - def get_status(): - while True: + def get_periodic_status(): + while self.connected: status = self.status - self.logger.debug(f'Periodic status call: {status!r}') + self.logger.trace(f'Periodic status call: {status!r}') self.db.insert_current('status', status) CountdownTimer(self.get_config('status_check_interval', default=60)).sleep() - self._status_thread = Thread(target=get_status, daemon=True) + self._status_thread = Thread(target=get_periodic_status, daemon=True) self._status_thread.start() - self.connected = True self.say("Hi there!") @property @@ -116,14 +123,6 @@ def connected(self): def connected(self, new_value): self.set_config('pocs.CONNECTED', new_value) - @property - def keep_running(self): - return self.get_config('pocs.KEEP_RUNNING', default=True) - - @keep_running.setter - def keep_running(self, new_value): - self.set_config('pocs.KEEP_RUNNING', new_value) - @property def do_states(self): return self.get_config('pocs.DO_STATES', default=True) @@ -132,8 +131,30 @@ def do_states(self): def do_states(self, new_value): self.set_config('pocs.DO_STATES', new_value) + @property + def keep_running(self): + """If POCS should keep running. + + Currently reads: + + * `connected` + * `do_states` + * `observatory.can_observe` + + Returns: + bool: If POCS should keep running. + """ + return self.connected and self.do_states and self.observatory.can_observe + @property def run_once(self): + """If POCS should exit the run loop after a single iteration. + + This value reads the `pocs.RUN_ONCE` config value. + + Returns: + bool: if machine should stop after single iteration, default False. + """ return self.get_config('pocs.RUN_ONCE', default=False) @run_once.setter @@ -172,6 +193,10 @@ def initialize(self): bool: True if all initialization succeeded, False otherwise. """ + if not self.observatory.can_observe: + self.say("Looks like we're missing some required hardware.") + return False + if not self.is_initialized: self.logger.info('*' * 80) self.say("Initializing the system! Woohoo!") @@ -186,6 +211,7 @@ def initialize(self): self.power_down() else: self.is_initialized = True + self.do_states = True return self.is_initialized @@ -236,14 +262,12 @@ def power_down(self): # Observatory shut down self.observatory.power_down() - self.keep_running = True - self.do_states = True - self.is_initialized = False - self.interrupted = False self.connected = False + self._status_thread.join(1) + # Clear all the config items. - self.logger.info("Power down complete") + self.logger.success("Power down complete") def reset_observing_run(self): """Reset an observing run loop. """ @@ -290,8 +314,12 @@ def is_safe(self, no_warning=False, horizon='observe'): # Check weather is_safe_values['good_weather'] = self.is_weather_safe() - # Hard-drive space - is_safe_values['free_space'] = self.has_free_space() + # Hard-drive space in root + is_safe_values['free_space_root'] = self.has_free_space('/') + + # Hard-drive space in images directory. + images_dir = self.get_config('directories.images') + is_safe_values['free_space_images'] = self.has_free_space(images_dir) safe = all(is_safe_values.values()) @@ -316,6 +344,15 @@ def is_safe(self, no_warning=False, horizon='observe'): return safe + def _in_simulator(self, key): + """Checks the config server for the given simulator key value.""" + with suppress(KeyError): + if key in self.get_config('simulator', default=list()): + self.logger.debug(f'Using {key} simulator') + return True + + return False + def is_dark(self, horizon='observe'): """Is it dark @@ -323,7 +360,7 @@ def is_dark(self, horizon='observe'): entry `location.flat_horizon` by default. Args: - horizon (str, optional): Which horizon to use, 'flat''focus', or + horizon (str, optional): Which horizon to use, 'flat', 'focus', or 'observe' (default). Returns: @@ -334,11 +371,8 @@ def is_dark(self, horizon='observe'): is_dark = self.observatory.is_dark(horizon=horizon) self.logger.debug(f'Observatory is_dark: {is_dark}') - # Check simulator - with suppress(KeyError): - if 'night' in self.get_config('simulator', default=[]): - self.logger.debug(f'Using night simulator') - is_dark = True + if self._in_simulator('night'): + return True self.logger.debug(f"Dark Check: {is_dark}") return is_dark @@ -356,18 +390,12 @@ def is_weather_safe(self, stale=180): # Always assume False self.logger.debug("Checking weather safety") - is_safe = False - # Check if we are using weather simulator - simulator_values = self.get_config('simulator', default=[]) - if len(simulator_values): - self.logger.debug(f'simulator_values: {simulator_values}') - - if 'weather' in simulator_values: - self.logger.info("Weather simulator always safe") + if self._in_simulator('weather'): return True # Get current weather readings from database + is_safe = False try: record = self.db.get_current('weather') if record is None: @@ -378,12 +406,11 @@ def is_weather_safe(self, stale=180): timestamp = record['date'].replace(tzinfo=None) # current_time is timezone naive age = (current_time().datetime - timestamp).total_seconds() - self.logger.debug(f"Weather Safety: {is_safe} [{age:.0f} sec old - {timestamp:%Y-%m-%d %H:%M:%S}]") + self.logger.debug( + f"Weather Safety: {is_safe} [{age:.0f} sec old - {timestamp:%Y-%m-%d %H:%M:%S}]") - except (TypeError, KeyError) as e: - self.logger.warning(f"No record found in DB: {e!r}") except Exception as e: # pragma: no cover - self.logger.error(f"Error checking weather: {e!r}") + self.logger.error(f"No weather record in database: {e!r}") else: if age >= stale: self.logger.warning("Weather record looks stale, marking unsafe.") @@ -391,10 +418,12 @@ def is_weather_safe(self, stale=180): return is_safe - def has_free_space(self, required_space=0.25 * u.gigabyte, low_space_percent=1.5): + def has_free_space(self, directory=None, required_space=0.25 * u.gigabyte, low_space_percent=1.5): """Does hard drive have disk space (>= 0.5 GB). Args: + directory (str, optional): The path to check free space on, the default + `None` will check `$PANDIR`. required_space (u.gigabyte, optional): Amount of free space required for operation low_space_percent (float, optional): Give warning if space is less @@ -404,8 +433,9 @@ def has_free_space(self, required_space=0.25 * u.gigabyte, low_space_percent=1.5 Returns: bool: True if enough space """ + directory = directory or os.getenv('PANDIR') req_space = required_space.to(u.gigabyte) - self._free_space = get_free_space() + self._free_space = get_free_space(directory=directory) space_is_low = self._free_space.value <= (req_space.value * low_space_percent) @@ -413,9 +443,9 @@ def has_free_space(self, required_space=0.25 * u.gigabyte, low_space_percent=1.5 has_space = bool(self._free_space.value >= req_space.value) if not has_space: - self.logger.error(f'No disk space: Free {self._free_space:.02f}\tReq: {req_space:.02f}') - elif space_is_low: - self.logger.warning(f'Low disk space: Free {self._free_space:.02f}\tReq: {req_space:.02f}') + self.logger.error(f'No disk space for {directory=}: Free {self._free_space:.02f}\t {req_space=:.02f}') + elif space_is_low: # pragma: no cover + self.logger.warning(f'Low disk space for {directory=}: Free {self._free_space:.02f}\t {req_space=:.02f}') return has_space @@ -438,11 +468,7 @@ def has_ac_power(self, stale=90): self.logger.debug("Checking for AC power") has_power = False - # TODO(wtgee): figure out if we really want to simulate no power - # Check if we are using power simulator - simulator_values = self.get_config('simulator', default=[]) - if 'power' in simulator_values: - self.logger.debug("AC power simulator always safe") + if self._in_simulator('power'): return True # Get current power readings from database @@ -459,7 +485,8 @@ def has_ac_power(self, stale=90): timestamp = record['date'].replace(tzinfo=None) # current_time is timezone naive age = (current_time().datetime - timestamp).total_seconds() - self.logger.debug(f"Power Safety: {has_power} [{age:.0f} sec old - {timestamp:%Y-%m-%d %H:%M:%S}]") + self.logger.debug( + f"Power Safety: {has_power} [{age:.0f} sec old - {timestamp:%Y-%m-%d %H:%M:%S}]") except (TypeError, KeyError) as e: self.logger.warning(f"No record found in DB: {e!r}") @@ -489,7 +516,7 @@ def wait(self, delay=None): delay {float|None} -- Number of seconds to wait. If default `None`, look up value in config, otherwise 2.5 seconds. """ - if delay is None: + if delay is None: # pragma: no cover delay = self.get_config('wait_delay', default=2.5) sleep_timer = CountdownTimer(delay) @@ -501,36 +528,3 @@ def wait(self, delay=None): is_expired = sleep_timer.expired() self.logger.debug(f'Leaving wait timer: expired={is_expired}') return is_expired - - ################################################################################################## - # Class Methods - ################################################################################################## - - @classmethod - def check_environment(cls): - """ Checks to see if environment is set up correctly - - There are a number of environmental variables that are expected - to be set in order for PANOPTES to work correctly. This method just - sanity checks our environment and shuts down otherwise. - - PANDIR Base directory for PANOPTES - POCS Base directory for POCS - """ - if sys.version_info[:2] < (3, 0): # pragma: no cover - warnings.warn("POCS requires Python 3.x to run") - - pandir = os.getenv('PANDIR') - if not os.path.exists(pandir): - sys.exit(f"$PANDIR dir does not exist or is empty: {pandir}") - - pocs = os.getenv('POCS') - if pocs is None: # pragma: no cover - sys.exit('Please make sure $POCS environment variable is set') - - if not os.path.exists(pocs): - sys.exit(f"$POCS directory does not exist or is empty: {pocs}") - - if not os.path.exists(f"{pandir}/logs"): - print(f"Creating log dir at {pandir}/logs") - os.makedirs(f"{pandir}/logs") diff --git a/src/panoptes/pocs/dome/__init__.py b/src/panoptes/pocs/dome/__init__.py index e04b3a551..faf846257 100644 --- a/src/panoptes/pocs/dome/__init__.py +++ b/src/panoptes/pocs/dome/__init__.py @@ -8,7 +8,7 @@ logger = get_logger() -def create_dome_from_config(config_port='6563', *args, **kwargs): +def create_dome_from_config(*args, **kwargs): """If there is a dome specified in the config, create a driver for it. A dome needs a config. We assume that there is at most one dome in the config, i.e. we don't @@ -17,7 +17,7 @@ def create_dome_from_config(config_port='6563', *args, **kwargs): by a single dome driver class. """ - dome_config = get_config('dome', port=config_port) + dome_config = get_config('dome') if dome_config is None: logger.info('No dome in config.') @@ -26,25 +26,25 @@ def create_dome_from_config(config_port='6563', *args, **kwargs): brand = dome_config['brand'] driver = dome_config['driver'] - logger.debug('Creating dome: brand={}, driver={}'.format(brand, driver)) + logger.debug(f'Creating dome: {brand=}, {driver=}') module = load_module(f'panoptes.pocs.dome.{driver}') - dome = module.Dome(config_port=config_port, *args, **kwargs) + dome = module.Dome(*args, **kwargs) logger.info(f'Created dome driver: brand={brand}, driver={driver}') return dome -def create_dome_simulator(config_port=6563, *args, **kwargs): - dome_config = get_config('dome', port=config_port) +def create_dome_simulator(*args, **kwargs): + dome_config = get_config('dome') brand = dome_config['brand'] driver = dome_config['driver'] - logger.debug('Creating dome simulator: brand={}, driver={}'.format(brand, driver)) + logger.debug(f'Creating dome simulator: {brand=}, {driver=}') module = load_module(f'panoptes.pocs.dome.{driver}') - dome = module.Dome(config_port=config_port, *args, **kwargs) - logger.info('Created dome driver: brand={}, driver={}'.format(brand, driver)) + dome = module.Dome(*args, **kwargs) + logger.info(f'Created dome driver: {brand=}, {driver=}') return dome diff --git a/src/panoptes/pocs/dome/astrohaven.py b/src/panoptes/pocs/dome/astrohaven.py index 9d7e4add1..d756aac86 100644 --- a/src/panoptes/pocs/dome/astrohaven.py +++ b/src/panoptes/pocs/dome/astrohaven.py @@ -3,18 +3,23 @@ import time -from pocs.dome import abstract_serial_dome +from panoptes.pocs.dome import abstract_serial_dome class Protocol: - # Response codes - BOTH_CLOSED = '0' - BOTH_OPEN = '3' + # Status codes, produced when not responding to an input. They are oriented towards + # reporting whether the two shutters are fully closed. + BOTH_CLOSED = '0' # Both A and B shutters are fully closed. - # TODO(jamessynge): Confirm and clarify meaning of '1' and '2' - B_IS_OPEN = '1' - A_IS_OPEN = '2' + A_IS_CLOSED = '1' # Only shutter A is fully closed. + B_IS_CLOSED = '2' # Only shutter B is fully closed. + BOTH_OPEN = '3' # Really means both NOT fully closed. + + # Status codes produced by the dome when not responding to a movement command. + STABLE_STATES = (BOTH_CLOSED, BOTH_OPEN, B_IS_CLOSED, A_IS_CLOSED) + + # Limit responses, when the limit has been reached on a direction of movement. A_OPEN_LIMIT = 'x' # Response to asking for A to open, and being at open limit A_CLOSE_LIMIT = 'X' # Response to asking for A to close, and being at close limit @@ -43,8 +48,10 @@ class AstrohavenDome(abstract_serial_dome.AbstractSerialDome): """ # TODO(jamessynge): Get these from the config file (i.e. per instance), with these values # as defaults, though LISTEN_TIMEOUT can just be the timeout config for SerialData. - LISTEN_TIMEOUT = 3 # Max number of seconds to wait for a response - MOVE_TIMEOUT = 10 # Max number of seconds to run the door motors + LISTEN_TIMEOUT = 3 # Max number of seconds to wait for a response. + MOVE_TIMEOUT = 10 # Max number of seconds to run the door motors. + MOVE_LISTEN_TIMEOUT = 0.1 # When moving, how long to wait for feedback. + NUM_CLOSE_FEEDBACKS = 2 # Number of target_feedback bytes needed. def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -68,7 +75,11 @@ def is_open(self): def open(self): self._full_move(Protocol.OPEN_A, Protocol.A_OPEN_LIMIT) self._full_move(Protocol.OPEN_B, Protocol.B_OPEN_LIMIT) - return self.is_open + v = self._read_state_until_stable() + if v == Protocol.BOTH_OPEN: + return True + self.logger.warning(f'AstrohavenDome.open wrong final state: {v!r}') + return False @property def is_closed(self): @@ -76,25 +87,39 @@ def is_closed(self): return v == Protocol.BOTH_CLOSED def close(self): - self._full_move(Protocol.CLOSE_A, Protocol.A_CLOSE_LIMIT) - self._full_move(Protocol.CLOSE_B, Protocol.B_CLOSE_LIMIT) - return self.is_closed + self._full_move(Protocol.CLOSE_A, Protocol.A_CLOSE_LIMIT, + feedback_countdown=AstrohavenDome.NUM_CLOSE_FEEDBACKS) + self._full_move(Protocol.CLOSE_B, Protocol.B_CLOSE_LIMIT, + feedback_countdown=AstrohavenDome.NUM_CLOSE_FEEDBACKS) + v = self._read_state_until_stable() + if v == Protocol.BOTH_CLOSED: + return True + self.logger.warning(f'AstrohavenDome.close wrong final state: {v!r}') + return False @property def status(self): - """Return a text string describing dome's current status.""" - if not self.is_connected: - return 'Not connected to the dome' - v = self._read_latest_state() - if v == Protocol.BOTH_CLOSED: - return 'Both sides closed' - if v == Protocol.B_IS_OPEN: - return 'Side B open, side A closed' - if v == Protocol.A_IS_OPEN: - return 'Side A open, side B closed' - if v == Protocol.BOTH_OPEN: - return 'Both sides open' - return 'Unexpected response from Astrohaven Dome Controller: %r' % v + """Return a dict with dome's current status.""" + + status_lookup = { + Protocol.BOTH_CLOSED: 'closed_both', + Protocol.A_IS_CLOSED: 'closed_a', + Protocol.B_IS_CLOSED: 'closed_b', + Protocol.BOTH_OPEN: 'open_both', + } + + state = self._read_latest_state() + + return_status = dict( + connected=self.is_connected, + ) + + try: + return_status['open'] = status_lookup[state] + except KeyError as e: + return_status['open'] = f'Unexpected response from Astrohaven Dome Controller: {state!r}' + + return return_status def __str__(self): if self.is_connected: @@ -115,27 +140,23 @@ def _read_latest_state(self): return chr(data[-1]) return None - def _nudge_shutter(self, send, target_feedback): - """Send one command to the dome, return whether the desired feedback was received. - - Args: - send: The command code to send; this is a string of one ASCII character. See - Protocol above for the command codes. - target_feedback: The response code to compare to the response from the dome; - this is a string of one ASCII character. See Protocol above for the codes; - while the dome is moving, it echoes the command code sent. + def _read_state_until_stable(self): + """Read the status until it reaches one of the stable values.""" + end_by = time.time() + AstrohavenDome.LISTEN_TIMEOUT + c = '' + while True: + data = self.serial.read_bytes(size=1) + if data: + c = chr(data[-1]) + if c in Protocol.STABLE_STATES: + return c + self.logger.debug(f'_read_state_until_stable not yet stable: {data=!r}') + if time.time() < end_by: + continue + pass + return c - Returns: - True if the output from the dome is target_feedback; False otherwise. - """ - self.serial.write(send) - # Wait a moment so that the response to our command has time to be emitted, and we don't - # get fooled by a status code received at about the same time that our command is sent. - time.sleep(0.1) - feedback = self._read_latest_state() - return feedback == target_feedback - - def _full_move(self, send, target_feedback): + def _full_move(self, send, target_feedback, feedback_countdown=1): """Send a command code until the target_feedback is recieved, or a timeout is reached. Args: @@ -148,14 +169,45 @@ def _full_move(self, send, target_feedback): True if the target_feedback is received from the dome before the MOVE_TIMEOUT; False otherwise. """ - end_by = time.time() + AstrohavenDome.MOVE_TIMEOUT - while not self._nudge_shutter(send, target_feedback): - if time.time() < end_by: - continue - self.logger.error('Timed out moving the dome. Check for hardware or communications ' + - 'problem. send=%r latest_state=%r', send, self._read_latest_state()) - return False - return True + # Set a short timeout on reading, so that we don't open or close slowly. + # In other words, we'll try to read status, but if it isn't available, + # we'll just send another command. + saved_timeout = self.serial.ser.timeout + self.serial.ser.timeout = AstrohavenDome.MOVE_LISTEN_TIMEOUT + try: + have_seen_send = False + end_by = time.time() + AstrohavenDome.MOVE_TIMEOUT + self.serial.reset_input_buffer() + # Note that there is no wait in this loop because we have a timeout on reading from + # the the dome controller, and we know that the dome doesn't echo every character that + # we send to it. + while True: + self.serial.write(send) + data = self.serial.read_bytes(size=1) + if data: + c = chr(data[-1]) + if c == target_feedback: + feedback_countdown -= 1 + self.logger.debug(f'Got target_feedback, {feedback_countdown=}') + if feedback_countdown <= 0: + # Woot! Moved the dome and got the desired response. + return True + elif c == send: + have_seen_send = True + elif not have_seen_send and c in Protocol.STABLE_STATES: # pragma: no cover + # At the start of looping, we may see the previous stable state until + # we start seeing the echo of `send`. + pass + else: # pragma: no cover + self.logger.warning(f'Unexpected value from dome! {send=!r} {target_feedback=!r} {data=!r}') + if time.time() < end_by: + continue + self.logger.error( + f'Timed out moving the dome. Check for hardware or communications problem. ' + f'{send=!r} {target_feedback=!r} {data=!r}') + return False + finally: + self.serial.ser.timeout = saved_timeout # Expose as Dome so that we can generically load by module name, without knowing the specific type diff --git a/src/panoptes/pocs/dome/protocol_astrohaven_simulator.py b/src/panoptes/pocs/dome/protocol_astrohaven_simulator.py index 9d58913be..23f41f83e 100644 --- a/src/panoptes/pocs/dome/protocol_astrohaven_simulator.py +++ b/src/panoptes/pocs/dome/protocol_astrohaven_simulator.py @@ -1,13 +1,12 @@ import datetime import queue from serial import serialutil -import sys import threading import time -from pocs.dome import astrohaven -from pocs.tests import serial_handlers -import pocs.utils.logger +from panoptes.pocs.dome import astrohaven +from panoptes.utils import serial_handlers +from panoptes.pocs.utils.logger import get_logger Protocol = astrohaven.Protocol CLOSED_POSITION = 0 @@ -38,24 +37,22 @@ def __init__(self, side, open_command, close_command, is_open_char, is_closed_ch self.max_position = max(CLOSED_POSITION, OPEN_POSITION) def handle_input(self, input_char): - ts = datetime.datetime.now() - msg = ts.strftime('%M:%S.%f') if input_char in self.open_commands: if self.is_open: return (False, self.is_open_char) - self.logger.info('Opening side %s, starting position %r' % (self.side, self.position)) + self.logger.debug(f'Opening side {self.side}, starting position {self.position}') self.adjust_position(NUDGE_OPEN_INCREMENT) if self.is_open: - self.logger.info('Opened side %s' % self.side) + self.logger.debug(f'Opened side {self.side}') return (True, self.is_open_char) return (True, input_char) elif input_char in self.close_commands: if self.is_closed: return (False, self.is_closed_char) - self.logger.info('Closing side %s, starting position %r' % (self.side, self.position)) + self.logger.debug(f'Closing side {self.side}, starting position {self.position}') self.adjust_position(NUDGE_CLOSED_INCREMENT) if self.is_closed: - self.logger.info('Closed side %s' % self.side) + self.logger.debug(f'Closed side {self.side}') return (True, self.is_closed_char) return (True, input_char) else: @@ -119,7 +116,7 @@ def run(self): return now = datetime.datetime.now() remaining = (self.next_output_time - now).total_seconds() - self.logger.info('AstrohavenPLCSimulator.run remaining=%r' % remaining) + self.logger.info(f'AstrohavenPLCSimulator.run remaining={remaining}') if remaining <= 0: self.do_output() continue @@ -128,7 +125,7 @@ def run(self): except queue.Empty: continue if self.handle_input(c): - # This sleep is here to reflect the fact that responses from the Astrohaven PLC + # This wait is here to reflect the fact that responses from the Astrohaven PLC # don't appear to be instantaneous, and the Wheaton originated driver had pauses # and drains of input from the PLC before accepting a response. time.sleep(0.2) @@ -142,7 +139,7 @@ def do_output(self): c = self.next_output_code if not c: c = self.compute_state() - self.logger.info('AstrohavenPLCSimulator.compute_state -> {!r}', c) + self.logger.debug(f'AstrohavenPLCSimulator.compute_state -> {c!r}') self.next_output_code = None # We drop output if the queue is full. if not self.status_queue.full(): @@ -150,7 +147,7 @@ def do_output(self): self.next_output_time = datetime.datetime.now() + self.delta def handle_input(self, c): - self.logger.info('AstrohavenPLCSimulator.handle_input {!r}', c) + self.logger.debug(f'AstrohavenPLCSimulator.handle_input {c!r}') (a_acted, a_resp) = self.shutter_a.handle_input(c) (b_acted, b_resp) = self.shutter_b.handle_input(c) # Use a_resp if a_acted or if there is no b_resp @@ -175,9 +172,9 @@ def compute_state(self): if self.shutter_b.is_closed: return Protocol.BOTH_CLOSED else: - return Protocol.B_IS_OPEN + return Protocol.A_IS_CLOSED elif self.shutter_b.is_closed: - return Protocol.A_IS_OPEN + return Protocol.B_IS_CLOSED else: return Protocol.BOTH_OPEN @@ -185,7 +182,7 @@ def compute_state(self): class AstrohavenSerialSimulator(serial_handlers.NoOpSerial): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.logger = pocs.utils.logger.get_root_logger() + self.logger = get_logger() self.plc_thread = None self.command_queue = queue.Queue(maxsize=50) self.status_queue = queue.Queue(maxsize=1000) @@ -259,7 +256,7 @@ def read(self, size=1): if timeout_obj.expired(): break response = bytes(response) - self.logger.info('AstrohavenSerialSimulator.read({}) -> {!r}', size, response) + self.logger.debug(f'AstrohavenSerialSimulator.read({size}) -> {response!r}') return response @property @@ -305,7 +302,7 @@ def write(self, data): if not isinstance(data, (bytes, bytearray)): raise ValueError("write takes bytes") data = bytes(data) # Make sure it can't change. - self.logger.info('AstrohavenSerialSimulator.write({!r})', data) + self.logger.info(f'AstrohavenSerialSimulator.write({data!r})') count = 0 timeout_obj = serialutil.Timeout(self.write_timeout) for b in data: @@ -360,7 +357,7 @@ def _reconfigure_port(self): _drain_queue(self.status_queue) self.stop.clear() self.plc_thread = threading.Thread( - name='Astrohaven PLC Simulator', target=lambda: self.plc.run()) + name='Astrohaven PLC Simulator', target=lambda: self.plc.run(), daemon=True) self.plc_thread.start() elif self.plc_thread and not need_thread: self.stop.set() diff --git a/src/panoptes/pocs/focuser/birger.py b/src/panoptes/pocs/focuser/birger.py index 4f4da4199..2140807bd 100644 --- a/src/panoptes/pocs/focuser/birger.py +++ b/src/panoptes/pocs/focuser/birger.py @@ -149,9 +149,9 @@ def __del__(self): self._serial_port.close() self.logger.debug('Closed serial port {}'.format(self._port)) -################################################################################################## -# Properties -################################################################################################## + ################################################################################################## + # Properties + ################################################################################################## @property def is_connected(self): @@ -211,9 +211,9 @@ def is_moving(self): """ True if the focuser is currently moving. """ return self._is_moving -################################################################################################## -# Public Methods -################################################################################################## + ################################################################################################## + # Public Methods + ################################################################################################## def connect(self, port): try: @@ -304,9 +304,9 @@ def move_by(self, increment): self.logger.debug("Moved by {} encoder units".format(moved_by)) return moved_by -################################################################################################## -# Private Methods -################################################################################################## + ################################################################################################## + # Private Methods + ################################################################################################## def _send_command(self, command, response_length=None, ignore_response=False): """ @@ -461,15 +461,15 @@ def _initialise_aperture(self): self.logger.debug('Initialising aperture motor') response = self._send_command('in', response_length=1)[0].rstrip() if response != 'DONE': - self.logger.error("{} got '{}', expected 'DONE'!".format(self, response)) + self.logger.error(f"{self} got {response=}, expected 'DONE'!") def _move_zero(self): response = self._send_command('mz', response_length=1)[0].rstrip() if response[:4] != 'DONE': - self.logger.error("{} got '{}', expected 'DONENNNNN,1'!".format(self, response)) + self.logger.error(f"{self} got {response=}, expected 'DONENNNNN,1'!") else: r = response[4:].rstrip() - self.logger.debug("Moved {} encoder units to close stop".format(r[:-2])) + self.logger.debug(f"Moved {r[:-2]} encoder units to close stop") return int(r[:-2]) def _zero_encoder(self): @@ -480,15 +480,15 @@ def _learn_focus_range(self): self.logger.debug('Learning absolute focus range') response = self._send_command('la', response_length=1)[0].rstrip() if response != 'DONE:LA': - self.logger.error("{} got '{}', expected 'DONE:LA'!".format(self, response)) + self.logger.error(f"{self} got {response=}, expected 'DONE:LA'!") def _move_inf(self): response = self._send_command('mi', response_length=1)[0].rstrip() if response[:4] != 'DONE': - self.logger.error("{} got '{}', expected 'DONENNNNN,1'!".format(self, response)) + self.logger.error(f"{self} got {response=}, expected 'DONENNNNN,1'!") else: r = response[4:].rstrip() - self.logger.debug("Moved {} encoder units to far stop".format(r[:-2])) + self.logger.debug(f"Moved {r[:-2]} encoder units to far stop") return int(r[:-2]) def _add_fits_keywords(self, header): diff --git a/src/panoptes/pocs/hardware.py b/src/panoptes/pocs/hardware.py index 940aea78e..e6be88023 100644 --- a/src/panoptes/pocs/hardware.py +++ b/src/panoptes/pocs/hardware.py @@ -1,43 +1,82 @@ """Information about hardware supported by Panoptes.""" +from panoptes.utils.config.client import get_config -ALL_NAMES = sorted(['camera', 'dome', 'mount', 'night', 'weather']) +ALL_NAMES = sorted([ + 'camera', + 'dome', + 'mount', + 'night', + 'power', + 'sensors', + 'theskyx', + 'weather', +]) -def get_all_names(all_names=ALL_NAMES, without=list()): +def get_all_names(all_names=ALL_NAMES, without=None): """Returns the names of all the categories of hardware that POCS supports. Note that this doesn't extend to the Arduinos for the telemetry and camera boards, for which no simulation is supported at this time. + + >>> from panoptes.pocs.hardware import get_all_names + >>> get_all_names() + ['camera', 'dome', 'mount', 'night', 'power', 'sensors', 'theskyx', 'weather'] + >>> get_all_names(without='mount') # Single item + ['camera', 'dome', 'night', 'power', 'sensors', 'theskyx', 'weather'] + >>> get_all_names(without=['mount', 'power']) # List + ['camera', 'dome', 'night', 'sensors', 'theskyx', 'weather'] + + >>> # You can alter available hardware if needed. + >>> get_all_names(['foo', 'bar', 'power'], without=['power']) + ['bar', 'foo'] + + Args: + all_names (list): The list of hardware. + without (iterable): Return all items expect those in the list. + + Returns: + list: The sorted list of available hardware except those listed in `without`. """ - return [v for v in all_names if v not in without] + # Make sure that 'all' gets expanded. + without = get_simulator_names(simulator=without) + return sorted([v for v in all_names if v not in without]) -def get_simulator_names(simulator=None, kwargs=None, config=None): + +def get_simulator_names(simulator=None, kwargs=None): """Returns the names of the simulators to be used in lieu of hardware drivers. Note that returning a list containing 'X' doesn't mean that the config calls for a driver of type 'X'; that is up to the code working with the config to create drivers for real or simulated hardware. - This funciton is intended to be called from PanBase or similar, which receives kwargs that - may include simulator, config or both. For example: - get_simulator_names(config=self.config, kwargs=kwargs) - Or: - get_simulator_names(simulator=simulator, config=self.config) + This function is intended to be called from `PanBase` or similar, which receives kwargs that + may include simulator, config or both. For example:: + + get_simulator_names(config=self.config, kwargs=kwargs) + + # Or: + + get_simulator_names(simulator=simulator, config=self.config) The reason this function doesn't just take **kwargs as its sole arg is that we need to allow for the case where the caller is passing in simulator (or config) twice, once on its own, and once in the kwargs (which won't be examined). Python doesn't permit a keyword argument to be passed in twice. + >>> from panoptes.pocs.hardware import get_simulator_names + >>> get_simulator_names() + [] + >>> get_simulator_names('all') + ['camera', 'dome', 'mount', 'night', 'power', 'sensors', 'theskyx', 'weather'] + + Args: - simulator: - An explicit list of names of hardware to be simulated (i.e. hardware drivers - to be replaced with simulators). - kwargs: - The kwargs passed in to the caller, which is inspected for an arg called 'simulator'. - config: - Dictionary created from pocs.yaml or similar. + simulator (list): An explicit list of names of hardware to be simulated + (i.e. hardware drivers to be replaced with simulators). + kwargs: The kwargs passed in to the caller, which is inspected for an arg + called 'simulator'. Returns: List of names of the hardware to be simulated. @@ -47,13 +86,13 @@ def get_simulator_names(simulator=None, kwargs=None, config=None): def extract_simulator(d): return (d or empty).get('simulator') - for v in [simulator, extract_simulator(kwargs), extract_simulator(config)]: + for v in [simulator, extract_simulator(kwargs), extract_simulator(get_config())]: if not v: continue if isinstance(v, str): v = [v] if 'all' in v: - return get_all_names() + return ALL_NAMES else: return sorted(v) return [] diff --git a/src/panoptes/pocs/mount/__init__.py b/src/panoptes/pocs/mount/__init__.py index 968b17fa9..2da0f085b 100644 --- a/src/panoptes/pocs/mount/__init__.py +++ b/src/panoptes/pocs/mount/__init__.py @@ -12,8 +12,7 @@ logger = get_logger() -def create_mount_from_config(config_port='6563', - mount_info=None, +def create_mount_from_config(mount_info=None, earth_location=None, *args, **kwargs): """Create a mount instance based on the provided config. @@ -23,7 +22,6 @@ def create_mount_from_config(config_port='6563', and the class must be called Mount. Args: - config_port: The port number of the config server, default 6563. mount_info: Optional param which overrides the 'mount' entry in config if provided. Useful for testing. earth_location: `astropy.coordinates.EarthLocation` instance, representing the @@ -45,7 +43,7 @@ def create_mount_from_config(config_port='6563', # If mount_info was not passed as a parameter, check config. if mount_info is None: logger.debug('No mount info provided, using values from config.') - mount_info = get_config('mount', default=None, port=config_port) + mount_info = get_config('mount', default=None) # If nothing in config, raise exception. if mount_info is None: @@ -56,7 +54,7 @@ def create_mount_from_config(config_port='6563', logger.debug('No location provided, using values from config.') # Get details from config. - site_details = create_location_from_config(config_port=config_port) + site_details = create_location_from_config() earth_location = site_details['earth_location'] driver = mount_info.get('driver') @@ -67,13 +65,13 @@ def create_mount_from_config(config_port='6563', logger.debug(f'Mount: driver={driver} model={model}') # Check if we should be using a simulator - use_simulator = 'mount' in get_config('simulator', default=[], port=config_port) + use_simulator = 'mount' in get_config('simulator', default=[]) logger.debug(f'Mount is simulator: {use_simulator}') # Create simulator if requested if use_simulator or (driver == 'simulator'): logger.debug(f'Creating mount simulator') - return create_mount_simulator() + return create_mount_simulator(mount_info=mount_info, earth_location=earth_location) # See if we have a serial connection try: @@ -97,32 +95,34 @@ def create_mount_from_config(config_port='6563', raise error.MountNotFound(e) # Make the mount include site information - mount = module.Mount(config_port=config_port, location=earth_location, *args, **kwargs) + mount = module.Mount(location=earth_location, *args, **kwargs) logger.success(f'{driver} mount created') return mount -def create_mount_simulator(config_port='6563', *args, **kwargs): +def create_mount_simulator(mount_info=None, + earth_location=None, + *args, **kwargs): # Remove mount simulator - current_simulators = get_config('simulator', default=[], port=config_port) + current_simulators = get_config('simulator', default=[]) logger.warning(f'Current simulators: {current_simulators}') with suppress(ValueError): current_simulators.remove('mount') - mount_config = { - 'model': 'simulator', + mount_config = mount_info or { + 'model': 'Mount Simulator', 'driver': 'simulator', 'serial': { - 'port': 'simulator' + 'port': '/dev/FAKE' } } # Set mount device info to simulator - set_config('mount', mount_config, port=config_port) + set_config('mount', mount_config) - earth_location = create_location_from_config(config_port=config_port)['earth_location'] + earth_location = earth_location or create_location_from_config()['earth_location'] logger.debug(f"Loading mount driver: pocs.mount.{mount_config['driver']}") try: @@ -130,8 +130,8 @@ def create_mount_simulator(config_port='6563', *args, **kwargs): except error.NotFound as e: raise error.MountNotFound(f'Error loading mount module: {e!r}') - mount = module.Mount(location=earth_location, config_port=config_port, *args, **kwargs) + mount = module.Mount(earth_location, *args, **kwargs) - logger.success(f"{mount_config['driver']} mount created") + logger.success(f"{mount_config['driver'].title()} mount created") return mount diff --git a/src/panoptes/pocs/mount/mount.py b/src/panoptes/pocs/mount/mount.py index 742400a24..cb0fc486d 100644 --- a/src/panoptes/pocs/mount/mount.py +++ b/src/panoptes/pocs/mount/mount.py @@ -1,4 +1,5 @@ import time +from contextlib import suppress from astropy import units as u from astropy.coordinates import EarthLocation @@ -43,7 +44,7 @@ def __init__(self, location, commands=None, *args, **kwargs): # Create an object for just the mount config items self.mount_config = self.get_config('mount') - self.logger.debug("Mount config: {}".format(self.mount_config)) + self.logger.debug(f"Mount config: {self.mount_config}") # setup commands for mount self.logger.debug("Setting up commands for mount") @@ -87,6 +88,14 @@ def __init__(self, location, commands=None, *args, **kwargs): self._current_coordinates = None self._park_coordinates = None + def __str__(self): + brand = self.mount_config.get('brand', '') + model = self.mount_config.get('model', '') + port = '' + with suppress(KeyError): + port = self.mount_config['serial']['port'] + return f'{brand} {model} ({port})' + def connect(self): # pragma: no cover raise NotImplementedError @@ -153,7 +162,7 @@ def is_connected(self): @property def is_initialized(self): - """ bool: Has mount been initialied with connection """ + """ bool: Has mount been initialised with connection """ return self._is_initialized @property diff --git a/src/panoptes/pocs/mount/simulator.py b/src/panoptes/pocs/mount/simulator.py index cc4ec2f7d..16e05f303 100644 --- a/src/panoptes/pocs/mount/simulator.py +++ b/src/panoptes/pocs/mount/simulator.py @@ -9,7 +9,6 @@ class Mount(AbstractMount): - """Mount class for a simulator. Use this when you don't actually have a mount attached. """ @@ -26,14 +25,13 @@ def __init__(self, location, commands=dict(), *args, **kwargs): self.logger.debug('Simulator mount created') + ################################################################################################## + # Properties + ################################################################################################## -################################################################################################## -# Properties -################################################################################################## - -################################################################################################## -# Public Methods -################################################################################################## + ################################################################################################## + # Public Methods + ################################################################################################## def initialize(self, unpark=False, *arg, **kwargs): """ Initialize the connection with the mount and setup for location. @@ -61,7 +59,7 @@ def initialize(self, unpark=False, *arg, **kwargs): if unpark: self.unpark() - return self.is_initialized + return self._is_initialized def connect(self): self.logger.debug("Connecting to mount simulator") @@ -196,16 +194,14 @@ def set_tracking_rate(self, direction='ra', delta=0.0): self.tracking_rate = 1.0 + delta self.logger.debug("Custom tracking rate sent") - -################################################################################################## -# Private Methods -################################################################################################## + ################################################################################################## + # Private Methods + ################################################################################################## def _setup_location_for_mount(self): """Sets the mount up to the current location. Mount must be initialized first. """ assert self.is_initialized, self.logger.warning('Mount has not been initialized') - assert self.location is not None, self.logger.warning( - 'Please set a location before attempting setup') + assert self.location is not None, self.logger.warning('Please set a location before attempting setup') self.logger.debug('Setting up mount for location') diff --git a/src/panoptes/pocs/observatory.py b/src/panoptes/pocs/observatory.py index b18ddb202..9c22c5ccd 100644 --- a/src/panoptes/pocs/observatory.py +++ b/src/panoptes/pocs/observatory.py @@ -1,7 +1,7 @@ import os import subprocess from collections import OrderedDict -import pendulum +from datetime import datetime from astropy import units as u from astropy.coordinates import get_moon @@ -28,11 +28,14 @@ def __init__(self, cameras=None, scheduler=None, dome=None, mount=None, *args, * dates and weather station. Adds cameras, scheduler, dome and mount. """ super().__init__(*args, **kwargs) + self.scheduler = None + self.dome = None + self.mount = None self.logger.info('Initializing observatory') # Setup information about site location self.logger.info('Setting up location') - site_details = create_location_from_config(config_port=self.config_port) + site_details = create_location_from_config() self.location = site_details['location'] self.earth_location = site_details['earth_location'] self.observer = site_details['observer'] @@ -51,11 +54,12 @@ def __init__(self, cameras=None, scheduler=None, dome=None, mount=None, *args, * self._primary_camera = None if cameras: - self.logger.info(f'Adding the cameras to the observatory: {cameras}') + self.logger.info(f'Adding cameras to the observatory: {cameras}') for cam_name, camera in cameras.items(): self.add_camera(cam_name, camera) - # TODO(jamessynge): Figure out serial port validation behavior here compared to that for the mount. + # TODO(jamessynge): Figure out serial port validation behavior here compared to that for + # the mount. self.set_dome(dome) self.set_scheduler(scheduler) @@ -172,7 +176,7 @@ def can_observe(self): if can_observe is False: for check_name, is_true in checks.items(): if not is_true: - self.logger.warning(f'{check_name.title()} not present, cannot observe') + self.logger.warning(f'{check_name.title()} not present') return can_observe @@ -190,7 +194,8 @@ def add_camera(self, cam_name, camera): assert isinstance(camera, AbstractCamera) self.logger.debug(f'Adding {cam_name}: {camera}') if cam_name in self.cameras: - self.logger.debug(f'{cam_name} already exists, replacing existing camera under that name.') + self.logger.debug( + f'{cam_name} already exists, replacing existing camera under that name.') self.cameras[cam_name] = camera if camera.is_primary: @@ -216,42 +221,35 @@ def set_scheduler(self, scheduler): Args: scheduler (`pocs.scheduler.BaseScheduler`): An instance of the `~BaseScheduler` class. """ - if isinstance(scheduler, BaseScheduler): - self.logger.info('Adding scheduler.') - self.scheduler = scheduler - elif scheduler is None: - self.logger.info('Removing scheduler.') - self.scheduler = None - else: - raise TypeError("Scheduler is not instance of BaseScheduler class, cannot add.") + self._set_hardware(scheduler, 'scheduler', BaseScheduler) def set_dome(self, dome): """Set's dome or remove the dome for the `Observatory`. Args: dome (`pocs.dome.AbstractDome`): An instance of the `~AbstractDome` class. """ - if isinstance(dome, AbstractDome): - self.logger.info('Adding dome.') - self.dome = dome - elif dome is None: - self.logger.info('Removing dome.') - self.dome = None - else: - raise TypeError('Dome is not instance of AbstractDome class, cannot add.') + self._set_hardware(dome, 'dome', AbstractDome) def set_mount(self, mount): """Sets the mount for the `Observatory`. Args: mount (`pocs.mount.AbstractMount`): An instance of the `~AbstractMount` class. """ - if isinstance(mount, AbstractMount): - self.logger.info('Adding mount') - self.mount = mount - elif mount is None: - self.logger.info('Removing mount') - self.mount = None + self._set_hardware(mount, 'mount', AbstractMount) + + def _set_hardware(self, new_hardware, hw_type, hw_class): + # Lookup the set method for the hardware type. + hw_attr = getattr(self, hw_type) + + if isinstance(new_hardware, hw_class): + self.logger.success(f'Adding {new_hardware}') + setattr(self, hw_type, new_hardware) + elif new_hardware is None: + if hw_attr is not None: + self.logger.success(f'Removing {hw_attr=}') + setattr(self, hw_type, None) else: - raise TypeError("Mount is not instance of AbstractMount class, cannot add.") + raise TypeError(f"{hw_type.title()} is not an instance of {str(hw_class)} class") ########################################################################## # Methods @@ -265,7 +263,7 @@ def initialize(self): self.dome.connect() def power_down(self): - """Power down the observatory. Currently does nothing + """Power down the observatory. Currently just disconnects hardware. """ self.logger.debug("Shutting down observatory") if self.mount: @@ -281,13 +279,14 @@ def status(self): now = current_time() try: - if self.mount.is_initialized: + if self.mount and self.mount.is_initialized: status['mount'] = self.mount.status current_coords = self.mount.get_current_coordinates() status['mount']['current_ha'] = self.observer.target_hour_angle(now, current_coords) if self.mount.has_target: target_coords = self.mount.get_target_coordinates() - status['mount']['mount_target_ha'] = self.observer.target_hour_angle(now, target_coords) + status['mount']['mount_target_ha'] = self.observer.target_hour_angle(now, + target_coords) except Exception as e: # pragma: no cover self.logger.warning(f"Can't get mount status: {e!r}") @@ -300,7 +299,8 @@ def status(self): try: if self.current_observation: status['observation'] = self.current_observation.status - status['observation']['field_ha'] = self.observer.target_hour_angle(now, self.current_observation.field) + status['observation']['field_ha'] = self.observer.target_hour_angle(now, + self.current_observation.field) except Exception as e: # pragma: no cover self.logger.warning(f"Can't get observation status: {e!r}") @@ -308,7 +308,7 @@ def status(self): status['observer'] = { 'siderealtime': str(self.sidereal_time), 'utctime': now, - 'localtime': pendulum.now(), + 'localtime': datetime.now(), 'local_evening_astro_time': self._evening_astro_time, 'local_morning_astro_time': self._morning_astro_time, 'local_sun_set_time': self._local_sunset, @@ -395,18 +395,17 @@ def cleanup_observations(self, upload_images=None, make_timelapse=None, keep_jpg 'fields', observation.field.field_name ) - self.logger.debug('Searching directory: {}', observation_dir) + self.logger.debug(f'Searching directory: {observation_dir}') for cam_name, camera in self.cameras.items(): - self.logger.debug('Cleanup for camera {} [{}]'.format( - cam_name, camera.uid)) + self.logger.debug(f'Cleanup for camera {cam_name} [{camera.uid}]') seq_dir = os.path.join( observation_dir, camera.uid, seq_time ) - self.logger.info('Cleaning directory {}'.format(seq_dir)) + self.logger.info(f'Cleaning directory {seq_dir}') process_cmd = [ process_script_path, @@ -501,7 +500,7 @@ def analyze_recent(self): # Get the image to compare image_id, image_path = self.current_observation.last_exposure - current_image = Image(image_path, location=self.earth_location, config_port=self._config_port) + current_image = Image(image_path, location=self.earth_location) solve_info = current_image.solve_field(skip_solved=False) diff --git a/src/panoptes/pocs/scheduler/__init__.py b/src/panoptes/pocs/scheduler/__init__.py index 88e15f0f7..691052824 100644 --- a/src/panoptes/pocs/scheduler/__init__.py +++ b/src/panoptes/pocs/scheduler/__init__.py @@ -6,8 +6,7 @@ from panoptes.pocs.scheduler.constraint import Duration from panoptes.pocs.scheduler.constraint import MoonAvoidance -# Below is needed for import -from panoptes.pocs.scheduler.scheduler import BaseScheduler # pragma: no flakes +from panoptes.pocs.scheduler.scheduler import BaseScheduler # noqa; needed for import from panoptes.utils import error from panoptes.utils import horizon as horizon_utils from panoptes.utils.library import load_module @@ -17,12 +16,12 @@ from panoptes.pocs.utils.location import create_location_from_config -def create_scheduler_from_config(config_port=6563, observer=None, *args, **kwargs): +def create_scheduler_from_config(observer=None, *args, **kwargs): """ Sets up the scheduler that will be used by the observatory """ logger = get_logger() - scheduler_config = get_config('scheduler', default=None, port=config_port) + scheduler_config = get_config('scheduler', default=None) logger.info(f'scheduler_config: {scheduler_config!r}') if scheduler_config is None or len(scheduler_config) == 0: @@ -31,15 +30,15 @@ def create_scheduler_from_config(config_port=6563, observer=None, *args, **kwarg if not observer: logger.debug(f'No Observer provided, creating from config.') - site_details = create_location_from_config(config_port=config_port) + site_details = create_location_from_config() observer = site_details['observer'] scheduler_type = scheduler_config.get('type', 'dispatch') # Read the targets from the file fields_file = scheduler_config.get('fields_file', 'simple.yaml') - fields_path = os.path.join(get_config('directories.targets', port=config_port), fields_file) - logger.debug('Creating scheduler: {}'.format(fields_path)) + fields_path = os.path.join(get_config('directories.targets'), fields_file) + logger.debug(f'Creating scheduler: {fields_path}') if os.path.exists(fields_path): @@ -47,9 +46,9 @@ def create_scheduler_from_config(config_port=6563, observer=None, *args, **kwarg # Load the required module module = load_module(f'panoptes.pocs.scheduler.{scheduler_type}') - obstruction_list = get_config('location.obstructions', default=[], port=config_port) + obstruction_list = get_config('location.obstructions', default=[]) default_horizon = get_config( - 'location.horizon', default=30 * u.degree, port=config_port) + 'location.horizon', default=30 * u.degree) horizon_line = horizon_utils.Horizon( obstructions=obstruction_list, @@ -58,21 +57,20 @@ def create_scheduler_from_config(config_port=6563, observer=None, *args, **kwarg # Simple constraint for now constraints = [ - Altitude(horizon=horizon_line, config_port=config_port), - MoonAvoidance(config_port=config_port), - Duration(default_horizon, weight=5., config_port=config_port) + Altitude(horizon=horizon_line), + MoonAvoidance(), + Duration(default_horizon, weight=5.) ] # Create the Scheduler instance scheduler = module.Scheduler(observer, fields_file=fields_path, constraints=constraints, - config_port=config_port) + *args, **kwargs) logger.debug("Scheduler created") except error.NotFound as e: raise error.NotFound(msg=e) else: - raise error.NotFound( - msg="Fields file does not exist: {}".format(fields_file)) + raise error.NotFound(msg=f"Fields file does not exist: {fields_file=}") return scheduler diff --git a/src/panoptes/pocs/scheduler/field.py b/src/panoptes/pocs/scheduler/field.py index 1b295faf7..d2055c721 100644 --- a/src/panoptes/pocs/scheduler/field.py +++ b/src/panoptes/pocs/scheduler/field.py @@ -14,18 +14,17 @@ def __init__(self, name, position, equinox='J2000', *args, **kwargs): Arguments: name {str} -- Name of the field, typically the name of object at - center `position` + center `position`. position {str} -- Center of field, can be anything accepted by - `~astropy.coordinates.SkyCoord` + `~astropy.coordinates.SkyCoord`. **kwargs {dict} -- Additional keywords to be passed to - `astroplan.ObservingBlock` + `astroplan.ObservingBlock`. """ PanBase.__init__(self, *args, **kwargs) - # Force an equinox - if equinox is None: - equinox = 'J2000' + # Force an equinox if they pass None (legacy). + equinox = equinox or 'J2000' super().__init__(SkyCoord(position, equinox=equinox, frame='icrs'), name=name, **kwargs) @@ -33,22 +32,10 @@ def __init__(self, name, position, equinox='J2000', *args, **kwargs): if not self._field_name: raise ValueError('Name is empty') - ################################################################################################## - # Properties - ################################################################################################## - @property def field_name(self): """ Flattened field name appropriate for paths """ return self._field_name - ################################################################################################## - # Methods - ################################################################################################## - - ################################################################################################## - # Private Methods - ################################################################################################## - def __str__(self): return self.name diff --git a/src/panoptes/pocs/scheduler/observation.py b/src/panoptes/pocs/scheduler/observation.py index ddc80f841..0baf68437 100644 --- a/src/panoptes/pocs/scheduler/observation.py +++ b/src/panoptes/pocs/scheduler/observation.py @@ -1,7 +1,7 @@ import os -from astropy import units as u from collections import OrderedDict +from astropy import units as u from panoptes.pocs.base import PanBase from panoptes.pocs.scheduler.field import Field @@ -11,7 +11,7 @@ class Observation(PanBase): @u.quantity_input(exptime=u.second) def __init__(self, field, exptime=120 * u.second, min_nexp=60, exp_set_size=10, priority=100, filter_name=None, *args, **kwargs): - """ An observation of a given `~pocs.scheduler.field.Field`. + """ An observation of a given `panoptes.pocs.scheduler.field.Field`. An observation consists of a minimum number of exposures (`min_nexp`) that must be taken at a set exposure time (`exptime`). These exposures come diff --git a/src/panoptes/pocs/scheduler/scheduler.py b/src/panoptes/pocs/scheduler/scheduler.py index dbe324a29..0c439aa7f 100644 --- a/src/panoptes/pocs/scheduler/scheduler.py +++ b/src/panoptes/pocs/scheduler/scheduler.py @@ -42,32 +42,26 @@ def __init__(self, observer, fields_list=None, fields_file=None, constraints=Non assert isinstance(observer, Observer) - self._fields_file = fields_file + self._observations = dict() + self._current_observation = None + self._fields_list = fields_list + + self.fields_file = fields_file # Setting the fields_list directly will clobber anything # from the fields_file. It comes second so we can specifically # clobber if passed. - self._fields_list = fields_list - self._observations = dict() self.observer = observer - self.constraints = constraints or list() - - self._current_observation = None self.observed_list = OrderedDict() - if not self.get_config('scheduler.check_file', default=False): + if self.get_config('scheduler.check_file', default=True): self.logger.debug("Reading initial set of fields") self.read_field_list() # Items common to each observation that shouldn't be computed each time. - self.common_properties = None - ########################################################################## - # Properties - ########################################################################## - @property def status(self): return { @@ -155,10 +149,7 @@ def fields_file(self, new_file): self.clear_available_observations() self._fields_file = new_file - if new_file is not None: - assert os.path.exists(new_file), \ - self.logger.error("Cannot load field list: {}".format(new_file)) - self.read_field_list() + self.read_field_list() @property def fields_list(self): @@ -184,10 +175,6 @@ def fields_list(self, new_list): self._fields_list = new_list self.read_field_list() - ########################################################################## - # Methods - ########################################################################## - def clear_available_observations(self): """Reset the list of available observations""" # Clear out existing list and observations @@ -229,27 +216,24 @@ def add_observation(self, field_config): Args: field_config (dict): Configuration items for `Observation` """ - if 'exptime' in field_config: - field_config['exptime'] = float(get_quantity_value( - field_config['exptime'], unit=u.second)) * u.second + with suppress(KeyError): + field_config['exptime'] = float(get_quantity_value(field_config['exptime'], unit=u.second)) * u.second - self.logger.debug("Adding {} to scheduler", field_config['name']) - field = Field(field_config['name'], field_config['position'], - config_port=self._config_port) - self.logger.debug("Created {} Field", field_config['name']) + self.logger.debug(f"Adding {field_config=} to scheduler") + field = Field(field_config['name'], field_config['position']) + self.logger.debug(f"Created {field.name=}") try: self.logger.debug(f"Creating observation for {field_config!r}") - obs = Observation(field, config_port=self._config_port, **field_config) - self.logger.debug(f"Observation created {obs}") + obs = Observation(field, **field_config) + self.logger.debug(f"Observation created for {field.name=}") except Exception as e: raise error.InvalidObservation(f"Skipping invalid field: {field_config!r} {e!r}") else: - self.logger.debug(f"Checking if {field.name} in self._observations") if field.name in self._observations: - self.logger.debug("Overriding existing entry for {}".format(field.name)) + self.logger.debug(f"Overriding existing entry for {field.name=}") self._observations[field.name] = obs - self.logger.debug(f"{obs} added") + self.logger.debug(f"{obs=} added") def remove_observation(self, field_name): """Removes an `Observation` from the scheduler @@ -265,8 +249,8 @@ def remove_observation(self, field_name): def read_field_list(self): """Reads the field file and creates valid `Observations` """ + self.logger.debug(f'Reading fields from file: {self.fields_file}') if self._fields_file is not None: - self.logger.debug('Reading fields from file: {}'.format(self.fields_file)) if not os.path.exists(self.fields_file): raise FileNotFoundError diff --git a/src/panoptes/pocs/sensors/arduino_io.py b/src/panoptes/pocs/sensors/arduino_io.py index a379d8dba..be4c8a2ef 100644 --- a/src/panoptes/pocs/sensors/arduino_io.py +++ b/src/panoptes/pocs/sensors/arduino_io.py @@ -52,7 +52,7 @@ def detect_board_on_port(port): attribute in the top-level object. Else returns None. """ logger = get_logger() - logger.debug('Attempting to connect to serial port: {}'.format(port)) + logger.debug(f'Attempting to connect to serial port: {port}') serial_reader = None try: # First open a connection to the device. @@ -60,9 +60,9 @@ def detect_board_on_port(port): serial_reader = open_serial_device(port) if not serial_reader.is_connected: serial_reader.connect() - logger.debug('Connected to {}', port) + logger.debug(f'Connected to {port=}') except Exception: - logger.warning('Could not connect to port: {}'.format(port)) + logger.warning(f'Could not connect to {port=}') return None try: reading = serial_reader.get_and_parse_reading(retry_limit=3) @@ -71,10 +71,10 @@ def detect_board_on_port(port): (ts, data) = reading if isinstance(data, dict) and 'name' in data and isinstance(data['name'], str): return data['name'] - logger.warning('Unable to find board name in reading: {}', reading) + logger.warning(f'Unable to find board name in {reading=}') return None except Exception as e: # pragma: no cover - logger.error('Exception while auto-detecting port {}: {}', port, e) + logger.error(f'Exception while auto-detecting {port=}: {e!r}') finally: if serial_reader: serial_reader.disconnect() @@ -141,7 +141,7 @@ def __init__(self, board, serial_data, db): # Using threading.Event rather than just a boolean field so that any thread # can get and set the stop_running property. self._stop_running = threading.Event() - self._logger.info('Created ArduinoIO instance for board {}', self.board) + self._logger.info(f'Created ArduinoIO instance for {self.board=}') @property def stop_running(self): @@ -153,7 +153,7 @@ def stop_running(self, value): self._stop_running.set() else: self._stop_running.clear() - self._logger.info('Updated ArduinoIO.stop_running to {!r}', self.stop_running) + self._logger.info(f'Updated ArduinoIO.stop_running to {self.stop_running!r}') def run(self): """Main loop for recording data and reading commands. @@ -180,13 +180,11 @@ def read_and_record(self): if not reading: # Consider adding an error counter. if not self._report_next_reading: - self._logger.warning( - 'Unable to read from {}. Will report when next successful read.', - self.port) + self._logger.warning(f'Unable to read from {self.port=}. Will report when next successful read.') self._report_next_reading = True return False if self._report_next_reading: - self._logger.info('Succeeded in reading from {}; got:\n{}', self.port, reading) + self._logger.info(f'Succeeded in reading from {self.port=}; got:\n{reading}') self._report_next_reading = False self.handle_reading(reading) return True @@ -205,7 +203,7 @@ def disconnect(self): if self._serial_data.is_connected: self._serial_data.disconnect() except Exception as e: - self._logger.error('Failed to disconnect from {} due to: {}', self.port, e) + self._logger.error(f'Failed to disconnect from {self.port=} due to: {e!r}') def reconnect(self): """Disconnect from and connect to the serial port. @@ -216,13 +214,13 @@ def reconnect(self): try: self.disconnect() except Exception: - self._logger.error('Unable to disconnect from {}', self.port) + self._logger.error(f'Unable to disconnect from {self.port=}') return False try: self.connect() return True except Exception: - self._logger.error('Unable to reconnect to {}', self.port) + self._logger.error(f'Unable to reconnect to {self.port=}') return False def get_reading(self): @@ -232,8 +230,8 @@ def get_reading(self): try: return self._serial_data.get_and_parse_reading(retry_limit=1) except serial.SerialException as e: - self._logger.error('Exception raised while reading from port {}', self.port) - self._logger.error('Exception: {}', "\n".join(traceback.format_exc())) + self._logger.error(f'Exception raised while reading from {self.port=}') + self._logger.error("\n".join(traceback.format_exc())) if self.reconnect(): return None raise e @@ -244,7 +242,7 @@ def handle_reading(self, reading): # instead of a string. Obviously it needs to be serialized eventually. timestamp, data = reading if data.get('name', self.board) != self.board: - msg = 'Board reports name {}, expected {}'.format(data['name'], self.board) + msg = f'Board reports {data["name"]}, expected {self.board}' self._logger.critical(msg) raise ArduinoDataError(msg) reading = dict(name=self.board, timestamp=timestamp, data=data) @@ -268,13 +266,13 @@ def handle_commands(self): topic, msg_obj = self._sub.receive_message(blocking=True, timeout_ms=0.05) if not topic: continue - self._logger.debug('Received a message for topic {}', topic) + self._logger.debug(f'Received a message for {topic=}') if topic.lower() == self._cmd_topic: try: self.handle_command(msg_obj) except Exception as e: - self._logger.error('Exception while handling command: {}', e) - self._logger.error('msg_obj: {}', msg_obj) + self._logger.error(f'Exception while handling command: {e!r}') + self._logger.error(f'{msg_obj=}') def handle_command(self, msg): """Handle one relay command. @@ -284,15 +282,15 @@ def handle_command(self, msg): exists on this device. """ if msg['command'] == 'shutdown': - self._logger.info('Received command to shutdown ArduinoIO for board {}', self.board) + self._logger.info(f'Received command to shutdown ArduinoIO for {self.board=}') self.stop_running = True elif msg['command'] == 'write_line': line = msg['line'].rstrip('\r\n') - self._logger.debug('Sending line to board {}: {}', self.board, line) + self._logger.debug(f'Sending line to {self.board=}: {line}') line = line + '\n' self.write(line) else: - self._logger.error('Ignoring command: {}', msg) + self._logger.error(f'Ignoring command: {msg}') def write(self, text): """Writes text (a string) to the port. diff --git a/src/panoptes/pocs/state/machine.py b/src/panoptes/pocs/state/machine.py index 4a5c38f10..afc981706 100644 --- a/src/panoptes/pocs/state/machine.py +++ b/src/panoptes/pocs/state/machine.py @@ -20,7 +20,7 @@ class PanStateMachine(Machine): def __init__(self, state_machine_table, **kwargs): if isinstance(state_machine_table, str): - self.logger.info("Loading state table: {}".format(state_machine_table)) + self.logger.info(f"Loading state table: {state_machine_table}") state_machine_table = PanStateMachine.load_state_table( state_table_name=state_machine_table) @@ -62,7 +62,6 @@ def __init__(self, state_machine_table, **kwargs): self._state_machine_table = state_machine_table self.next_state = None - self.run_once = kwargs.get('run_once', False) self.logger.debug("State machine created") @@ -83,8 +82,8 @@ def next_state(self, value): # Methods ################################################################################################## - def run(self, exit_when_done=False, run_once=False): - """Runs the state machine loop + def run(self, exit_when_done=False, run_once=False, initial_next_state='ready'): + """Runs the state machine loop. This runs the state machine in a loop. Setting the machine property `is_running` to False will stop the loop. @@ -94,79 +93,80 @@ def run(self, exit_when_done=False, run_once=False): has become False, otherwise will wait (default) run_once (bool, optional): If the machine loop should only run one time, defaults to False to loop continuously. + initial_next_state (str, optional): The first state the machine should move to from + the `sleeping` state, default `ready`. """ - assert self.is_initialized, self.logger.error("POCS not initialized") + if not self.is_initialized: + self.logger.warning("POCS not initialized") + return False run_once = run_once or self.run_once - # Start with `get_ready` - self.next_state = 'ready' - - _loop_iteration = 0 - self.logger.debug(f'Starting run loop with keep_running={self.keep_running} ' - f'and connected={self.connected}') - while self.keep_running and self.connected: - state_changed = False - self.logger.info(f'Run loop: state={self.state} next_state={self.next_state} do_states={self.do_states}') - - # If we are processing the states - self.logger.debug(f'Observatory can_observe: {self.observatory.can_observe}') - if self.do_states and self.observatory.can_observe: - - # BEFORE TRANSITION - - # Wait for safety readying at given horizon level. - required_horizon = self._horizon_lookup.get(self.next_state, 'observe') - delay = self.get_config('wait_delay', default=60 * 3) # Check every 3 minutes - self.logger.debug(f'Checking for horizon={required_horizon} for {self.next_state}') - while not self.is_safe(no_warning=True, horizon=required_horizon): - self.logger.info(f'Waiting for horizon={required_horizon} for state={self.next_state}') - self.wait(delay=delay) - - # ENTER STATE - - self.logger.info(f'Going to next_state={self.next_state}') - try: - # The state's `on_enter` logic will be performed here. - state_changed = self.goto_next_state() - except Exception as e: - self.logger.critical(f"Problem going from {self.state} to {self.next_state}, exiting loop [{e!r}]") - # TODO should we automatically park here? - self.stop_states() - break + self.next_state = initial_next_state + + _transition_iteration = 0 + max_transition_attempts = self.get_config('max_transition_attempts', default=5) + + self.logger.debug(f'Starting run loop') + while self.keep_running: + # BEFORE TRANSITION TO STATE + self.logger.info(f'Run loop: {self.state=} {self.next_state=}') + + # Before moving to next state, check for required horizon level and wait if necessary. + required_horizon = self._horizon_lookup.get(self.next_state, 'observe') + self.logger.debug(f'Checking for {required_horizon=} for {self.next_state=}') + while not self.is_safe(no_warning=True, horizon=required_horizon): + self.logger.info(f'Waiting for {required_horizon=} for {self.next_state=}') + check_delay = self.get_config('wait_delay', + default=60 * 3) # Check every 3 minutes. + self.wait(delay=check_delay) + + # TRANSITION TO STATE + self.logger.info(f'Going to {self.next_state=}') + try: + # The state's `on_enter` logic will be performed here. + state_changed = self.goto_next_state() + except Exception as e: + self.logger.critical(f"Problem going from {self.state=} to {self.next_state=}" + f", exiting loop [{e!r}]") + # TODO should we automatically park here? + self.stop_states() + break - # AFTER TRANSITION - - # If we didn't successfully transition, wait a while then try again - max_iterations = self.get_config('pocs.MAX_TRANSITION_ATTEMPTS', default=5) - if not state_changed: - self.logger.warning(f"Failed to move from {self.state} to {self.next_state}") - if self.is_safe() is False: - self.logger.warning("Conditions have become unsafe; setting next state to 'parking'") - self.next_state = 'parking' - elif _loop_iteration > max_iterations: - self.logger.warning(f"Stuck in current state for {max_iterations} iterations, parking") - self.next_state = 'parking' - else: - _loop_iteration = _loop_iteration + 1 - self.logger.warning(f"Sleeping before trying again ({_loop_iteration}/{max_iterations})") - self.wait(with_status=False) + # AFTER TRANSITION TO STATE (NOW INSIDE STATE) + + # If we didn't successfully transition, wait a while then try again + if not state_changed: + self.logger.warning(f"Failed to move from {self.state=} to {self.next_state=}") + if self.is_safe() is False: + self.logger.warning( + "Conditions have become unsafe; setting next state to 'parking'") + self.next_state = 'parking' + elif _transition_iteration > max_transition_attempts: + self.logger.warning( + f"Stuck in current state for {max_transition_attempts=}, parking") + self.next_state = 'parking' else: - _loop_iteration = 0 - - # Note that `self.state` below has changed from above - # If we are in ready state then we are making one attempt through the loop. - if self.state == 'ready': - self._obs_run_retries -= 1 - - if self.state == 'sleeping' and run_once: + _transition_iteration = _transition_iteration + 1 + self.logger.warning( + f"Sleeping before trying again ({_transition_iteration}/" + f"{max_transition_attempts})") + self.wait(with_status=False, delay=7) # wait 7 seconds (no good reason) + else: + _transition_iteration = 0 + + # Note that `self.state` below has changed from above + + # We started in the sleeping state, so if we are back here we have + # done a full iteration. + if self.state == 'sleeping': + self._obs_run_retries -= 1 + if run_once: self.stop_states() - elif exit_when_done: - break - elif not self.interrupted: - # Sleep for one minute - self.logger.debug(f'Sleeping in run loop - why am I here?') - self.wait(delay=60) + + if exit_when_done: + self.logger.info(f'Leaving run loop {exit_when_done=}') + break def goto_next_state(self): """Make a transition to the next state. @@ -197,7 +197,7 @@ def goto_next_state(self): return state_changed def stop_states(self): - """ Stops the machine loop on the next iteration """ + """ Stops the machine loop on the next iteration by setting do_states=False """ self.logger.success("Stopping POCS states") self.do_states = False @@ -277,7 +277,8 @@ def after_state(self, event_data): event_data(transitions.EventData): Contains information about the event """ - self.logger.debug(f"After {event_data.event.name} transition. In {event_data.state.name} state") + self.logger.debug( + f"After {event_data.event.name} transition. In {event_data.state.name} state") ################################################################################################## # Class Methods @@ -310,7 +311,8 @@ def load_state_table(cls, state_table_name='simple_state_table'): with open(state_table_file, 'r') as f: state_table = from_yaml(f.read()) except Exception as err: - raise error.InvalidConfig(f'Problem loading state table yaml file: {err!r} {state_table_file}') + raise error.InvalidConfig( + f'Problem loading state table yaml file: {err!r} {state_table_file}') return state_table @@ -334,7 +336,6 @@ def _update_status(self, event_data): def _load_state(self, state, state_info=None): self.logger.debug(f"Loading state: {state}") - state_machine = None try: state_module = load_module('panoptes.{}.{}.{}'.format( self._states_location.replace("/", "."), @@ -347,7 +348,7 @@ def _load_state(self, state, state_info=None): on_enter_method = getattr(state_module, 'on_enter') setattr(self, f'on_enter_{state}', on_enter_method) - self.logger.debug(f"Added `on_enter` method from {state_module} {on_enter_method}") + self.logger.trace(f"Added `on_enter` method from {state_module} {on_enter_method}") if state_info is None: state_info = dict() @@ -357,7 +358,7 @@ def _load_state(self, state, state_info=None): self._horizon_lookup[state] = state_info['horizon'] del state_info['horizon'] - self.logger.debug(f"Creating state={state} with {state_info}") + self.logger.debug(f"Creating {state=} with {state_info=}") state_machine = MachineState(name=state, **state_info) # Add default callbacks. @@ -370,13 +371,11 @@ def _load_state(self, state, state_info=None): return state_machine def _load_transition(self, transition): - self.logger.debug(f"Loading transition: {transition}") - # Add `check_safety` as the first transition for all states conditions = listify(transition.get('conditions', [])) conditions.insert(0, 'check_safety') transition['conditions'] = conditions - self.logger.debug(f"Returning transition: {transition}") + self.logger.trace(f"Returning transition: {transition}") return transition diff --git a/src/panoptes/pocs/state/states/default/pointing.py b/src/panoptes/pocs/state/states/default/pointing.py index 945b91ec3..385617945 100644 --- a/src/panoptes/pocs/state/states/default/pointing.py +++ b/src/panoptes/pocs/state/states/default/pointing.py @@ -65,7 +65,6 @@ def waiting_cb(): pointing_image = Image( pointing_path, location=pocs.observatory.earth_location, - config_port=pocs.config_port ) pocs.logger.debug(f"Pointing image: {pointing_image}") diff --git a/src/panoptes/pocs/tests/bisque/test_mount.py b/src/panoptes/pocs/tests/bisque/test_mount.py index f82396b57..669f3340b 100644 --- a/src/panoptes/pocs/tests/bisque/test_mount.py +++ b/src/panoptes/pocs/tests/bisque/test_mount.py @@ -14,8 +14,8 @@ @pytest.fixture -def location(dynamic_config_server, config_port): - config = get_config(port=config_port) +def location(): + config = get_config() loc = config['location'] return EarthLocation(lon=loc['longitude'], lat=loc['latitude'], height=loc['elevation']) diff --git a/src/panoptes/pocs/tests/bisque/test_run.py b/src/panoptes/pocs/tests/bisque/test_run.py index da7cfa822..26a5014cd 100644 --- a/src/panoptes/pocs/tests/bisque/test_run.py +++ b/src/panoptes/pocs/tests/bisque/test_run.py @@ -14,8 +14,8 @@ @pytest.fixture -def location(dynamic_config_server, config_port): - config = get_config(port=config_port) +def location(): + config = get_config() loc = config['location'] return EarthLocation(lon=loc['longitude'], lat=loc['latitude'], height=loc['elevation']) @@ -31,13 +31,13 @@ def target_down(location): @pytest.fixture -def pocs(target, dynamic_config_server, config_port): +def pocs(target): try: del os.environ['POCSTIME'] except KeyError: pass - config = get_config(port=config_port) + config = get_config() pocs = POCS(simulator=['weather', 'night', 'camera'], run_once=True, config=config, db='panoptes_testing') diff --git a/src/panoptes/pocs/tests/test_astrohaven_dome.py b/src/panoptes/pocs/tests/test_astrohaven_dome.py index 710046ff9..3069a3cfe 100644 --- a/src/panoptes/pocs/tests/test_astrohaven_dome.py +++ b/src/panoptes/pocs/tests/test_astrohaven_dome.py @@ -1,36 +1,36 @@ # Test the Astrohaven dome interface using a simulated dome controller. +from contextlib import suppress -import copy import pytest import serial -import pocs.dome -from pocs.dome import astrohaven +from panoptes.pocs import hardware +from panoptes.pocs.dome import astrohaven +from panoptes.pocs.dome import create_dome_simulator + +from panoptes.utils.config.client import set_config @pytest.fixture(scope='function') -def dome(config): +def dome(): # Install our test handlers for the duration. - serial.protocol_handler_packages.append('pocs.dome') + serial.protocol_handler_packages.append('panoptes.pocs.dome') # Modify the config so that the dome uses the right controller and port. - config = copy.deepcopy(config) - dome_config = config.setdefault('dome', {}) - dome_config.update({ + set_config('simulator', hardware.get_all_names(without=['dome'])) + set_config('dome', { 'brand': 'Astrohaven', 'driver': 'astrohaven', 'port': 'astrohaven_simulator://', }) - del config['simulator'] - the_dome = pocs.dome.create_dome_from_config(config) + the_dome = create_dome_simulator() + yield the_dome - try: + with suppress(Exception): the_dome.disconnect() - except Exception: - pass # Remove our test handlers. - serial.protocol_handler_packages.remove('pocs.dome') + serial.protocol_handler_packages.remove('panoptes.pocs.dome') def test_create(dome): @@ -64,11 +64,17 @@ def test_open_and_close_slit(dome): dome.connect() assert dome.open() is True - assert dome.status == 'Both sides open' + assert dome.status['open'] == 'open_both' assert dome.is_open is True + # Try to open shutter + assert dome.open() is True + assert dome.close() is True - assert dome.status == 'Both sides closed' + assert dome.status['open'] == 'closed_both' assert dome.is_closed is True + # Try to close again + assert dome.close() is True + dome.disconnect() diff --git a/src/panoptes/pocs/tests/test_base.py b/src/panoptes/pocs/tests/test_base.py index fbcddb2ef..fb9de8b77 100644 --- a/src/panoptes/pocs/tests/test_base.py +++ b/src/panoptes/pocs/tests/test_base.py @@ -1,24 +1,14 @@ import pytest -from pocs import PanBase +from panoptes.pocs.base import PanBase +from panoptes.utils.database import PanDB -def test_check_config1(config): - del config['mount'] - base = PanBase() - with pytest.raises(SystemExit): - base._check_config(config) +def test_with_logger(): + PanBase() -def test_check_config2(config): - del config['directories'] - base = PanBase() - with pytest.raises(SystemExit): - base._check_config(config) - -def test_check_config3(config): - del config['state_machine'] - base = PanBase() - with pytest.raises(SystemExit): - base._check_config(config) +def test_with_db(): + base = PanBase(db=PanDB(db_type='memory', db_name='tester')) + assert isinstance(base, PanBase) diff --git a/src/panoptes/pocs/tests/test_base_scheduler.py b/src/panoptes/pocs/tests/test_base_scheduler.py index b9ad4c898..d3e620740 100644 --- a/src/panoptes/pocs/tests/test_base_scheduler.py +++ b/src/panoptes/pocs/tests/test_base_scheduler.py @@ -14,18 +14,18 @@ @pytest.fixture -def constraints(dynamic_config_server, config_port): - return [MoonAvoidance(config_port=config_port), Duration(30 * u.deg, config_port=config_port)] +def constraints(): + return [MoonAvoidance(), Duration(30 * u.deg)] @pytest.fixture -def simple_fields_file(dynamic_config_server, config_port): - return get_config('directories.targets', port=config_port) + '/simulator.yaml' +def simple_fields_file(): + return get_config('directories.targets') + '/simulator.yaml' @pytest.fixture -def observer(dynamic_config_server, config_port): - loc = get_config('location', port=config_port) +def observer(): + loc = get_config('location') location = EarthLocation(lon=loc['longitude'], lat=loc['latitude'], height=loc['elevation']) return Observer(location=location, name="Test Observer", timezone=loc['timezone']) @@ -75,72 +75,58 @@ def field_list(): @pytest.fixture -def scheduler(dynamic_config_server, config_port, field_list, observer, constraints): +def scheduler(field_list, observer, constraints): return Scheduler(observer, fields_list=field_list, - constraints=constraints, - config_port=config_port) + constraints=constraints) -def test_scheduler_load_no_params(dynamic_config_server, config_port): +def test_scheduler_load_no_params(): with pytest.raises(TypeError): - Scheduler(config_port=config_port) + Scheduler() -def test_no_observer(dynamic_config_server, config_port, simple_fields_file): +def test_no_observer(simple_fields_file): with pytest.raises(TypeError): - Scheduler(fields_file=simple_fields_file, config_port=config_port) + Scheduler(fields_file=simple_fields_file) -def test_bad_observer(dynamic_config_server, config_port, simple_fields_file, constraints): +def test_bad_observer(simple_fields_file, constraints): with pytest.raises(TypeError): Scheduler(fields_file=simple_fields_file, - constraints=constraints, - config_port=config_port) + constraints=constraints) -def test_loading_target_file_check_file(dynamic_config_server, - config_port, - observer, +def test_loading_target_file_check_file(observer, simple_fields_file, constraints): - set_config('scheduler.check_file', False, port=config_port) + set_config('scheduler.check_file', False) scheduler = Scheduler(observer, fields_file=simple_fields_file, - constraints=constraints, - config_port=config_port - ) + constraints=constraints) # Check the hidden property as the public one # will populate if not found. assert len(scheduler._observations) -def test_loading_target_file_no_check_file(dynamic_config_server, - config_port, - observer, - simple_fields_file, - constraints): - # If check_file is True then we will check the file - # before each call to `get_observation`, but *not* - # when the Scheduler is initialized. - set_config('scheduler.check_file', True, port=config_port) +def test_loading_target_file_check_file(observer, + simple_fields_file, + constraints): + set_config('scheduler.check_file', True) scheduler = Scheduler(observer, fields_file=simple_fields_file, constraints=constraints, - config_port=config_port ) # Check the hidden property as the public one # will populate if not found. - assert len(scheduler._observations) == 0 + assert len(scheduler._observations) > 0 -def test_loading_target_file_via_property(dynamic_config_server, - config_port, - simple_fields_file, +def test_loading_target_file_via_property(simple_fields_file, observer, constraints): scheduler = Scheduler(observer, fields_file=simple_fields_file, - constraints=constraints, config_port=config_port) + constraints=constraints) scheduler._observations = dict() assert scheduler.observations is not None @@ -149,9 +135,9 @@ def test_with_location(scheduler): assert isinstance(scheduler, Scheduler) -def test_loading_bad_target_file(dynamic_config_server, config_port, observer): +def test_loading_bad_target_file(observer): with pytest.raises(FileNotFoundError): - Scheduler(observer, fields_file='/var/path/foo.bar', config_port=config_port) + Scheduler(observer, fields_file='/var/path/foo.bar') def test_new_fields_file(scheduler, simple_fields_file): diff --git a/src/panoptes/pocs/tests/test_camera.py b/src/panoptes/pocs/tests/test_camera.py index 779ae8af6..f488d4800 100644 --- a/src/panoptes/pocs/tests/test_camera.py +++ b/src/panoptes/pocs/tests/test_camera.py @@ -62,13 +62,13 @@ 'simulator', 'simulator_focuser', 'simulator_filterwheel', 'simulator_sdk', 'sbig', 'fli', 'zwo' ]) -def camera(request, dynamic_config_server, config_port): +def camera(request): CamClass = request.param[0] cam_params = request.param[1] if isinstance(cam_params, dict): # Simulator - camera = CamClass(config_port=config_port, **cam_params) + camera = CamClass(**cam_params) else: # Lookup real hardware device name in real life config server. for cam_config in get_config('cameras.devices'): @@ -80,7 +80,7 @@ def camera(request, dynamic_config_server, config_port): assert camera.is_ready yield camera - # simulator_sdk needs this explictly removed for some reason. + # simulator_sdk needs this explicitly removed for some reason. with suppress(AttributeError): type(camera)._assigned_cameras.discard(camera.uid) @@ -112,90 +112,89 @@ def test_create_camera_simulator(): create_camera_simulator(num_cameras=0) -def test_create_cameras_from_config_no_autodetect(dynamic_config_server, config_port): - set_config('cameras.auto_detect', False, port=config_port) +def test_create_cameras_from_config_no_autodetect(): + set_config('cameras.auto_detect', False) set_config('cameras.devices', [ dict(model='canon_gphoto2', port='/dev/fake01'), dict(model='canon_gphoto2', port='/dev/fake02'), - ], port=config_port) + ]) with pytest.raises(error.CameraNotFound): - create_cameras_from_config(config_port=config_port) + create_cameras_from_config() -def test_create_cameras_from_config_autodetect(dynamic_config_server, config_port): - set_config('cameras.auto_detect', True, port=config_port) +def test_create_cameras_from_config_autodetect(): + set_config('cameras.auto_detect', True) with pytest.raises(error.CameraNotFound): - create_cameras_from_config(config_port=config_port) + create_cameras_from_config() # Hardware independent tests, mostly use simulator: -def test_sim_create_focuser(dynamic_config_server, config_port): - sim_camera = SimCamera(focuser={'model': 'simulator', 'focus_port': '/dev/ttyFAKE'}, - config_port=config_port) +def test_sim_create_focuser(): + sim_camera = SimCamera(focuser={'model': 'simulator', 'focus_port': '/dev/ttyFAKE'}) assert isinstance(sim_camera.focuser, Focuser) -def test_sim_passed_focuser(dynamic_config_server, config_port): - sim_focuser = Focuser(port='/dev/ttyFAKE', config_port=config_port) - sim_camera = SimCamera(focuser=sim_focuser, config_port=config_port) +def test_sim_passed_focuser(): + sim_focuser = Focuser(port='/dev/ttyFAKE') + sim_camera = SimCamera(focuser=sim_focuser) assert sim_camera.focuser is sim_focuser -def test_sim_bad_focuser(dynamic_config_server, config_port): +def test_sim_bad_focuser(): with pytest.raises((NotFound)): - SimCamera(focuser={'model': 'NOTAFOCUSER'}, config_port=config_port) + SimCamera(focuser={'model': 'NOTAFOCUSER'}) -def test_sim_worse_focuser(dynamic_config_server, config_port): - sim_camera = SimCamera(focuser='NOTAFOCUSER', config_port=config_port) +def test_sim_worse_focuser(): + sim_camera = SimCamera(focuser='NOTAFOCUSER') # Will log an error but raise no exceptions assert sim_camera.focuser is None -def test_sim_string(dynamic_config_server, config_port): - sim_camera = SimCamera(config_port=config_port) +def test_sim_string(): + sim_camera = SimCamera() assert str(sim_camera) == 'Simulated Camera ({}) on None'.format(sim_camera.uid) - sim_camera = SimCamera(name='Sim', port='/dev/ttyFAKE', config_port=config_port) + sim_camera = SimCamera(name='Sim', port='/dev/ttyFAKE') assert str(sim_camera) == 'Sim ({}) on /dev/ttyFAKE'.format(sim_camera.uid) -def test_sim_file_extension(dynamic_config_server, config_port): - sim_camera = SimCamera(config_port=config_port) +def test_sim_file_extension(): + sim_camera = SimCamera() assert sim_camera.file_extension == 'fits' - sim_camera = SimCamera(file_extension='FIT', config_port=config_port) + sim_camera = SimCamera(file_extension='FIT') assert sim_camera.file_extension == 'FIT' -def test_sim_readout_time(dynamic_config_server, config_port): - sim_camera = SimCamera(config_port=config_port) +def test_sim_readout_time(): + sim_camera = SimCamera() assert sim_camera.readout_time == 1.0 - sim_camera = SimCamera(readout_time=2.0, config_port=config_port) + sim_camera = SimCamera(readout_time=2.0) assert sim_camera.readout_time == 2.0 -def test_sdk_no_serial_number(dynamic_config_server, config_port): +def test_sdk_no_serial_number(): with pytest.raises(ValueError): - SimSDKCamera(config_port=config_port) + SimSDKCamera() -def test_sdk_camera_not_found(dynamic_config_server, config_port): +def test_sdk_camera_not_found(): with pytest.raises(error.PanError): - SimSDKCamera(serial_number='SSC404', config_port=config_port) + SimSDKCamera(serial_number='SSC404') -def test_sdk_already_in_use(dynamic_config_server, config_port): - sim_camera = SimSDKCamera(serial_number='SSC999', config_port=config_port) +def test_sdk_already_in_use(): + sim_camera = SimSDKCamera(serial_number='SSC999') assert sim_camera with pytest.raises(error.PanError): - SimSDKCamera(serial_number='SSC999', config_port=config_port) + SimSDKCamera(serial_number='SSC999') # Hardware independent tests for SBIG camera -def test_sbig_driver_bad_path(dynamic_config_server, config_port): +def test_sbig_driver_bad_path(): """ Manually specify an incorrect path for the SBIG shared library. The CDLL loader should raise OSError when it fails. Can't test a successful @@ -203,11 +202,11 @@ def test_sbig_driver_bad_path(dynamic_config_server, config_port): CDLL unload problem. """ with pytest.raises(OSError): - SBIGDriver(library_path='no_library_here', config_port=config_port) + SBIGDriver(library_path='no_library_here') @pytest.mark.filterwarnings('ignore:Could not connect to SBIG Camera') -def test_sbig_bad_serial(dynamic_config_server, config_port): +def test_sbig_bad_serial(): """ Attempt to create an SBIG camera instance for a specific non-existent camera. No actual cameras are required to run this test but the SBIG @@ -216,7 +215,7 @@ def test_sbig_bad_serial(dynamic_config_server, config_port): if find_library('sbigudrv') is None: pytest.skip("Test requires SBIG camera driver to be installed") with pytest.raises(error.PanError): - SBIGCamera(serial_number='NOTAREALSERIALNUMBER', config_port=config_port) + SBIGCamera(serial_number='NOTAREALSERIALNUMBER') # *Potentially* hardware dependant tests: @@ -463,12 +462,12 @@ def test_exposure_timeout(camera, tmpdir, caplog): assert exposure_event.is_set() -def test_observation(dynamic_config_server, config_port, camera, images_dir): +def test_observation(camera, images_dir): """ Tests functionality of take_observation() """ - field = Field('Test Observation', '20h00m43.7135s +22d42m39.0645s', config_port=config_port) - observation = Observation(field, exptime=1.5 * u.second, config_port=config_port) + field = Field('Test Observation', '20h00m43.7135s +22d42m39.0645s') + observation = Observation(field, exptime=1.5 * u.second) observation.seq_time = '19991231T235959' camera.take_observation(observation, headers={}) time.sleep(7) @@ -479,12 +478,12 @@ def test_observation(dynamic_config_server, config_port, camera, images_dir): os.remove(fn) -def test_observation_nofilter(dynamic_config_server, config_port, camera, images_dir): +def test_observation_nofilter(camera, images_dir): """ Tests functionality of take_observation() """ - field = Field('Test Observation', '20h00m43.7135s +22d42m39.0645s', config_port=config_port) - observation = Observation(field, exptime=1.5 * u.second, filter_name=None, config_port=config_port) + field = Field('Test Observation', '20h00m43.7135s +22d42m39.0645s') + observation = Observation(field, exptime=1.5 * u.second, filter_name=None) observation.seq_time = '19991231T235959' camera.take_observation(observation, headers={}) time.sleep(7) diff --git a/src/panoptes/pocs/tests/test_constraints.py b/src/panoptes/pocs/tests/test_constraints.py index 80a8df555..bebc07551 100644 --- a/src/panoptes/pocs/tests/test_constraints.py +++ b/src/panoptes/pocs/tests/test_constraints.py @@ -23,16 +23,16 @@ @pytest.fixture(scope='function') -def observer(dynamic_config_server, config_port): - loc = get_config('location', port=config_port) +def observer(): + loc = get_config('location') location = EarthLocation(lon=loc['longitude'], lat=loc['latitude'], height=loc['elevation']) return Observer(location=location, name="Test Observer", timezone=loc['timezone']) @pytest.fixture(scope='function') -def horizon_line(dynamic_config_server, config_port): - obstruction_list = get_config('location.obstructions', default=list(), port=config_port) - default_horizon = get_config('location.horizon', port=config_port) +def horizon_line(): + obstruction_list = get_config('location.obstructions', default=list()) + default_horizon = get_config('location.horizon') horizon_line = horizon_utils.Horizon( obstructions=obstruction_list, diff --git a/src/panoptes/pocs/tests/test_dispatch_scheduler.py b/src/panoptes/pocs/tests/test_dispatch_scheduler.py index c9ec6b51c..5013fd44a 100644 --- a/src/panoptes/pocs/tests/test_dispatch_scheduler.py +++ b/src/panoptes/pocs/tests/test_dispatch_scheduler.py @@ -15,24 +15,24 @@ @pytest.fixture -def constraints(dynamic_config_server, config_port): - return [MoonAvoidance(config_port=config_port), Duration(30 * u.deg, config_port=config_port)] +def constraints(): + return [MoonAvoidance(), Duration(30 * u.deg)] @pytest.fixture -def observer(dynamic_config_server, config_port): - loc = get_config('location', port=config_port) +def observer(): + loc = get_config('location') location = EarthLocation(lon=loc['longitude'], lat=loc['latitude'], height=loc['elevation']) return Observer(location=location, name="Test Observer", timezone=loc['timezone']) @pytest.fixture() -def field_file(dynamic_config_server, config_port): - scheduler_config = get_config('scheduler', default={}, port=config_port) +def field_file(): + scheduler_config = get_config('scheduler', default={}) # Read the targets from the file fields_file = scheduler_config.get('fields_file', 'simple.yaml') - fields_path = os.path.join(get_config('directories.targets', port=config_port), fields_file) + fields_path = os.path.join(get_config('directories.targets'), fields_file) return fields_path @@ -82,19 +82,17 @@ def field_list(): @pytest.fixture -def scheduler(dynamic_config_server, config_port, field_list, observer, constraints): +def scheduler(field_list, observer, constraints): return Scheduler(observer, fields_list=field_list, - constraints=constraints, - config_port=config_port) + constraints=constraints) @pytest.fixture -def scheduler_from_file(dynamic_config_server, config_port, field_file, observer, constraints): +def scheduler_from_file(field_file, observer, constraints): return Scheduler(observer, fields_file=field_file, - constraints=constraints, - config_port=config_port) + constraints=constraints) @pytest.fixture @@ -111,9 +109,7 @@ def test_get_observation(scheduler): assert isinstance(best[1], float) -def test_get_observation_reread(dynamic_config_server, - config_port, - field_list, +def test_get_observation_reread(field_list, observer, temp_file, constraints): @@ -125,8 +121,7 @@ def test_get_observation_reread(dynamic_config_server, scheduler = Scheduler(observer, fields_file=temp_file, - constraints=constraints, - config_port=config_port) + constraints=constraints) # Get observation as above best = scheduler.get_observation(time=time) diff --git a/src/panoptes/pocs/tests/test_dome_simulator.py b/src/panoptes/pocs/tests/test_dome_simulator.py index 08a58b948..fe1af389f 100644 --- a/src/panoptes/pocs/tests/test_dome_simulator.py +++ b/src/panoptes/pocs/tests/test_dome_simulator.py @@ -7,13 +7,13 @@ @pytest.fixture(scope="function") -def dome(dynamic_config_server, config_port): +def dome(): set_config('dome', { 'brand': 'Simulacrum', 'driver': 'simulator', - }, port=config_port) + }) - the_dome = create_dome_simulator(config_port=config_port) + the_dome = create_dome_simulator() yield the_dome diff --git a/src/panoptes/pocs/tests/test_filterwheel.py b/src/panoptes/pocs/tests/test_filterwheel.py index dcdc55808..0e5197b6c 100644 --- a/src/panoptes/pocs/tests/test_filterwheel.py +++ b/src/panoptes/pocs/tests/test_filterwheel.py @@ -10,11 +10,10 @@ @pytest.fixture(scope='function') -def filterwheel(dynamic_config_server, config_port): +def filterwheel(): sim_filterwheel = SimFilterWheel(filter_names=['one', 'deux', 'drei', 'quattro'], move_time=0.1 * u.second, - timeout=0.5 * u.second, - config_port=config_port) + timeout=0.5 * u.second) return sim_filterwheel @@ -26,31 +25,30 @@ def test_init(filterwheel): assert filterwheel.is_connected -def test_camera_init(dynamic_config_server, config_port): +def test_camera_init(): sim_camera = SimCamera(filterwheel={'model': 'simulator', - 'filter_names': ['one', 'deux', 'drei', 'quattro']}, - config_port=config_port) + 'filter_names': ['one', 'deux', 'drei', 'quattro']}) assert isinstance(sim_camera.filterwheel, SimFilterWheel) assert sim_camera.filterwheel.is_connected assert sim_camera.filterwheel.uid assert sim_camera.filterwheel.camera is sim_camera -def test_camera_no_filterwheel(dynamic_config_server, config_port): - sim_camera = SimCamera(config_port=config_port) +def test_camera_no_filterwheel(): + sim_camera = SimCamera() assert sim_camera.filterwheel is None -def test_camera_association_on_init(dynamic_config_server, config_port): - sim_camera = SimCamera(config_port=config_port) +def test_camera_association_on_init(): + sim_camera = SimCamera() sim_filterwheel = SimFilterWheel(filter_names=['one', 'deux', 'drei', 'quattro'], - camera=sim_camera, config_port=config_port) + camera=sim_camera) assert sim_filterwheel.camera is sim_camera -def test_with_no_name(dynamic_config_server, config_port): +def test_with_no_name(): with pytest.raises(ValueError): - SimFilterWheel(config_port=config_port) + SimFilterWheel() # Basic property getting and (not) setting @@ -137,11 +135,10 @@ def test_move_bad_name(filterwheel): filterwheel.move_to('cinco') -def test_move_timeout(dynamic_config_server, config_port, caplog): +def test_move_timeout(caplog): slow_filterwheel = SimFilterWheel(filter_names=['one', 'deux', 'drei', 'quattro'], move_time=0.5, - timeout=0.2, - config_port=config_port) + timeout=0.2) with pytest.raises(error.Timeout): # Move should take 0.3 seconds, more than timeout. slow_filterwheel.position = 4 @@ -152,12 +149,11 @@ def test_move_timeout(dynamic_config_server, config_port, caplog): @pytest.mark.parametrize("name, unidirectional, expected", [("unidirectional", True, 0.3), ("bidirectional", False, 0.1)]) -def test_move_times(dynamic_config_server, config_port, name, unidirectional, expected): +def test_move_times(name, unidirectional, expected): sim_filterwheel = SimFilterWheel(filter_names=['one', 'deux', 'drei', 'quattro'], move_time=0.1 * u.second, unidirectional=unidirectional, - timeout=0.5 * u.second, - config_port=config_port) + timeout=0.5 * u.second) sim_filterwheel.position = 1 assert timeit("sim_filterwheel.position = 2", number=1, globals=locals()) == \ pytest.approx(0.1, rel=1e-1) @@ -167,10 +163,9 @@ def test_move_times(dynamic_config_server, config_port, name, unidirectional, ex pytest.approx(expected, rel=2e-1) -def test_move_exposing(dynamic_config_server, config_port, tmpdir): +def test_move_exposing(tmpdir): sim_camera = SimCamera(filterwheel={'model': 'simulator', - 'filter_names': ['one', 'deux', 'drei', 'quattro']}, - config_port=config_port) + 'filter_names': ['one', 'deux', 'drei', 'quattro']}) fits_path = str(tmpdir.join('test_exposure.fits')) exp_event = sim_camera.take_exposure(filename=fits_path, seconds=0.1) with pytest.raises(error.PanError): diff --git a/src/panoptes/pocs/tests/test_focuser.py b/src/panoptes/pocs/tests/test_focuser.py index a9fb15f96..93c84125c 100644 --- a/src/panoptes/pocs/tests/test_focuser.py +++ b/src/panoptes/pocs/tests/test_focuser.py @@ -3,7 +3,7 @@ import pytest from threading import Thread -from panoptes.utils.config import load_config +from panoptes.utils.config.helpers import load_config from panoptes.pocs.focuser.simulator import Focuser as SimFocuser from panoptes.pocs.focuser.birger import Focuser as BirgerFocuser @@ -13,11 +13,12 @@ params = [SimFocuser, BirgerFocuser, FocusLynxFocuser] ids = ['simulator', 'birger', 'focuslynx'] + # Ugly hack to access id inside fixture @pytest.fixture(scope='function', params=zip(params, ids), ids=ids) -def focuser(request, dynamic_config_server, config_port): +def focuser(request): if request.param[0] == SimFocuser: # Simulated focuser, just create one and return it return request.param[0]() @@ -44,7 +45,7 @@ def focuser(request, dynamic_config_server, config_port): request.param[1])) # Create and return a Focuser based on the first config - return request.param[0](**focuser_configs[0], config_port=config_port) + return request.param[0](**focuser_configs[0]) @pytest.fixture(scope='function') diff --git a/src/panoptes/pocs/tests/test_images.py b/src/panoptes/pocs/tests/test_images.py index b1ccfa567..846e67c30 100644 --- a/src/panoptes/pocs/tests/test_images.py +++ b/src/panoptes/pocs/tests/test_images.py @@ -21,47 +21,45 @@ def copy_file_to_dir(to_dir, file): return result -def test_fits_exists(dynamic_config_server, config_port, unsolved_fits_file): +def test_fits_exists(unsolved_fits_file): with pytest.raises(AssertionError): - Image(unsolved_fits_file.replace('.fits', '.fit'), config_port=config_port) + Image(unsolved_fits_file.replace('.fits', '.fit')) -def test_fits_extension(dynamic_config_server, config_port): +def test_fits_extension(): with pytest.raises(AssertionError): - Image(os.path.join(os.environ['POCS'], 'pocs', 'images.py'), config_port=config_port) + Image(os.path.join(os.environ['POCS'], 'pocs', 'images.py')) -def test_fits_noheader(dynamic_config_server, config_port, noheader_fits_file): +def test_fits_noheader(noheader_fits_file): with pytest.raises(KeyError): - Image(noheader_fits_file, config_port=config_port) + Image(noheader_fits_file) -def test_solve_timeout(dynamic_config_server, config_port, tiny_fits_file): +def test_solve_timeout(tiny_fits_file): with tempfile.TemporaryDirectory() as tmpdir: tiny_fits_file = copy_file_to_dir(tmpdir, tiny_fits_file) - im0 = Image(tiny_fits_file, config_port=config_port) + im0 = Image(tiny_fits_file) assert str(im0) with pytest.raises(Timeout): im0.solve_field(verbose=True, replace=False, radius=4, timeout=1) -def test_fail_solve(dynamic_config_server, config_port, tiny_fits_file): +def test_fail_solve(tiny_fits_file): with tempfile.TemporaryDirectory() as tmpdir: tiny_fits_file = copy_file_to_dir(tmpdir, tiny_fits_file) - im0 = Image(tiny_fits_file, config_port=config_port) + im0 = Image(tiny_fits_file) assert str(im0) with pytest.raises(SolveError): im0.solve_field(verbose=True, replace=False, radius=4) -def test_solve_field_unsolved(dynamic_config_server, - config_port, - unsolved_fits_file, +def test_solve_field_unsolved(unsolved_fits_file, solved_fits_file): # We place the input images into a temp directory so that output images # are also in the temp directory. with tempfile.TemporaryDirectory() as tmpdir: - im0 = Image(copy_file_to_dir(tmpdir, unsolved_fits_file), config_port=config_port) + im0 = Image(copy_file_to_dir(tmpdir, unsolved_fits_file)) assert isinstance(im0, Image) assert im0.wcs is None @@ -77,15 +75,15 @@ def test_solve_field_unsolved(dynamic_config_server, assert im0.ha is not None # Compare it to another file of known offset. - im1 = Image(copy_file_to_dir(tmpdir, solved_fits_file), config_port=config_port) + im1 = Image(copy_file_to_dir(tmpdir, solved_fits_file)) offset_info = im0.compute_offset(im1) # print('offset_info:', offset_info) expected_offset = [10.1 * u.arcsec, 5.29 * u.arcsec, 8.77 * u.arcsec] assert u.allclose(offset_info, expected_offset, rtol=0.005) -def test_solve_field_solved(dynamic_config_server, config_port, solved_fits_file): - im0 = Image(solved_fits_file, config_port=config_port) +def test_solve_field_solved(solved_fits_file): + im0 = Image(solved_fits_file) assert isinstance(im0, Image) assert im0.wcs is not None @@ -100,24 +98,22 @@ def test_solve_field_solved(dynamic_config_server, config_port, solved_fits_file assert isinstance(im0.pointing, SkyCoord) -def test_pointing_error_no_wcs(dynamic_config_server, config_port, unsolved_fits_file): - im0 = Image(unsolved_fits_file, config_port=config_port) +def test_pointing_error_no_wcs(unsolved_fits_file): + im0 = Image(unsolved_fits_file) with pytest.raises(AssertionError): im0.pointing_error -def test_pointing_error_passed_wcs(dynamic_config_server, - config_port, - unsolved_fits_file, +def test_pointing_error_passed_wcs(unsolved_fits_file, solved_fits_file): - im0 = Image(unsolved_fits_file, wcs_file=solved_fits_file, config_port=config_port) + im0 = Image(unsolved_fits_file, wcs_file=solved_fits_file) assert isinstance(im0.pointing_error, OffsetError) -def test_pointing_error(dynamic_config_server, config_port, solved_fits_file): - im0 = Image(solved_fits_file, config_port=config_port) +def test_pointing_error(solved_fits_file): + im0 = Image(solved_fits_file) im0.solve_field(verbose=True, replace=False, radius=4) diff --git a/src/panoptes/pocs/tests/test_ioptron.py b/src/panoptes/pocs/tests/test_ioptron.py index 470d3322a..1f9a7829f 100644 --- a/src/panoptes/pocs/tests/test_ioptron.py +++ b/src/panoptes/pocs/tests/test_ioptron.py @@ -13,13 +13,13 @@ @pytest.fixture -def location(dynamic_config_server, config_port): - loc = get_config('location', port=config_port) +def location(): + loc = get_config('location') return EarthLocation(lon=loc['longitude'], lat=loc['latitude'], height=loc['elevation']) @pytest.fixture(scope="function") -def mount(dynamic_config_server, config_port, location): +def mount(location): with suppress(KeyError): del os.environ['POCSTIME'] @@ -27,9 +27,9 @@ def mount(dynamic_config_server, config_port, location): { 'brand': 'bisque', 'template_dir': 'resources/bisque', - }, port=config_port) + }) - return Mount(location=location, config_port=config_port) + return Mount(location=location) @pytest.mark.with_mount @@ -42,13 +42,10 @@ def test_loading_without_config(): @pytest.mark.with_mount class TestMount(object): - """ Test the mount """ @pytest.fixture(autouse=True) def setup(self): - - # Don't use config_port because we use real live config_server location = create_location_from_config() # Can't supply full location, need earth_location @@ -91,7 +88,6 @@ def test_unpark_park(self): def test_get_tracking_correction(mount): - offsets = [ # HA, ΔRA, ΔDec, Magnitude (2, -13.0881456, 1.4009, 12.154), @@ -148,7 +144,6 @@ def test_get_tracking_correction(mount): def test_get_tracking_correction_custom(mount): - min_tracking = 105 max_tracking = 950 diff --git a/src/panoptes/pocs/tests/test_mount.py b/src/panoptes/pocs/tests/test_mount.py index 8dcb43abe..42923dbaf 100644 --- a/src/panoptes/pocs/tests/test_mount.py +++ b/src/panoptes/pocs/tests/test_mount.py @@ -10,6 +10,21 @@ from panoptes.utils import error from panoptes.utils.config.client import get_config from panoptes.utils.config.client import set_config +from panoptes.utils.serializers import to_json + +import requests + +config_host = 'localhost' +config_port = 6563 +url = f'http://{config_host}:{config_port}/reset-config' + + +def reset_conf(): + response = requests.post(url, + data=to_json({'reset': True}), + headers={'Content-Type': 'application/json'} + ) + assert response.ok def test_create_mount_simulator(): @@ -18,72 +33,81 @@ def test_create_mount_simulator(): assert isinstance(mount, AbstractMount) is True -def test_create_mount_simulator_with_config(dynamic_config_server, config_port): +def test_create_mount_simulator_with_config(): # Remove mount from list of simulators. set_config('simulator', hardware.get_all_names(without=['mount'])) # But setting the driver to `simulator` should return simulator. - set_config('mount.driver', 'simulator', port=config_port) + set_config('mount.driver', 'simulator') - mount = create_mount_from_config(config_port=config_port) + mount = create_mount_from_config() assert isinstance(mount, AbstractMount) is True + reset_conf() -def test_create_mount_without_mount_info(dynamic_config_server, config_port): +def test_create_mount_without_mount_info(): # Set the mount config to none and then don't pass anything for error. - set_config('mount', None, port=config_port) + set_config('mount', None) set_config('simulator', hardware.get_all_names(without=['mount'])) with pytest.raises(error.MountNotFound): - create_mount_from_config(config_port=config_port, mount_info=None) + create_mount_from_config(mount_info=None) + reset_conf() -def test_create_mount_with_mount_info(dynamic_config_server, config_port): + +def test_create_mount_with_mount_info(): # Pass the mount info directly with nothing in config. - mount_info = get_config('mount', port=config_port) + mount_info = get_config('mount', default=dict()) mount_info['driver'] = 'simulator' # Remove info from config. - set_config('mount', None, port=config_port) + set_config('mount', None) set_config('simulator', hardware.get_all_names(without=['mount'])) - assert isinstance(create_mount_from_config(config_port=config_port, - mount_info=mount_info), AbstractMount) is True + assert isinstance(create_mount_from_config(mount_info=mount_info), AbstractMount) is True + + reset_conf() -def test_create_mount_with_earth_location(dynamic_config_server, config_port): +def test_create_mount_with_earth_location(): # Get location to pass manually. loc = create_location_from_config() # Set config to not have a location. - set_config('location', None, port=config_port) - assert isinstance(create_mount_from_config(config_port=config_port, - earth_location=loc), AbstractMount) is True + set_config('location', None) + set_config('simulator', hardware.get_all_names()) + assert isinstance(create_mount_from_config(earth_location=loc['earth_location']), AbstractMount) is True + + reset_conf() -def test_create_mount_without_earth_location(dynamic_config_server, config_port): - set_config('location', None, port=config_port) +def test_create_mount_without_earth_location(): + set_config('location', None) with pytest.raises(error.PanError): - create_mount_from_config(config_port=config_port, earth_location=None) + create_mount_from_config(earth_location=None) + reset_conf() -def test_bad_mount_port(dynamic_config_server, config_port): +def test_bad_mount_port(): # Remove the mount from the list of simulators so it thinks we have a real one. - simulators = get_config('simulator', port=config_port) - with suppress(KeyError): + simulators = get_config('simulator') + with suppress(KeyError, AttributeError): simulators.remove('mount') - set_config('simulator', simulators, port=config_port) + set_config('simulator', simulators) # Set a bad port, which should cause a fail before actual mount creation. set_config('mount.serial.port', 'foobar') with pytest.raises(error.MountNotFound): - create_mount_from_config(config_port=config_port) + create_mount_from_config() + reset_conf() -def test_bad_mount_driver(dynamic_config_server, config_port): +def test_bad_mount_driver(): # Remove the mount from the list of simulators so it thinks we have a real one. - simulators = get_config('simulator', port=config_port) - with suppress(KeyError): + simulators = get_config('simulator') + with suppress(KeyError, AttributeError): simulators.remove('mount') - set_config('simulator', simulators, port=config_port) + set_config('simulator', simulators) # Set a bad port, which should cause a fail before actual mount creation. set_config('mount.serial.driver', 'foobar') with pytest.raises(error.MountNotFound): - create_mount_from_config(config_port=config_port) + create_mount_from_config() + reset_conf() diff --git a/src/panoptes/pocs/tests/test_mount_simulator.py b/src/panoptes/pocs/tests/test_mount_simulator.py index 32b49d087..0012ab140 100644 --- a/src/panoptes/pocs/tests/test_mount_simulator.py +++ b/src/panoptes/pocs/tests/test_mount_simulator.py @@ -12,8 +12,8 @@ @pytest.fixture -def location(dynamic_config_server, config_port): - loc = get_config('location', port=config_port) +def location(): + loc = get_config('location') return EarthLocation(lon=loc['longitude'], lat=loc['latitude'], height=loc['elevation']) @@ -22,14 +22,14 @@ def target(location): return altaz_to_radec(obstime='2016-08-13 21:03:01', location=location, alt=45, az=90) -def test_no_location(dynamic_config_server, config_port): +def test_no_location(): with pytest.raises(TypeError): - Mount(config_port=config_port) + Mount() @pytest.fixture(scope='function') -def mount(dynamic_config_server, config_port, location): - return Mount(location=location, config_port=config_port) +def mount(location): + return Mount(location=location) def test_connect(mount): @@ -83,22 +83,22 @@ def test_status(mount): assert 'mount_target_ra' in status2 -def test_update_location_no_init(dynamic_config_server, config_port, mount): - loc = get_config('location', port=config_port) +def test_update_location_no_init(mount): + loc = get_config('location') location2 = EarthLocation( lon=loc['longitude'], lat=loc['latitude'], height=loc['elevation'] - - 1000 * - u.meter) + 1000 * + u.meter) with pytest.raises(AssertionError): mount.location = location2 -def test_update_location(dynamic_config_server, config_port, mount): - loc = get_config('location', port=config_port) +def test_update_location(mount): + loc = get_config('location') mount.initialize() @@ -107,8 +107,8 @@ def test_update_location(dynamic_config_server, config_port, mount): lon=loc['longitude'], lat=loc['latitude'], height=loc['elevation'] - - 1000 * - u.meter) + 1000 * + u.meter) mount.location = location2 assert location1 != location2 diff --git a/src/panoptes/pocs/tests/test_observation.py b/src/panoptes/pocs/tests/test_observation.py index b40893215..3c3af7651 100644 --- a/src/panoptes/pocs/tests/test_observation.py +++ b/src/panoptes/pocs/tests/test_observation.py @@ -7,103 +7,103 @@ @pytest.fixture -def field(dynamic_config_server, config_port): - return Field('Test Observation', '20h00m43.7135s +22d42m39.0645s', config_port=config_port) +def field(): + return Field('Test Observation', '20h00m43.7135s +22d42m39.0645s') -def test_create_observation_no_field(dynamic_config_server, config_port): +def test_create_observation_no_field(): with pytest.raises(TypeError): - Observation(config_port=config_port) + Observation() -def test_create_observation_bad_field(dynamic_config_server, config_port): +def test_create_observation_bad_field(): with pytest.raises(AssertionError): - Observation('20h00m43.7135s +22d42m39.0645s', config_port=config_port) + Observation('20h00m43.7135s +22d42m39.0645s') -def test_create_observation_exptime_no_units(dynamic_config_server, config_port, field): +def test_create_observation_exptime_no_units(field): with pytest.raises(TypeError): - Observation(field, exptime=1.0, config_port=config_port) + Observation(field, exptime=1.0) -def test_create_observation_exptime_bad(dynamic_config_server, config_port, field): +def test_create_observation_exptime_bad(field): with pytest.raises(AssertionError): - Observation(field, exptime=0.0 * u.second, config_port=config_port) + Observation(field, exptime=0.0 * u.second) -def test_create_observation_exptime_minutes(dynamic_config_server, config_port, field): - obs = Observation(field, exptime=5.0 * u.minute, config_port=config_port) +def test_create_observation_exptime_minutes(field): + obs = Observation(field, exptime=5.0 * u.minute) assert obs.exptime == 300 * u.second -def test_bad_priority(dynamic_config_server, config_port, field): +def test_bad_priority(field): with pytest.raises(AssertionError): - Observation(field, priority=-1, config_port=config_port) + Observation(field, priority=-1) -def test_good_priority(dynamic_config_server, config_port, field): - obs = Observation(field, priority=5.0, config_port=config_port) +def test_good_priority(field): + obs = Observation(field, priority=5.0) assert obs.priority == 5.0 -def test_priority_str(dynamic_config_server, config_port, field): - obs = Observation(field, priority="5", config_port=config_port) +def test_priority_str(field): + obs = Observation(field, priority="5") assert obs.priority == 5.0 -def test_bad_min_set_combo(dynamic_config_server, config_port, field): +def test_bad_min_set_combo(field): with pytest.raises(AssertionError): - Observation(field, exp_set_size=7, config_port=config_port) + Observation(field, exp_set_size=7) with pytest.raises(AssertionError): - Observation(field, min_nexp=57, config_port=config_port) + Observation(field, min_nexp=57) -def test_small_sets(dynamic_config_server, config_port, field): +def test_small_sets(field): obs = Observation(field, exptime=1 * u.second, min_nexp=1, - exp_set_size=1, config_port=config_port) + exp_set_size=1) assert obs.minimum_duration == 1 * u.second assert obs.set_duration == 1 * u.second -def test_good_min_set_combo(dynamic_config_server, config_port, field): - obs = Observation(field, min_nexp=21, exp_set_size=3, config_port=config_port) +def test_good_min_set_combo(field): + obs = Observation(field, min_nexp=21, exp_set_size=3) assert isinstance(obs, Observation) -def test_default_min_duration(dynamic_config_server, config_port, field): - obs = Observation(field, config_port=config_port) +def test_default_min_duration(field): + obs = Observation(field) assert obs.minimum_duration == 7200 * u.second -def test_default_set_duration(dynamic_config_server, config_port, field): - obs = Observation(field, config_port=config_port) +def test_default_set_duration(field): + obs = Observation(field) assert obs.set_duration == 1200 * u.second -def test_print(dynamic_config_server, config_port, field): +def test_print(field): obs = Observation(field, exptime=17.5 * u.second, min_nexp=27, - exp_set_size=9, config_port=config_port) + exp_set_size=9) test_str = "Test Observation: 17.5 s exposures in blocks of 9, minimum 27, priority 100" assert str(obs) == test_str -def test_seq_time(dynamic_config_server, config_port, field): +def test_seq_time(field): obs = Observation(field, exptime=17.5 * u.second, min_nexp=27, - exp_set_size=9, config_port=config_port) + exp_set_size=9) assert obs.seq_time is None -def test_no_exposures(dynamic_config_server, config_port, field): +def test_no_exposures(field): obs = Observation(field, exptime=17.5 * u.second, min_nexp=27, - exp_set_size=9, config_port=config_port) + exp_set_size=9) assert obs.first_exposure is None assert obs.last_exposure is None assert obs.pointing_image is None -def test_last_exposure_and_reset(dynamic_config_server, config_port, field): +def test_last_exposure_and_reset(field): obs = Observation(field, exptime=17.5 * u.second, min_nexp=27, - exp_set_size=9, config_port=config_port) + exp_set_size=9) status = obs.status assert status['current_exp'] == obs.current_exp_num diff --git a/src/panoptes/pocs/tests/test_observatory.py b/src/panoptes/pocs/tests/test_observatory.py index acf74309e..c33f58f66 100644 --- a/src/panoptes/pocs/tests/test_observatory.py +++ b/src/panoptes/pocs/tests/test_observatory.py @@ -7,6 +7,7 @@ from panoptes.pocs import __version__ from panoptes.utils import error from panoptes.utils.config.client import set_config +from panoptes.utils.serializers import to_json from panoptes.pocs import hardware from panoptes.pocs.mount import AbstractMount @@ -21,26 +22,39 @@ from panoptes.pocs.scheduler import create_scheduler_from_config from panoptes.pocs.utils.location import create_location_from_config +import requests + +config_host = 'localhost' +config_port = 6563 +url = f'http://{config_host}:{config_port}/reset-config' + + +def reset_conf(): + response = requests.post(url, + data=to_json({'reset': True}), + headers={'Content-Type': 'application/json'} + ) + assert response.ok + @pytest.fixture(scope='function') -def cameras(dynamic_config_server, config_port): - return create_camera_simulator(config_port=config_port) +def cameras(): + return create_camera_simulator() @pytest.fixture(scope='function') -def mount(dynamic_config_server, config_port): +def mount(): return create_mount_simulator() @pytest.fixture -def observatory(dynamic_config_server, config_port, mount, cameras, images_dir): +def observatory(mount, cameras, images_dir): """Return a valid Observatory instance with a specific config.""" - site_details = create_location_from_config(config_port=config_port) - scheduler = create_scheduler_from_config(config_port=config_port, - observer=site_details['observer']) + site_details = create_location_from_config() + scheduler = create_scheduler_from_config(observer=site_details['observer']) - obs = Observatory(scheduler=scheduler, config_port=config_port) + obs = Observatory(scheduler=scheduler) obs.set_mount(mount) for cam_name, cam in cameras.items(): obs.add_camera(cam_name, cam) @@ -48,64 +62,63 @@ def observatory(dynamic_config_server, config_port, mount, cameras, images_dir): return obs -def test_camera_already_exists(dynamic_config_server, config_port, observatory, cameras): +def test_camera_already_exists(observatory, cameras): for cam_name, cam in cameras.items(): observatory.add_camera(cam_name, cam) -def test_remove_cameras(dynamic_config_server, config_port, observatory, cameras): +def test_remove_cameras(observatory, cameras): for cam_name, cam in cameras.items(): observatory.remove_camera(cam_name) -def test_bad_site(dynamic_config_server, config_port): - set_config('location', {}, port=config_port) +def test_bad_site(): + set_config('location', {}) with pytest.raises(error.PanError): - Observatory(config_port=config_port) + Observatory() + + reset_conf() -def test_cannot_observe(dynamic_config_server, config_port, caplog): - obs = Observatory(config_port=config_port) +def test_cannot_observe(caplog): + obs = Observatory() - site_details = create_location_from_config(config_port=config_port) + site_details = create_location_from_config() cameras = create_camera_simulator() assert obs.can_observe is False time.sleep(0.5) # log sink time log_record = caplog.records[-1] - assert log_record.message.endswith("not present, cannot observe") and log_record.levelname == "WARNING" - obs.scheduler = create_scheduler_from_config(observer=site_details['observer'], config_port=config_port) + assert log_record.message.endswith("not present") and log_record.levelname == "WARNING" + obs.scheduler = create_scheduler_from_config(observer=site_details['observer']) assert obs.can_observe is False time.sleep(0.5) # log sink time log_record = caplog.records[-1] - assert log_record.message.endswith("not present, cannot observe") and log_record.levelname == "WARNING" + assert log_record.message.endswith("not present") and log_record.levelname == "WARNING" for cam_name, cam in cameras.items(): obs.add_camera(cam_name, cam) assert obs.can_observe is False log_record = caplog.records[-1] time.sleep(0.5) # log sink time - assert log_record.message.endswith("not present, cannot observe") and log_record.levelname == "WARNING" + assert log_record.message.endswith("not present") and log_record.levelname == "WARNING" -def test_camera_wrong_type(dynamic_config_server, config_port): +def test_camera_wrong_type(): # Remove mount simulator - set_config('simulator', hardware.get_all_names(without='camera'), port=config_port) + set_config('simulator', hardware.get_all_names(without='camera')) with pytest.raises(AttributeError): - Observatory(cameras=[Time.now()], - config_port=config_port) + Observatory(cameras=[Time.now()]) with pytest.raises(AssertionError): - Observatory(cameras={'Cam00': Time.now()}, - config_port=config_port) + Observatory(cameras={'Cam00': Time.now()}) -def test_camera(dynamic_config_server, config_port): - cameras = create_camera_simulator(config_port=config_port) - obs = Observatory(cameras=cameras, - config_port=config_port) +def test_camera(): + cameras = create_camera_simulator() + obs = Observatory(cameras=cameras) assert obs.has_cameras @@ -118,15 +131,20 @@ def test_primary_camera_no_primary_camera(observatory): assert observatory.primary_camera is not None -def test_set_scheduler(dynamic_config_server, config_port, observatory, caplog): - site_details = create_location_from_config(config_port=config_port) - scheduler = create_scheduler_from_config( - observer=site_details['observer'], config_port=config_port) +def test_set_scheduler(observatory, caplog): + site_details = create_location_from_config() + scheduler = create_scheduler_from_config(observer=site_details['observer']) + + assert observatory.current_observation is None + observatory.set_scheduler(scheduler=None) assert observatory.scheduler is None + observatory.set_scheduler(scheduler=scheduler) + assert observatory.scheduler is not None - err_msg = 'Scheduler is not instance of BaseScheduler class, cannot add.' + err_msg = 'Scheduler is not an instance of .*BaseScheduler' + with pytest.raises(TypeError, match=err_msg): observatory.set_scheduler('scheduler') err_msg = ".*missing 1 required positional argument.*" @@ -134,20 +152,20 @@ def test_set_scheduler(dynamic_config_server, config_port, observatory, caplog): observatory.set_scheduler() -def test_set_dome(dynamic_config_server, config_port): +def test_set_dome(): set_config('dome', { 'brand': 'Simulacrum', 'driver': 'simulator', - }, port=config_port) - dome = create_dome_simulator(config_port=config_port) + }) + dome = create_dome_simulator() - obs = Observatory(dome=dome, config_port=config_port) + obs = Observatory(dome=dome) assert obs.has_dome is True obs.set_dome(dome=None) assert obs.has_dome is False obs.set_dome(dome=dome) assert obs.has_dome is True - err_msg = 'Dome is not instance of AbstractDome class, cannot add.' + err_msg = 'Dome is not an instance of .*AbstractDome' with pytest.raises(TypeError, match=err_msg): obs.set_dome('dome') err_msg = ".*missing 1 required positional argument.*" @@ -155,8 +173,8 @@ def test_set_dome(dynamic_config_server, config_port): obs.set_dome() -def test_set_mount(dynamic_config_server, config_port): - obs = Observatory(config_port=config_port) +def test_set_mount(): + obs = Observatory() assert obs.mount is None obs.set_mount(mount=None) @@ -166,12 +184,12 @@ def test_set_mount(dynamic_config_server, config_port): 'brand': 'Simulacrum', 'driver': 'simulator', 'model': 'simulator', - }, port=config_port) - mount = create_mount_from_config(config_port=config_port) + }) + mount = create_mount_from_config() obs.set_mount(mount=mount) assert isinstance(obs.mount, AbstractMount) is True - err_msg = 'Mount is not instance of AbstractMount class, cannot add.' + err_msg = 'Mount is not an instance of .*AbstractMount' with pytest.raises(TypeError, match=err_msg): obs.set_mount(mount='mount') err_msg = ".*missing 1 required positional argument.*" @@ -235,10 +253,11 @@ def test_standard_headers(observatory): observatory.scheduler.fields_file = None observatory.scheduler.fields_list = [ - {'name': 'HAT-P-20', - 'priority': '100', - 'position': '07h27m39.89s +24d20m14.7s', - }, + { + 'name': 'HAT-P-20', + 'priority': '100', + 'position': '07h27m39.89s +24d20m14.7s', + }, ] observatory.get_observation() @@ -254,7 +273,8 @@ def test_standard_headers(observatory): 'moon_fraction': 0.7880103086091879, 'moon_separation': 148.34401, 'observer': 'Generic PANOPTES Unit', - 'origin': 'Project PANOPTES'} + 'origin': 'Project PANOPTES' + } assert headers['airmass'] == pytest.approx(test_headers['airmass'], rel=1e-2) assert headers['ha_mnt'] == pytest.approx(test_headers['ha_mnt'], rel=1e-2) @@ -276,7 +296,7 @@ def test_sidereal_time(observatory): assert abs(st.value - 9.145547849536634) < 1e-4 -def test_get_observation(observatory): +def test_get_observation(observatory, caplog): os.environ['POCSTIME'] = '2016-08-13 15:00:00' observation = observatory.get_observation() assert isinstance(observation, Observation) @@ -376,21 +396,21 @@ def test_no_dome(observatory): assert observatory.close_dome() -def test_operate_dome(dynamic_config_server, config_port): +def test_operate_dome(): # Remove dome and night simulator - set_config('simulator', hardware.get_all_names(without=['dome', 'night']), port=config_port) + set_config('simulator', hardware.get_all_names(without=['dome', 'night'])) set_config('dome', { 'brand': 'Simulacrum', 'driver': 'simulator', - }, port=config_port) + }) set_config('dome', { 'brand': 'Simulacrum', 'driver': 'simulator', - }, port=config_port) - dome = create_dome_simulator(config_port=config_port) - observatory = Observatory(dome=dome, config_port=config_port) + }) + dome = create_dome_simulator() + observatory = Observatory(dome=dome) assert observatory.has_dome assert observatory.open_dome() diff --git a/src/panoptes/pocs/tests/test_pocs.py b/src/panoptes/pocs/tests/test_pocs.py index 4092447a2..a84b299e6 100644 --- a/src/panoptes/pocs/tests/test_pocs.py +++ b/src/panoptes/pocs/tests/test_pocs.py @@ -3,13 +3,16 @@ import time import pytest +import requests + +from astropy import units as u from panoptes.pocs import hardware from panoptes.pocs.core import POCS from panoptes.pocs.observatory import Observatory -from panoptes.utils import CountdownTimer from panoptes.utils.config.client import set_config +from panoptes.utils.serializers import to_json, to_yaml from panoptes.pocs.mount import create_mount_simulator from panoptes.pocs.camera import create_cameras_from_config @@ -17,32 +20,44 @@ from panoptes.pocs.scheduler import create_scheduler_from_config from panoptes.pocs.utils.location import create_location_from_config +config_host = 'localhost' +config_port = 6563 +url = f'http://{config_host}:{config_port}/reset-config' + + +def reset_conf(): + response = requests.post(url, + data=to_json({'reset': True}), + headers={'Content-Type': 'application/json'} + ) + assert response.ok + @pytest.fixture(scope='function') -def cameras(dynamic_config_server, config_port): - return create_cameras_from_config(config_port=config_port) +def cameras(): + return create_cameras_from_config() @pytest.fixture(scope='function') -def mount(dynamic_config_server, config_port): +def mount(): return create_mount_simulator() @pytest.fixture(scope='function') -def site_details(dynamic_config_server, config_port): - return create_location_from_config(config_port=config_port) +def site_details(): + return create_location_from_config() @pytest.fixture(scope='function') -def scheduler(dynamic_config_server, config_port, site_details): - return create_scheduler_from_config(config_port=config_port, observer=site_details['observer']) +def scheduler(site_details): + return create_scheduler_from_config(observer=site_details['observer']) @pytest.fixture(scope='function') -def observatory(dynamic_config_server, config_port, cameras, mount, site_details, scheduler): +def observatory(cameras, mount, site_details, scheduler): """Return a valid Observatory instance with a specific config.""" - obs = Observatory(scheduler=scheduler, config_port=config_port) + obs = Observatory(scheduler=scheduler, simulator=['power', 'weather']) for cam_name, cam in cameras.items(): obs.add_camera(cam_name, cam) @@ -52,26 +67,27 @@ def observatory(dynamic_config_server, config_port, cameras, mount, site_details @pytest.fixture(scope='function') -def dome(config_port): +def dome(): set_config('dome', { 'brand': 'Simulacrum', 'driver': 'simulator', - }, port=config_port) + }) - return create_dome_simulator(config_port=config_port) + return create_dome_simulator() @pytest.fixture(scope='function') -def pocs(dynamic_config_server, config_port, observatory): +def pocs(observatory): os.environ['POCSTIME'] = '2020-01-01 08:00:00' - pocs = POCS(observatory, run_once=True, config_port=config_port) + pocs = POCS(observatory, run_once=True, simulators=['power']) yield pocs pocs.power_down() + reset_conf() @pytest.fixture(scope='function') -def pocs_with_dome(dynamic_config_server, config_port, pocs, dome): +def pocs_with_dome(pocs, dome): # Add dome to config os.environ['POCSTIME'] = '2020-01-01 08:00:00' pocs.observatory.set_dome(dome) @@ -81,52 +97,38 @@ def pocs_with_dome(dynamic_config_server, config_port, pocs, dome): @pytest.fixture(scope='module') def valid_observation(): - return {'name': 'HIP 36850', - 'position': '113.65 deg +31.887 deg', - 'priority': '100', - 'exptime': 2, - 'min_nexp': 2, - 'exp_set_size': 2, - } - - -def test_bad_pandir_env(pocs): - pandir = os.getenv('PANDIR') - os.environ['PANDIR'] = '/foo/bar' - with pytest.raises(SystemExit): - POCS.check_environment() - os.environ['PANDIR'] = pandir - - -def test_bad_pocs_env(pocs): - pocs_dir = os.getenv('POCS') - os.environ['POCS'] = '/foo/bar' - with pytest.raises(SystemExit): - POCS.check_environment() - os.environ['POCS'] = pocs_dir - - -def test_make_log_dir(tmp_path, pocs): - log_dir = tmp_path / 'logs' - assert os.path.exists(log_dir) is False - - old_pandir = os.environ['PANDIR'] - os.environ['PANDIR'] = str(tmp_path.resolve()) - POCS.check_environment() - - assert os.path.exists(log_dir) is True - os.removedirs(log_dir) - - os.environ['PANDIR'] = old_pandir + return { + 'name': 'HIP 36850', + 'position': '113.65 deg +31.887 deg', + 'priority': '100', + 'exptime': 2, + 'min_nexp': 2, + 'exp_set_size': 2, + } + + +def test_observatory_cannot_observe(pocs): + scheduler = pocs.observatory.scheduler + pocs.observatory.scheduler = None + assert pocs.initialize() is False + pocs.observatory.scheduler = scheduler + assert pocs.initialize() + assert pocs.is_initialized + # Make sure we can do it twice. + assert pocs.initialize() + assert pocs.is_initialized -def test_simple_simulator(pocs): +def test_simple_simulator(pocs, caplog): assert isinstance(pocs, POCS) + pocs.set_config('simulator', 'all') assert pocs.is_initialized is not True - with pytest.raises(AssertionError): - pocs.run() + # Not initialized returns false and gives warning. + assert pocs.run() is False + log_record = caplog.records[-1] + assert log_record.message == 'POCS not initialized' and log_record.levelname == "WARNING" pocs.initialize() assert pocs.is_initialized @@ -143,30 +145,30 @@ def test_simple_simulator(pocs): assert pocs.is_safe() -def test_is_weather_and_dark_simulator(dynamic_config_server, config_port, pocs): +def test_is_weather_and_dark_simulator(pocs): pocs.initialize() # Night simulator - set_config('simulator', ['camera', 'mount', 'weather', 'night'], port=config_port) + pocs.set_config('simulator', 'all') os.environ['POCSTIME'] = '2020-01-01 08:00:00' # is dark assert pocs.is_dark() is True os.environ['POCSTIME'] = '2020-01-01 18:00:00' # is day assert pocs.is_dark() is True # No night simulator - set_config('simulator', ['camera', 'mount', 'weather'], port=config_port) + pocs.set_config('simulator', hardware.get_all_names(without=['night'])) os.environ['POCSTIME'] = '2020-01-01 08:00:00' # is dark assert pocs.is_dark() is True os.environ['POCSTIME'] = '2020-01-01 18:00:00' # is day assert pocs.is_dark() is False - set_config('simulator', ['camera', 'mount', 'weather', 'night'], port=config_port) + pocs.set_config('simulator', ['camera', 'mount', 'weather', 'night']) assert pocs.is_weather_safe() is True -def test_is_weather_safe_no_simulator(dynamic_config_server, config_port, pocs): +def test_is_weather_safe_no_simulator(pocs): pocs.initialize() - set_config('simulator', ['camera', 'mount', 'night'], port=config_port) + pocs.set_config('simulator', hardware.get_all_names(without=['weather'])) # Set a specific time os.environ['POCSTIME'] = '2020-01-01 18:00:00' @@ -180,7 +182,8 @@ def test_is_weather_safe_no_simulator(dynamic_config_server, config_port, pocs): assert pocs.is_weather_safe() is False -def test_unsafe_park(dynamic_config_server, config_port, pocs): +def test_unsafe_park(pocs): + pocs.set_config('simulator', 'all') pocs.initialize() assert pocs.is_initialized is True os.environ['POCSTIME'] = '2020-01-01 08:00:00' @@ -192,7 +195,7 @@ def test_unsafe_park(dynamic_config_server, config_port, pocs): # My time goes fast... os.environ['POCSTIME'] = '2020-01-01 18:00:00' - set_config('simulator', hardware.get_all_names(without=['night']), port=config_port) + pocs.set_config('simulator', hardware.get_all_names(without=['night'])) assert pocs.is_safe() is False @@ -204,12 +207,12 @@ def test_unsafe_park(dynamic_config_server, config_port, pocs): pocs.power_down() -def test_no_ac_power(dynamic_config_server, config_port, pocs): +def test_no_ac_power(pocs): # Simulator makes AC power safe assert pocs.has_ac_power() is True # Remove 'power' from simulator - set_config('simulator', hardware.get_all_names(without=['power']), port=config_port) + pocs.set_config('simulator', hardware.get_all_names(without=['power'])) pocs.initialize() @@ -265,9 +268,9 @@ def test_power_down_dome_while_running(pocs_with_dome): assert not pocs.observatory.dome.is_connected -def test_run_no_targets_and_exit(dynamic_config_server, config_port, pocs): - os.environ['POCSTIME'] = '2020-01-01 18:00:00' - set_config('simulator', hardware.get_all_names(), port=config_port) +def test_run_no_targets_and_exit(pocs): + os.environ['POCSTIME'] = '2020-01-01 19:00:00' + pocs.set_config('simulator', 'all') pocs.state = 'sleeping' @@ -278,24 +281,6 @@ def test_run_no_targets_and_exit(dynamic_config_server, config_port, pocs): assert pocs.state == 'sleeping' -def test_run_complete(dynamic_config_server, config_port, pocs, valid_observation): - os.environ['POCSTIME'] = '2020-01-01 08:00:00' - set_config('simulator', hardware.get_all_names(), port=config_port) - - pocs.state = 'sleeping' - pocs._do_states = True - - pocs.observatory.scheduler.clear_available_observations() - pocs.observatory.scheduler.add_observation(valid_observation) - - pocs.initialize() - assert pocs.is_initialized is True - - pocs.run(exit_when_done=True, run_once=True) - assert pocs.state == 'sleeping' - pocs.power_down() - - def test_pocs_park_to_ready_with_observations(pocs): # We don't want to run_once here pocs.run_once = False @@ -327,16 +312,21 @@ def test_pocs_park_to_ready_with_observations(pocs): def test_pocs_park_to_ready_without_observations(pocs): os.environ['POCSTIME'] = '2020-01-01 08:00:00' + pocs.logger.warning(f'Inserting safe weather reading') + pocs.db.insert_current('weather', {'safe': True}) assert pocs.is_safe() is True assert pocs.state == 'sleeping' pocs.next_state = 'ready' assert pocs.initialize() + pocs.logger.warning(f'Moving to ready') assert pocs.goto_next_state() assert pocs.state == 'ready' + pocs.logger.warning(f'Moving to scheduling') assert pocs.goto_next_state() assert pocs.observatory.current_observation is not None pocs.next_state = 'parking' + pocs.logger.warning(f'Moving to parking') assert pocs.goto_next_state() assert pocs.state == 'parking' assert pocs.observatory.current_observation is None @@ -356,7 +346,6 @@ def test_pocs_park_to_ready_without_observations(pocs): def test_run_wait_until_safe(observatory, valid_observation, - config_port, ): os.environ['POCSTIME'] = '2020-01-01 08:00:00' @@ -365,22 +354,24 @@ def test_run_wait_until_safe(observatory, observatory.logger.info('start_pocs ENTER') # Remove weather simulator, else it would always be safe. - set_config('simulator', hardware.get_all_names(without=['weather']), port=config_port) + observatory.set_config('simulator', hardware.get_all_names(without=['weather'])) - pocs = POCS(observatory, config_port=config_port) + pocs = POCS(observatory) pocs.set_config('wait_delay', 5) # Check safety every 5 seconds. pocs.observatory.scheduler.clear_available_observations() pocs.observatory.scheduler.add_observation(valid_observation) + assert pocs.connected is True + assert pocs.is_initialized is False pocs.initialize() pocs.logger.info('Starting observatory run') # Weather is bad and unit is is connected but not set. assert pocs.is_weather_safe() is False + assert pocs.is_initialized assert pocs.connected assert pocs.do_states - assert pocs.is_initialized assert pocs.next_state is None pocs.set_config('wait_delay', 1) @@ -414,7 +405,8 @@ def start_pocs(): assert pocs.is_safe() is True while pocs.next_state != 'slewing': - pocs.logger.warning(f'Waiting to get to scheduling state. Currently next_state={pocs.next_state}') + pocs.logger.warning( + f'Waiting to get to scheduling state. Currently next_state={pocs.next_state}') time.sleep(1) pocs.logger.warning(f'Stopping states via pocs.DO_STATES') @@ -426,17 +418,16 @@ def start_pocs(): assert pocs_thread.is_alive() is False -def test_run_power_down_interrupt(config_port, - observatory, +def test_run_power_down_interrupt(observatory, valid_observation, ): os.environ['POCSTIME'] = '2020-01-01 08:00:00' observatory.logger.info('start_pocs ENTER') # Remove weather simulator, else it would always be safe. - set_config('simulator', hardware.get_all_names(), port=config_port) + observatory.set_config('simulator', 'all') - pocs = POCS(observatory, config_port=config_port) + pocs = POCS(observatory) pocs.set_config('wait_delay', 5) # Check safety every 5 seconds. pocs.observatory.scheduler.clear_available_observations() @@ -461,7 +452,8 @@ def start_pocs(): pocs_thread.start() while pocs.next_state != 'scheduling': - pocs.logger.debug(f'Waiting to get to scheduling state. Currently next_state={pocs.next_state}') + pocs.logger.debug( + f'Waiting to get to scheduling state. Currently next_state={pocs.next_state}') time.sleep(1) pocs.logger.warning(f'Stopping states via pocs.DO_STATES') @@ -471,3 +463,41 @@ def start_pocs(): pocs_thread.join(timeout=300) assert pocs_thread.is_alive() is False + + +def test_custom_state_file(observatory, temp_file): + state_table = POCS.load_state_table() + assert isinstance(state_table, dict) + + with open(temp_file, 'w') as f: + f.write(to_yaml(state_table)) + + file_path = os.path.abspath(temp_file) + + pocs = POCS(observatory, state_machine_file=file_path, run_once=True, simulators=['power']) + pocs.initialize() + pocs.power_down() + reset_conf() + + +def test_free_space(pocs, caplog): + assert pocs.has_free_space() + + assert pocs.has_free_space(required_space=999 * u.terabyte) is False + assert 'No disk space' in caplog.records[-1].message + assert caplog.records[-1].levelname == 'ERROR' + + +def test_run_complete(pocs, valid_observation): + os.environ['POCSTIME'] = '2020-01-01 08:00:00' + pocs.set_config('simulator', 'all') + + pocs.observatory.scheduler.clear_available_observations() + pocs.observatory.scheduler.add_observation(valid_observation) + + pocs.initialize() + assert pocs.is_initialized is True + + pocs.run(exit_when_done=True, run_once=True) + assert pocs.state == 'sleeping' + pocs.power_down() diff --git a/src/panoptes/pocs/tests/test_scheduler.py b/src/panoptes/pocs/tests/test_scheduler.py index 34e56969b..7c7e9cb51 100644 --- a/src/panoptes/pocs/tests/test_scheduler.py +++ b/src/panoptes/pocs/tests/test_scheduler.py @@ -1,32 +1,51 @@ import pytest +import requests from panoptes.utils import error from panoptes.utils.config.client import set_config from panoptes.pocs.scheduler import create_scheduler_from_config from panoptes.pocs.scheduler import BaseScheduler from panoptes.pocs.utils.location import create_location_from_config +from panoptes.utils.serializers import to_json +config_host = 'localhost' +config_port = 6563 +url = f'http://{config_host}:{config_port}/reset-config' -def test_bad_scheduler_type(dynamic_config_server, config_port): - set_config('scheduler.type', 'foobar', port=config_port) - site_details = create_location_from_config(config_port=config_port) + +def reset_conf(): + response = requests.post(url, + data=to_json({'reset': True}), + headers={'Content-Type': 'application/json'} + ) + assert response.ok + + +def test_bad_scheduler_type(): + set_config('scheduler.type', 'foobar') + site_details = create_location_from_config() with pytest.raises(error.NotFound): - create_scheduler_from_config(observer=site_details['observer'], config_port=config_port) + create_scheduler_from_config(observer=site_details['observer']) + reset_conf() -def test_bad_scheduler_fields_file(dynamic_config_server, config_port): - set_config('scheduler.fields_file', 'foobar', port=config_port) - site_details = create_location_from_config(config_port=config_port) + +def test_bad_scheduler_fields_file(): + set_config('scheduler.fields_file', 'foobar') + site_details = create_location_from_config() with pytest.raises(error.NotFound): - create_scheduler_from_config(observer=site_details['observer'], config_port=config_port) + create_scheduler_from_config(observer=site_details['observer']) + + reset_conf() def test_no_observer(): assert isinstance(create_scheduler_from_config(observer=None), BaseScheduler) is True -def test_no_scheduler_in_config(dynamic_config_server, config_port): - set_config('scheduler', None, port=config_port) - site_details = create_location_from_config(config_port=config_port) +def test_no_scheduler_in_config(): + set_config('scheduler', None) + site_details = create_location_from_config() assert create_scheduler_from_config( - observer=site_details['observer'], config_port=config_port) is None + observer=site_details['observer']) is None + reset_conf() diff --git a/src/panoptes/pocs/tests/utils/test_logger.py b/src/panoptes/pocs/tests/utils/test_logger.py index cec6409ab..7a78d43ca 100644 --- a/src/panoptes/pocs/tests/utils/test_logger.py +++ b/src/panoptes/pocs/tests/utils/test_logger.py @@ -11,9 +11,8 @@ def profile(): def test_base_logger(caplog, profile, tmp_path): logger = get_logger(log_dir=str(tmp_path), - full_log_file=None, - profile=profile) + full_log_file=None) logger.debug('Hello') - time.sleep(0.5) + time.sleep(1) # Wait for log to make it there. assert caplog.records[-1].message == 'Hello' assert caplog.records[-1].levelname == 'DEBUG' diff --git a/src/panoptes/pocs/utils/location.py b/src/panoptes/pocs/utils/location.py index 3201c2c0a..d596c0e35 100644 --- a/src/panoptes/pocs/utils/location.py +++ b/src/panoptes/pocs/utils/location.py @@ -9,26 +9,25 @@ logger = get_logger() -def create_location_from_config(config_port=6563): +def create_location_from_config(): """ - Sets up the site and location details. - - Note: - These items are read from the 'site' config directive and include: - * name - * latitude - * longitude - * timezone - * pressure - * elevation - * horizon + Sets up the site and location details. + + These items are read from the 'site' config directive and include: + * name + * latitude + * longitude + * timezone + * pressure + * elevation + * horizon """ logger.debug('Setting up site details') try: - config_site = get_config('location', default=None, port=config_port) + config_site = get_config('location', default=None) if config_site is None: raise error.PanError(msg='location information not found in config.') diff --git a/src/panoptes/pocs/utils/logger.py b/src/panoptes/pocs/utils/logger.py index d9408b37f..4732ad6b0 100644 --- a/src/panoptes/pocs/utils/logger.py +++ b/src/panoptes/pocs/utils/logger.py @@ -1,4 +1,7 @@ import os +import sys +from contextlib import suppress + from loguru import logger as loguru_logger @@ -7,7 +10,8 @@ class PanLogger: Also provides a `handlers` dictionary to track attached handlers by id. - See https://loguru.readthedocs.io/en/stable/resources/recipes.html#dynamically-formatting-messages-to-properly-align-values-with-padding + See https://loguru.readthedocs.io/en/stable/resources/recipes.html#dynamically-formatting + -messages-to-properly-align-values-with-padding """ @@ -16,7 +20,7 @@ def __init__(self): # Level Time_UTC Time_Local dynamic_padding Message self.fmt = "{level:.1s} " \ "{time:MM-DD HH:mm:ss.ss!UTC}" \ - "({time:HH:mm:ss.ss}) " \ + " ({time:HH:mm:ss.ss}) " \ "| {name} {function}:{line}{extra[padding]} | " \ "{message}\n" self.handlers = dict() @@ -32,11 +36,12 @@ def format(self, record): LOGGER_INFO = PanLogger() -def get_logger(profile='panoptes', - console_log_file='panoptes.log', +def get_logger(console_log_file='panoptes.log', full_log_file='panoptes_{time:YYYYMMDD!UTC}.log', log_dir=None, - log_level='DEBUG'): + console_log_level='DEBUG', + stderr_log_level='INFO', + ): """Creates a root logger for PANOPTES used by the PanBase object. Two log files are created, one suitable for viewing on the console (via `tail`) @@ -49,7 +54,6 @@ def get_logger(profile='panoptes', `$PANDIR/logs` if `$PANDIR` exists, otherwise defaults to `.`. Args: - profile (str, optional): The name of the logger to use, defaults to 'panoptes'. console_log_file (str|None, optional): Filename for the file that is suitable for tailing in a shell (i.e., read by humans). This file is rotated daily however the files are not retained. @@ -58,9 +62,11 @@ def get_logger(profile='panoptes', website. Defaults to `panoptes_{time:YYYYMMDD!UTC}.log.gz` with a daily rotation at 11:30am and a 7 day retention policy. If `None` then no file will be generated. log_dir (str|None, optional): The directory to place the log file, see note. - log_level (str, optional): Log level for console output, defaults to 'DEBUG'. + stderr_log_level (str, optional): The log level to show on stderr, default INFO. + console_log_level (str, optional): Log level for console file output, defaults to 'DEBUG'. Note that it should be a string that matches standard `logging` levels and - also includes `TRACE` (below `DEBUG`) and `SUCCESS` (above `INFO`). + also includes `TRACE` (below `DEBUG`) and `SUCCESS` (above `INFO`). Also note this + is not the stderr output, but the output to the file to be tailed. Returns: `loguru.logger`: A configured instance of the logger. @@ -74,6 +80,22 @@ def get_logger(profile='panoptes', log_dir = os.path.normpath(log_dir) os.makedirs(log_dir, exist_ok=True) + if 'stderr' not in LOGGER_INFO.handlers: + # Remove default and add in custom stderr. + with suppress(ValueError): + loguru_logger.remove(0) + + stderr_format = "{level:.1s} " \ + "{time:MM-DD HH:mm:ss.ss!UTC} " \ + "{message}" + + stderr_id = loguru_logger.add( + sys.stdout, + format=stderr_format, + level=stderr_log_level + ) + LOGGER_INFO.handlers['stderr'] = stderr_id + # Log file for tailing on the console. if 'console' not in LOGGER_INFO.handlers: console_log_path = os.path.normpath(os.path.join(log_dir, console_log_file)) @@ -86,8 +108,9 @@ def get_logger(profile='panoptes', colorize=True, backtrace=True, diagnose=True, + catch=True, compression='gz', - level=log_level) + level=console_log_level) LOGGER_INFO.handlers['console'] = console_id # Log file for ingesting into log file service. @@ -105,10 +128,13 @@ def get_logger(profile='panoptes', level='TRACE') LOGGER_INFO.handlers['archive'] = archive_id - # Customize colors + # Customize colors. loguru_logger.level('TRACE', color='') loguru_logger.level('DEBUG', color='') loguru_logger.level('INFO', color='') loguru_logger.level('SUCCESS', color='') + # Enable the logging. + loguru_logger.enable('panoptes') + return loguru_logger diff --git a/tests/pocs_testing.yaml b/tests/pocs_testing.yaml index b59135be3..7d4f0685f 100644 --- a/tests/pocs_testing.yaml +++ b/tests/pocs_testing.yaml @@ -13,6 +13,10 @@ name: Generic PANOPTES Unit pan_id: PAN000 +pocs: + INITIALIZED: false + CONNECTED: false + INTERRUPTED: false location: name: Mauna Loa Observatory latitude: 19.54 deg @@ -34,11 +38,11 @@ directories: targets: POCS/resources/targets mounts: POCS/resources/mounts db: - name: panoptes + name: panoptes_testing type: file scheduler: type: dispatch - fields_file: simple.yaml + fields_file: simulator.yaml check_file: False mount: brand: ioptron