From 07f0e432ca68e639adbfdbbd5d112549825964a2 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Thu, 31 Jan 2019 12:20:29 +1100 Subject: [PATCH 001/105] Updating docker install for lean images --- .dockerignore | 27 +++++++++++++++++++ Dockerfile | 36 ++++++++++++------------- scripts/install/install-dependencies.sh | 4 +-- 3 files changed, 46 insertions(+), 21 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..4df3e7e47 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,27 @@ +docs/* +.git/* + +# PANOPTES specific files +conf_files/*_local.yaml +examples/notebooks/*.fits +examples/notebooks/*.jpeg + +# Development support +sftp-config.json + +# emacs backups +*~ +\#*\# + +# TeX products +*.aux +*.log +*.pdf +*.toc + +# Compiled files +*.py[co] +*.a +*.o +*.so +__pycache__ diff --git a/Dockerfile b/Dockerfile index a9e77c9d6..a10497981 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # PANOPTES development container -FROM ubuntu as build-env +FROM ubuntu:18.04 as build-env MAINTAINER Developers for PANOPTES project ARG pan_dir=/var/panoptes @@ -15,34 +15,32 @@ ENV PAWS $PANDIR/PAWS ENV PANUSER root ENV SOLVE_FIELD=/usr/bin/solve-field +COPY . /var/panoptes/POCS + # Use "bash" as replacement for "sh" # Note: I don't think this is the preferred way to do this anymore RUN rm /bin/sh && ln -s /bin/bash /bin/sh \ && apt-get update --fix-missing \ - && apt-get -y full-upgrade \ - && apt-get -y install wget build-essential zlib1g-dev bzip2 ca-certificates astrometry.net git \ + && apt-get -y install \ + astrometry.net \ + byobu \ + bzip2 \ + ca-certificates \ + git \ + wget \ + zlib1g-dev \ && rm -rf /var/lib/apt/lists/* \ - && wget --quiet https://raw.githubusercontent.com/panoptes/POCS/develop/scripts/install/install-dependencies.sh -O ~/install-pocs-dependencies.sh \ - && wget --quiet https://raw.githubusercontent.com/panoptes/POCS/develop/scripts/install/apt-packages-list.txt -O ~/apt-packages-list.txt \ - && wget --quiet https://raw.githubusercontent.com/panoptes/POCS/develop/scripts/install/conda-packages-list.txt -O ~/conda-packages-list.txt \ - && /bin/bash ~/install-pocs-dependencies.sh --no-conda --no-mongodb \ - && rm ~/install-pocs-dependencies.sh \ - && rm ~/conda-packages-list.txt \ - && rm ~/apt-packages-list.txt \ - && echo 'export PATH=/opt/conda/bin:$PATH' > /root/.bashrc \ - && mkdir -p $POCS \ && mkdir -p $PAWS \ + && mkdir -p $POCS \ && mkdir -p $PANLOG \ + && echo 'export PATH=/opt/conda/bin:$PATH' > /root/.bashrc \ && wget --quiet https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/anaconda.sh \ && /bin/bash ~/anaconda.sh -b -p /opt/conda \ && rm ~/anaconda.sh \ - && cd $pan_dir \ - && wget --quiet https://github.com/panoptes/POCS/archive/develop.tar.gz -O POCS.tar.gz \ - && tar zxf POCS.tar.gz -C $POCS --strip-components=1 \ - && rm POCS.tar.gz \ - && cd $POCS && /opt/conda/bin/pip install -Ur requirements.txt \ - && /opt/conda/bin/pip install -U setuptools \ - && /opt/conda/bin/python setup.py install \ + && cd $POCS \ + && /bin/bash scripts/install/install-dependencies.sh --no-conda --no-mongodb \ + && /opt/conda/bin/pip install -Ur requirements.txt \ + && /opt/conda/bin/pip install -e . \ && cd $PANDIR \ && /opt/conda/bin/conda clean --all --yes diff --git a/scripts/install/install-dependencies.sh b/scripts/install/install-dependencies.sh index 7fc9c98a0..76e38b639 100755 --- a/scripts/install/install-dependencies.sh +++ b/scripts/install/install-dependencies.sh @@ -87,11 +87,11 @@ if [[ "${GUESSED_A_VAR}" == "1" ]] ; then " fi -# Python 3.6 works around a problem building astroscrappy in 3.7. PYTHON_VERSION="3.6" ASTROMETRY_VERSION="0.76" ASTROMETRY_DIR="${PANDIR}/astrometry" -CONDA_INSTALL_DIR="${PANDIR}/miniconda" +# CONDA_INSTALL_DIR="${PANDIR}/miniconda" +CONDA_INSTALL_DIR="/opt/conda" CONDA_SH="${CONDA_INSTALL_DIR}/etc/profile.d/conda.sh" TIMESTAMP="$(date "+%Y%m%d.%H%M%S")" INSTALL_LOGS_DIR="${PANDIR}/logs/install/${TIMESTAMP}" From ed52e71cce077be27d532f8975847fa5dd1421d8 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Thu, 31 Jan 2019 12:46:09 +1100 Subject: [PATCH 002/105] Removing dependency --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e3b5c90a0..59571724e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,5 @@ astroplan astropy >= 3.0.0 -ccdproc codecov coveralls dateparser From 1066a007fcfe714a6b9eff5e1f8842bec7c93c1d Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Fri, 1 Feb 2019 17:06:31 +1100 Subject: [PATCH 003/105] Always add a symlink to `panoptes.log` for the `all` handler. --- pocs/utils/logger.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/pocs/utils/logger.py b/pocs/utils/logger.py index 93f24feac..a523a54ef 100644 --- a/pocs/utils/logger.py +++ b/pocs/utils/logger.py @@ -249,14 +249,17 @@ def get_root_logger(profile='panoptes', log_config=None): # Create a symlink to the log file with just the name of the script and the handler # (level), as this makes it easier to find the latest file. # Use a relative path, so that if we move PANLOG the paths aren't broken. - log_symlink = os.path.join(log_dir, '{}-{}.log'.format(invoked_script, handler)) - log_symlink_target = os.path.relpath(full_log_fname, start=log_dir) - try: - os.unlink(log_symlink) - except FileNotFoundError: # pragma: no cover - pass - finally: - os.symlink(log_symlink_target, log_symlink) + for f in ['{}-{}.log'.format(invoked_script, handler), 'panoptes.log']: + if f == 'panoptes.log' and handler != 'all': + continue + log_symlink = os.path.join(log_dir, f) + log_symlink_target = os.path.relpath(full_log_fname, start=log_dir) + try: + os.unlink(log_symlink) + except FileNotFoundError: # pragma: no cover + pass + finally: + os.symlink(log_symlink_target, log_symlink) # Configure the logger logging.config.dictConfig(log_config) From 14123d664ff6b37d04dfa88b221259bce761e438 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Fri, 1 Feb 2019 18:12:56 +1100 Subject: [PATCH 004/105] Cleaing up Dockerfile * Use apt packages from defined list * Don't use conda (our whole container is our app so using system python is fine) * Astrometry comes from apt; index files will be a docker volume --- Dockerfile | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/Dockerfile b/Dockerfile index a10497981..12a8223a2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,6 @@ ENV SHELL /bin/bash ENV PANDIR $pan_dir ENV PANLOG $PANDIR/logs ENV POCS $PANDIR/POCS -ENV PAWS $PANDIR/PAWS ENV PANUSER root ENV SOLVE_FIELD=/usr/bin/solve-field @@ -20,29 +19,20 @@ COPY . /var/panoptes/POCS # Use "bash" as replacement for "sh" # Note: I don't think this is the preferred way to do this anymore RUN rm /bin/sh && ln -s /bin/bash /bin/sh \ - && apt-get update --fix-missing \ - && apt-get -y install \ + && apt-get update \ + && apt-get --yes install \ + `cut '-d#' -f1 $POCS/scripts/install/apt-packages-list.txt | sort | uniq` \ astrometry.net \ - byobu \ - bzip2 \ - ca-certificates \ - git \ + python3-pip \ wget \ - zlib1g-dev \ && rm -rf /var/lib/apt/lists/* \ - && mkdir -p $PAWS \ && mkdir -p $POCS \ && mkdir -p $PANLOG \ - && echo 'export PATH=/opt/conda/bin:$PATH' > /root/.bashrc \ - && wget --quiet https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/anaconda.sh \ - && /bin/bash ~/anaconda.sh -b -p /opt/conda \ - && rm ~/anaconda.sh \ + && mkdir -p ${PANDIR}/astrometry/data \ + && echo "add_path /var/panoptes/astrometry/data" >> /etc/astrometry.cfg \ && cd $POCS \ - && /bin/bash scripts/install/install-dependencies.sh --no-conda --no-mongodb \ - && /opt/conda/bin/pip install -Ur requirements.txt \ - && /opt/conda/bin/pip install -e . \ - && cd $PANDIR \ - && /opt/conda/bin/conda clean --all --yes + && pip3 install -Ur requirements.txt \ + && pip3 install -e . WORKDIR ${POCS} From a8360c33deae5f93927a1736a1b1a8deb7f0fa36 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Fri, 1 Feb 2019 18:14:28 +1100 Subject: [PATCH 005/105] Adding docker-compose files. Base file for running unit could be used with just `docker-compose up` Whereas development can use `docker-compose -f docker-compose.yml -f docker-compose.dev.yaml` which will merely bind the host `$POCS` dir to the container. --- docker-compose.dev.yml | 7 +++++++ docker-compose.yml | 23 +++++++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 docker-compose.dev.yml create mode 100644 docker-compose.yml diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 000000000..ebf1915d0 --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,7 @@ +version: "3.2" +services: + pocs: + volumes: + - type: bind + source: /var/panoptes/POCS + target: /var/panoptes/POCS diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..047e8c675 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,23 @@ +version: '3.2' +services: + pocs: + image: gcr.io/panoptes-survey/pocs + privileged: true + env_file: .env + command: tail -F $PANLOG/pocs_shell-info.log + volumes: + - type: volume + source: astrometry_index + target: /var/panoptes/astrometry/data + read_only: true + - type: bind + source: /var/panoptes/images + target: /var/panoptes/images + - type: bind + source: /var/panoptes/json_store + target: /var/panoptes/json_store + - type: bind + source: /var/panoptes/logs + target: /var/panoptes/logs +volumes: + astrometry_index: From 5cc4a6d13fcbea1c453831383c3b7e01977821a4 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Mon, 4 Feb 2019 16:11:04 +1100 Subject: [PATCH 006/105] Allow for different host names in both publishers and sockets. This makes it so we can connect across a different network, not just localhost. The main usage will probably be within a private network while running docker so we turn a blind eye to any security. --- pocs/utils/messaging.py | 31 ++++++++++++------- .../docker_files/Dockerfile | 0 .../docker_files/docker-compose.dev.yml | 0 .../docker_files/docker-compose.yml | 2 +- 4 files changed, 21 insertions(+), 12 deletions(-) rename Dockerfile => resources/docker_files/Dockerfile (100%) rename docker-compose.dev.yml => resources/docker_files/docker-compose.dev.yml (100%) rename docker-compose.yml => resources/docker_files/docker-compose.yml (88%) diff --git a/pocs/utils/messaging.py b/pocs/utils/messaging.py index d7790371e..11eb14bc0 100644 --- a/pocs/utils/messaging.py +++ b/pocs/utils/messaging.py @@ -11,7 +11,6 @@ from json import loads from pocs.utils import current_time -from pocs.utils import CountdownTimer from pocs.utils.logger import get_root_logger @@ -122,13 +121,15 @@ def run_forwarder(cls, subscriber, publisher, ready_fn=None, done_fn=None): @classmethod def create_publisher(cls, port, bind=False, connect=True): - """ Create a publisher + """Create a publisher Args: port (int): The port (on localhost) to bind to. + bind (bool, optional): If socket should be bound (i.e. a server), default False. + connect (bool, optional): If socked should just connect once, default True. Returns: - A ZMQ PUB socket + `zmq.Socket`: A ZMQ socket. """ obj = cls() @@ -137,21 +138,29 @@ def create_publisher(cls, port, bind=False, connect=True): socket = obj.context.socket(zmq.PUB) if bind: - socket.bind('tcp://*:{}'.format(port)) + host = '*' + socket.bind(f'tcp://{host}:{port}') elif connect: - socket.connect('tcp://localhost:{}'.format(port)) + host = 'localhost' + socket.connect(f'tcp://{host}:{port}') obj.socket = socket return obj @classmethod - def create_subscriber(cls, port, topic='', bind=False, connect=True): - """ Create a listener + def create_subscriber(cls, port, topic='', host='localhost', bind=False, connect=True): + """Create a listener Args: - port (int): The port (on localhost) to bind to. - topic (str): Which topic or topic prefix to subscribe to. + port (int): The port (on localhost) to bind to. + topic (str): Which topic or topic prefix to subscribe to. + host (str, optional): Host name to connect to, default 'localhost'. + bind (bool, optional): If subscriber should bind as a server, default False. + connect (bool, optional): If socket should make one-time connection, default True. + + Returns: + `zmq.Socket`: A ZMQ socket. """ obj = cls() @@ -161,11 +170,11 @@ def create_subscriber(cls, port, topic='', bind=False, connect=True): if bind: try: - socket.bind('tcp://*:{}'.format(port)) + socket.bind(f'tcp://*:{port}') except zmq.error.ZMQError: obj.logger.debug('Problem binding port {}'.format(port)) elif connect: - socket.connect('tcp://localhost:{}'.format(port)) + socket.connect(f'tcp://{host}:{port}') socket.setsockopt_string(zmq.SUBSCRIBE, topic) diff --git a/Dockerfile b/resources/docker_files/Dockerfile similarity index 100% rename from Dockerfile rename to resources/docker_files/Dockerfile diff --git a/docker-compose.dev.yml b/resources/docker_files/docker-compose.dev.yml similarity index 100% rename from docker-compose.dev.yml rename to resources/docker_files/docker-compose.dev.yml diff --git a/docker-compose.yml b/resources/docker_files/docker-compose.yml similarity index 88% rename from docker-compose.yml rename to resources/docker_files/docker-compose.yml index 047e8c675..44caddfea 100644 --- a/docker-compose.yml +++ b/resources/docker_files/docker-compose.yml @@ -4,7 +4,7 @@ services: image: gcr.io/panoptes-survey/pocs privileged: true env_file: .env - command: tail -F $PANLOG/pocs_shell-info.log + command: bash -c "echo 'POCS Docker started' && tail -f /dev/null" volumes: - type: volume source: astrometry_index From aafc143e036af7b6211910203e4ba52ef474bfae Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Tue, 5 Feb 2019 07:22:21 +1100 Subject: [PATCH 007/105] POCS Docker This presents a number of changes for building and running POCS and friends in a docker environment. As of this PR, starting the environment (via docker-compose) will launch four separate services (container images): * messaging-hub * weather * paws * pocs As of now, it will also map the following directories to the host: * $PANDIR/json_store * $PANDIR/logs * $PANDIR/images * $PANDIR/POCS It also appropriately sets up the internal network that allows the services to communicate with each other over zeromq and exposes port 8080 on the host machine for reaching PAWS. The environment can be started with a helper script: ```bash $POCS/scripts/run_pocs_docker.sh [help|jupyterlab] ``` As of now there is one additonal option that can be passed (`jupyterlab`) which will start a jupyterlab environment within the running pocs service, exposing port 8888 on the host for access. The POCS repo is installed in development mode (i.e. `pip install -e .`) so that changes to the `$POCS` dir on the host will be reflected in the running service. To do still: * Environment sensors a la the weather container. * Convenience script to make it easier for non-technical users to install docker, authenticate, pull images, etc. * Lots more I'm sure. Note: this runs all of the docker images in `privileged` mode, meaning they have access to the host. Also: * Dockerfile to create a `pocs-base` image that contains the POCS repository installed on an ubuntu 18.04 base image. This includes an install of all the dependencies, including astrometry.net. Some of the apt-packages have been cleaned up or changed as part of this PR. * docker-compose.yaml files for easily starting all of the containers and helpers. * Bash script for starting environments. * Helper scripts for starting services. The POCS script (`$POCS/scripts/docker/run_pocs.sh`) still needs some work. * Small README updates (more likely to come). --- .../docker_files/Dockerfile => Dockerfile | 5 +- README.md | 3 +- conf_files/pocs.yaml | 3 + pocs/core.py | 5 +- pocs/utils/logger.py | 17 ++--- pocs/utils/messaging.py | 3 +- requirements.txt | 2 +- resources/docker_files/docker-compose.dev.yml | 7 -- .../docker-compose.jupyterlab.yml | 9 +++ resources/docker_files/docker-compose.yml | 64 ++++++++++++++++++- resources/docker_files/env_file | 3 + scripts/docker/run_pocs.py | 21 ++++++ scripts/docker/run_weather.py | 40 ++++++++++++ scripts/install/apt-packages-list.txt | 20 ++---- scripts/run_messaging_hub.py | 2 +- scripts/run_social_messaging.py | 2 +- scripts/start_pocs_docker.sh | 34 ++++++++++ 17 files changed, 193 insertions(+), 47 deletions(-) rename resources/docker_files/Dockerfile => Dockerfile (89%) delete mode 100644 resources/docker_files/docker-compose.dev.yml create mode 100644 resources/docker_files/docker-compose.jupyterlab.yml create mode 100644 resources/docker_files/env_file create mode 100755 scripts/docker/run_pocs.py create mode 100755 scripts/docker/run_weather.py create mode 100755 scripts/start_pocs_docker.sh diff --git a/resources/docker_files/Dockerfile b/Dockerfile similarity index 89% rename from resources/docker_files/Dockerfile rename to Dockerfile index 12a8223a2..2a8ffc3ac 100644 --- a/resources/docker_files/Dockerfile +++ b/Dockerfile @@ -14,7 +14,10 @@ ENV POCS $PANDIR/POCS ENV PANUSER root ENV SOLVE_FIELD=/usr/bin/solve-field -COPY . /var/panoptes/POCS +COPY . $POCS + +# Note that pocs-base has the default ubuntu environment, so +# we need to specify python3 so we don't get python2 # Use "bash" as replacement for "sh" # Note: I don't think this is the preferred way to do this anymore diff --git a/README.md b/README.md index 77898af38..a7839231a 100644 --- a/README.md +++ b/README.md @@ -125,8 +125,7 @@ are some helper scripts to make this easier (from [here](https://cloud.google.co ``` gcloud components install docker-credential-gcr -docker-credential-gcr configure-docker -docker-credential-gcr gcr-login +gcloud auth configure-docker ``` #### Pull POCS container diff --git a/conf_files/pocs.yaml b/conf_files/pocs.yaml index c04c1b21b..7eebc392a 100644 --- a/conf_files/pocs.yaml +++ b/conf_files/pocs.yaml @@ -66,6 +66,9 @@ messaging: # Must match ports in peas.yaml. cmd_port: 6500 msg_port: 6510 + messaging_host: localhost + # If running in Docker uncomment the line below + # messaging_host: messaging ########################## Observations ######################################## # An observation folder contains a contiguous sequence of images of a target/field diff --git a/pocs/core.py b/pocs/core.py index 8306b7058..ef2463834 100644 --- a/pocs/core.py +++ b/pocs/core.py @@ -662,6 +662,7 @@ def _setup_messaging(self): cmd_port = self.config['messaging']['cmd_port'] msg_port = self.config['messaging']['msg_port'] + messaging_host = self.config['messaging']['messaging_host'] def create_forwarder(port): try: @@ -683,10 +684,10 @@ def create_forwarder(port): self._cmd_queue = multiprocessing.Queue() self._sched_queue = multiprocessing.Queue() - self._msg_publisher = PanMessaging.create_publisher(msg_port) + self._msg_publisher = PanMessaging.create_publisher(msg_port, host=messaging_host) def check_message_loop(cmd_queue): - cmd_subscriber = PanMessaging.create_subscriber(cmd_port + 1) + cmd_subscriber = PanMessaging.create_subscriber(cmd_port + 1, host=messaging_host) poller = zmq.Poller() poller.register(cmd_subscriber.socket, zmq.POLLIN) diff --git a/pocs/utils/logger.py b/pocs/utils/logger.py index a523a54ef..3488d3ea0 100644 --- a/pocs/utils/logger.py +++ b/pocs/utils/logger.py @@ -10,6 +10,7 @@ from tempfile import gettempdir import time from warnings import warn +from contextlib import suppress from pocs.utils.config import load_config @@ -249,17 +250,11 @@ def get_root_logger(profile='panoptes', log_config=None): # Create a symlink to the log file with just the name of the script and the handler # (level), as this makes it easier to find the latest file. # Use a relative path, so that if we move PANLOG the paths aren't broken. - for f in ['{}-{}.log'.format(invoked_script, handler), 'panoptes.log']: - if f == 'panoptes.log' and handler != 'all': - continue - log_symlink = os.path.join(log_dir, f) - log_symlink_target = os.path.relpath(full_log_fname, start=log_dir) - try: - os.unlink(log_symlink) - except FileNotFoundError: # pragma: no cover - pass - finally: - os.symlink(log_symlink_target, log_symlink) + log_symlink = os.path.join(log_dir, '{}-{}.log'.format(invoked_script, handler)) + log_symlink_target = os.path.relpath(full_log_fname, start=log_dir) + with suppress(FileNotFoundError): + os.unlink(log_symlink) + os.symlink(log_symlink_target, log_symlink) # Configure the logger logging.config.dictConfig(log_config) diff --git a/pocs/utils/messaging.py b/pocs/utils/messaging.py index 11eb14bc0..8e01ebb95 100644 --- a/pocs/utils/messaging.py +++ b/pocs/utils/messaging.py @@ -120,7 +120,7 @@ def run_forwarder(cls, subscriber, publisher, ready_fn=None, done_fn=None): done_fn() @classmethod - def create_publisher(cls, port, bind=False, connect=True): + def create_publisher(cls, port, host='localhost', bind=False, connect=True): """Create a publisher Args: @@ -141,7 +141,6 @@ def create_publisher(cls, port, bind=False, connect=True): host = '*' socket.bind(f'tcp://{host}:{port}') elif connect: - host = 'localhost' socket.connect(f'tcp://{host}:{port}') obj.socket = socket diff --git a/requirements.txt b/requirements.txt index 59571724e..28673372d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ coveralls dateparser gcloud google-cloud-storage +jupyterlab matplotlib >= 2.0.0,<3.0.0 mocket numpy >= 1.12.1, !=1.15.3 @@ -17,7 +18,6 @@ PyYAML >= 3.11 pyzmq >= 15.3.0 readline requests -scikit_image >= 0.12.3 scipy >= 0.17.1 transitions >= 0.4.0 tweepy diff --git a/resources/docker_files/docker-compose.dev.yml b/resources/docker_files/docker-compose.dev.yml deleted file mode 100644 index ebf1915d0..000000000 --- a/resources/docker_files/docker-compose.dev.yml +++ /dev/null @@ -1,7 +0,0 @@ -version: "3.2" -services: - pocs: - volumes: - - type: bind - source: /var/panoptes/POCS - target: /var/panoptes/POCS diff --git a/resources/docker_files/docker-compose.jupyterlab.yml b/resources/docker_files/docker-compose.jupyterlab.yml new file mode 100644 index 000000000..57c9b6d84 --- /dev/null +++ b/resources/docker_files/docker-compose.jupyterlab.yml @@ -0,0 +1,9 @@ +version: "3.2" +services: + pocs: + command: bash -c "jupyter-lab --ip 0.0.0.0 --allow-root --no-browser" + ports: + - target: 8888 + published: 8888 + protocol: tcp + mode: host \ No newline at end of file diff --git a/resources/docker_files/docker-compose.yml b/resources/docker_files/docker-compose.yml index 44caddfea..b5db7f33d 100644 --- a/resources/docker_files/docker-compose.yml +++ b/resources/docker_files/docker-compose.yml @@ -1,10 +1,64 @@ version: '3.2' services: + messaging: + image: gcr.io/panoptes-survey/pocs-base + privileged: true + env_file: env_file + volumes: + - type: bind + source: /var/panoptes/logs + target: /var/panoptes/logs + - type: bind + source: /var/panoptes/POCS + target: /var/panoptes/POCS + command: bash -c "python3 $POCS/scripts/run_messaging_hub.py --from_config" + weather: + image: gcr.io/panoptes-survey/pocs-base + privileged: true + env_file: env_file + depends_on: + - "messaging" + volumes: + - type: bind + source: /var/panoptes/json_store + target: /var/panoptes/json_store + - type: bind + source: /var/panoptes/logs + target: /var/panoptes/logs + - type: bind + source: /var/panoptes/POCS + target: /var/panoptes/POCS + command: bash -c "python3 $POCS/scripts/docker/run_weather.py" + paws: + image: gcr.io/panoptes-survey/paws + privileged: true + env_file: env_file + depends_on: + - "messaging" + ports: + - target: 8888 + published: 8080 + protocol: tcp + mode: host + volumes: + - type: bind + source: /var/panoptes/json_store + target: /var/panoptes/json_store + - type: bind + source: /var/panoptes/logs + target: /var/panoptes/logs + - type: bind + source: /var/panoptes/POCS + target: /var/panoptes/POCS + - type: bind + source: /var/panoptes/PAWS + target: /var/panoptes/PAWS pocs: - image: gcr.io/panoptes-survey/pocs + image: gcr.io/panoptes-survey/pocs-base privileged: true - env_file: .env - command: bash -c "echo 'POCS Docker started' && tail -f /dev/null" + env_file: env_file + depends_on: + - "messaging" volumes: - type: volume source: astrometry_index @@ -19,5 +73,9 @@ services: - type: bind source: /var/panoptes/logs target: /var/panoptes/logs + - type: bind + source: /var/panoptes/POCS + target: /var/panoptes/POCS + command: bash -c "python3 $POCS/scripts/docker/run_pocs.py" volumes: astrometry_index: diff --git a/resources/docker_files/env_file b/resources/docker_files/env_file new file mode 100644 index 000000000..9d410fb26 --- /dev/null +++ b/resources/docker_files/env_file @@ -0,0 +1,3 @@ +PANDIR=/var/panoptes +POCS=/var/panoptes/POCS +PANLOG=/var/panoptes/logs diff --git a/scripts/docker/run_pocs.py b/scripts/docker/run_pocs.py new file mode 100755 index 000000000..15605a36c --- /dev/null +++ b/scripts/docker/run_pocs.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 + +from pocs.observatory import Observatory +from pocs.core import POCS +from pocs.utils import error + + +def main(simulator='all'): + try: + print("Setting up POCS") + observatory = Observatory(simulator=simulator) + pocs = POCS(observatory, messaging=True) + pocs.initialize() + print("POCS initialized") + # pocs.run() + except error.PanError as e: + print('Problem setting up POCS: {}'.format(e)) + + +if __name__ == '__main__': + main() diff --git a/scripts/docker/run_weather.py b/scripts/docker/run_weather.py new file mode 100755 index 000000000..6e7a6722d --- /dev/null +++ b/scripts/docker/run_weather.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 + +from peas.weather import AAGCloudSensor + +from pocs.utils.config import load_config +from pocs.utils import CountdownTimer +from pocs.utils import error + + +def main(config, loop_delay=60): + try: + try: + port = config['weather']['aag_cloud']['serial_port'] + except KeyError: + port = '/dev/ttyUSB0' + print("Loading AAG Cloud Sensor on {}".format(port)) + weather = AAGCloudSensor(serial_address=port, store_result=True) + if weather.AAG is None: + print(f'AAG not set up properly, check port') + return + print("Weather station set up") + except error.PanError as e: + print('Problem setting up PEAS: {}'.format(e)) + + timer = CountdownTimer(loop_delay) + + # Loop forever. + print(f'Starting weather station loop with {loop_delay}s delay') + while True: + timer.restart() + weather.capture(store_result=True, send_message=True) + + timer.sleep() + + # TODO: listen for messages and react to them + + +if __name__ == '__main__': + config = load_config() + main(config=config) diff --git a/scripts/install/apt-packages-list.txt b/scripts/install/apt-packages-list.txt index 08af9bd2b..d8ba5747a 100644 --- a/scripts/install/apt-packages-list.txt +++ b/scripts/install/apt-packages-list.txt @@ -31,6 +31,9 @@ zlib1g-dev # for rendering. libcairo2-dev +# ZMQ is used for our messaging system. +libzmq3-dev + # exiftool is used in a couple of places to extract info from the .cr2 file exiftool @@ -43,12 +46,9 @@ libcfitsio-bin openssh-client openssh-server -# Graphviz is used for rendering the state machine of POCS. -graphviz -libgraphviz-dev - # Linux tool used during building and installing software (e.g. astrometry). pkg-config +gcc # Improves interaction with pocs_shell (via readline). libncurses5-dev @@ -63,18 +63,6 @@ libgeos-dev byobu tmux -# Used for creating a timelapse. -ffmpeg - -# Used for colorizing log files. -grc - # GNU coreutils provides basic commands that we depend on (echo, ls), and is # almost certainly already installed, but let's make sure. coreutils - -# Want git for upgrading to the latest version. Probably already installed -# but not necessarily (i.e. it is possible to download a zip of POCS, which -# includes this file). -git - diff --git a/scripts/run_messaging_hub.py b/scripts/run_messaging_hub.py index c4752bf59..5a99fece7 100755 --- a/scripts/run_messaging_hub.py +++ b/scripts/run_messaging_hub.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import argparse import sys diff --git a/scripts/run_social_messaging.py b/scripts/run_social_messaging.py index fc4cb24c2..e79626798 100755 --- a/scripts/run_social_messaging.py +++ b/scripts/run_social_messaging.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import argparse import sys diff --git a/scripts/start_pocs_docker.sh b/scripts/start_pocs_docker.sh new file mode 100755 index 000000000..c0b501aea --- /dev/null +++ b/scripts/start_pocs_docker.sh @@ -0,0 +1,34 @@ +#!/bin/bash -e + +usage() { + echo -n "################################################## +# Start POCS via Docker. +# +################################################## + + $ $(basename $0) [START] + + Options: + START Program to start. Currently only option is 'jupyterlab'. If no + option is given then start the state machine in the background. +" +} + +if [ $# -eq 0 ]; then + docker-compose \ + -f $POCS/resources/docker_files/docker-compose.yml \ + -p panoptes up +else + START=${1} + + if [ ${START} = 'help' ] || [ ${START} = '-h' ] || [ ${START} = '--help' ]; then + usage + exit 1 + else + docker-compose \ + -f $POCS/resources/docker_files/docker-compose.yml \ + -f $POCS/resources/docker_files/docker-compose.${START}.yml \ + -p panoptes up + fi + +fi From 69f5796b8cfd062cc0c3c08ad33a66b090f2dcad Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Wed, 6 Feb 2019 08:06:01 +1100 Subject: [PATCH 008/105] Fixing a few items related to running the docker images * Giving a jupyterconsole option for starting POCS * Fixing weather * Fixing ports --- requirements.txt | 1 + resources/docker_files/docker-compose.jupyterconsole.yml | 4 ++++ resources/docker_files/docker-compose.jupyterlab.yml | 2 +- resources/docker_files/docker-compose.yml | 2 +- scripts/docker/run_weather.py | 4 ++-- 5 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 resources/docker_files/docker-compose.jupyterconsole.yml diff --git a/requirements.txt b/requirements.txt index 28673372d..c598230f5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ coveralls dateparser gcloud google-cloud-storage +jupyter-console jupyterlab matplotlib >= 2.0.0,<3.0.0 mocket diff --git a/resources/docker_files/docker-compose.jupyterconsole.yml b/resources/docker_files/docker-compose.jupyterconsole.yml new file mode 100644 index 000000000..e9f3e6602 --- /dev/null +++ b/resources/docker_files/docker-compose.jupyterconsole.yml @@ -0,0 +1,4 @@ +version: "3.2" +services: + pocs: + command: bash -c "jupyter-console" diff --git a/resources/docker_files/docker-compose.jupyterlab.yml b/resources/docker_files/docker-compose.jupyterlab.yml index 57c9b6d84..ca9250ec0 100644 --- a/resources/docker_files/docker-compose.jupyterlab.yml +++ b/resources/docker_files/docker-compose.jupyterlab.yml @@ -4,6 +4,6 @@ services: command: bash -c "jupyter-lab --ip 0.0.0.0 --allow-root --no-browser" ports: - target: 8888 - published: 8888 + published: 9000 protocol: tcp mode: host \ No newline at end of file diff --git a/resources/docker_files/docker-compose.yml b/resources/docker_files/docker-compose.yml index b5db7f33d..387b82b36 100644 --- a/resources/docker_files/docker-compose.yml +++ b/resources/docker_files/docker-compose.yml @@ -37,7 +37,7 @@ services: - "messaging" ports: - target: 8888 - published: 8080 + published: 8888 protocol: tcp mode: host volumes: diff --git a/scripts/docker/run_weather.py b/scripts/docker/run_weather.py index 6e7a6722d..23fa4cc66 100755 --- a/scripts/docker/run_weather.py +++ b/scripts/docker/run_weather.py @@ -36,5 +36,5 @@ def main(config, loop_delay=60): if __name__ == '__main__': - config = load_config() - main(config=config) + config = load_config('peas') + main(config) From 6cdae3b8443244d262b5a09c8ff31dfe041c308c Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Wed, 6 Feb 2019 12:04:50 +1100 Subject: [PATCH 009/105] Passing the `host` from the config to the messaging hub --- conf_files/peas.yaml | 3 +-- pocs/utils/messaging.py | 11 ++++++----- .../docker_files/docker-compose.jupyterconsole.yml | 4 ---- resources/docker_files/docker-compose.yml | 3 ++- scripts/run_messaging_hub.py | 12 +++++++++--- 5 files changed, 18 insertions(+), 15 deletions(-) delete mode 100644 resources/docker_files/docker-compose.jupyterconsole.yml diff --git a/conf_files/peas.yaml b/conf_files/peas.yaml index bf5fb8c10..87eab6bac 100644 --- a/conf_files/peas.yaml +++ b/conf_files/peas.yaml @@ -6,8 +6,7 @@ environment: weather: station: mongo aag_cloud: - # serial_port: '/dev/ttyUSB1' - serial_port: '/dev/tty.USA19H2P1.1' + serial_port: '/dev/ttyUSB1' threshold_cloudy: -25 threshold_very_cloudy: -15. threshold_windy: 50. diff --git a/pocs/utils/messaging.py b/pocs/utils/messaging.py index 8e01ebb95..87b8428cd 100644 --- a/pocs/utils/messaging.py +++ b/pocs/utils/messaging.py @@ -93,10 +93,10 @@ def create_forwarder(cls, sub_port, pub_port, ready_fn=None, done_fn=None): PanMessaging.run_forwarder(subscriber, publisher, ready_fn=ready_fn, done_fn=done_fn) @classmethod - def create_forwarder_sockets(cls, sub_port, pub_port): + def create_forwarder_sockets(cls, sub_port, pub_port, host='localhost'): cls.logger.info('Creating forwarder sockets for {} -> {}', sub_port, pub_port) - subscriber = PanMessaging.create_subscriber(sub_port, bind=True, connect=False) - publisher = PanMessaging.create_publisher(pub_port, bind=True, connect=False) + subscriber = PanMessaging.create_subscriber(sub_port, bind=True, connect=False, host=host) + publisher = PanMessaging.create_publisher(pub_port, bind=True, connect=False, host=host) return subscriber, publisher @classmethod @@ -133,7 +133,7 @@ def create_publisher(cls, port, host='localhost', bind=False, connect=True): """ obj = cls() - obj.logger.debug("Creating publisher. Binding to port {} ".format(port)) + obj.logger.debug("Creating publisher. Host {} Port {} ".format(host, port)) socket = obj.context.socket(zmq.PUB) @@ -163,7 +163,8 @@ def create_subscriber(cls, port, topic='', host='localhost', bind=False, connect """ obj = cls() - obj.logger.debug("Creating subscriber. Port: {} \tTopic: {}".format(port, topic)) + obj.logger.debug( + "Creating subscriber. Host: {} Port: {} \tTopic: {}".format(host, port, topic)) socket = obj.context.socket(zmq.SUB) diff --git a/resources/docker_files/docker-compose.jupyterconsole.yml b/resources/docker_files/docker-compose.jupyterconsole.yml deleted file mode 100644 index e9f3e6602..000000000 --- a/resources/docker_files/docker-compose.jupyterconsole.yml +++ /dev/null @@ -1,4 +0,0 @@ -version: "3.2" -services: - pocs: - command: bash -c "jupyter-console" diff --git a/resources/docker_files/docker-compose.yml b/resources/docker_files/docker-compose.yml index 387b82b36..fadc3bfca 100644 --- a/resources/docker_files/docker-compose.yml +++ b/resources/docker_files/docker-compose.yml @@ -76,6 +76,7 @@ services: - type: bind source: /var/panoptes/POCS target: /var/panoptes/POCS - command: bash -c "python3 $POCS/scripts/docker/run_pocs.py" + # No-op for now - connect via "docker exec -it panoptes_pocs_1 jupyter-console" + command: tail -f /dev/null volumes: astrometry_index: diff --git a/scripts/run_messaging_hub.py b/scripts/run_messaging_hub.py index 5a99fece7..e97b553bb 100755 --- a/scripts/run_messaging_hub.py +++ b/scripts/run_messaging_hub.py @@ -32,14 +32,14 @@ def run_forwarder(sub_port, pub_port, sub, pub): say('Forwarder for {} -> {} has stopped', sub_port, pub_port) -def run_forwarders(port_pairs): +def run_forwarders(port_pairs, host='localhost'): the_root_logger.info('Creating sockets') socket_pairs = [] for sub, pub in port_pairs: say('Creating sockets for {} -> {}', sub, pub) try: - socket_pairs.append(PanMessaging.create_forwarder_sockets(sub, pub)) + socket_pairs.append(PanMessaging.create_forwarder_sockets(sub, pub, host=host)) except Exception as e: say('Unable to create sockets: {}', e, error=True) sys.exit(1) @@ -97,6 +97,10 @@ def run_forwarders(port_pairs): action='append', type=int, help='First port of a pair to be forwarded. The other is the next integer.') + parser.add_argument( + '--host', + default='localhost', + help='Host of the machine that has messaging, default localhost') parser.add_argument( '--from_config', action='store_true', @@ -109,6 +113,7 @@ def arg_error(msg): sys.exit(1) all_ports = [] + host = args.host def validate_unique_port(port): """Confirm that the port is valid and unique among all the ports.""" @@ -134,6 +139,7 @@ def add_pair(sub, pub=None): config = load_config(config_files=['pocs']) add_pair(config['messaging']['cmd_port']) add_pair(config['messaging']['msg_port']) + host = config['messaging']['messaging_host'] if args.pairs: for sub, pub in args.pairs: @@ -148,4 +154,4 @@ def add_pair(sub, pub=None): the_root_logger = get_root_logger() - run_forwarders(sub_and_pub_pairs) + run_forwarders(sub_and_pub_pairs, host=host) From 85a43a0f89fbce3504be65feb82fbe83144e53dd Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Wed, 6 Feb 2019 12:40:49 +1100 Subject: [PATCH 010/105] Passing host along properly for messaging --- pocs/core.py | 2 +- pocs/utils/messaging.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pocs/core.py b/pocs/core.py index ef2463834..b3401f1c8 100644 --- a/pocs/core.py +++ b/pocs/core.py @@ -666,7 +666,7 @@ def _setup_messaging(self): def create_forwarder(port): try: - PanMessaging.create_forwarder(port, port + 1) + PanMessaging.create_forwarder(port, port + 1, host=messaging_host) except Exception: pass diff --git a/pocs/utils/messaging.py b/pocs/utils/messaging.py index 87b8428cd..1cde589a8 100644 --- a/pocs/utils/messaging.py +++ b/pocs/utils/messaging.py @@ -138,8 +138,7 @@ def create_publisher(cls, port, host='localhost', bind=False, connect=True): socket = obj.context.socket(zmq.PUB) if bind: - host = '*' - socket.bind(f'tcp://{host}:{port}') + socket.bind(f'tcp://*:{port}') elif connect: socket.connect(f'tcp://{host}:{port}') From aa03c088d0bc446dcae8f01b6b4ed73af7fa0bcb Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Wed, 6 Feb 2019 15:08:27 +1100 Subject: [PATCH 011/105] PAWS needs access to the images --- resources/docker_files/docker-compose.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/resources/docker_files/docker-compose.yml b/resources/docker_files/docker-compose.yml index fadc3bfca..d4431a568 100644 --- a/resources/docker_files/docker-compose.yml +++ b/resources/docker_files/docker-compose.yml @@ -53,6 +53,9 @@ services: - type: bind source: /var/panoptes/PAWS target: /var/panoptes/PAWS + - type: bind + source: /var/panoptes/images + target: /var/panoptes/images pocs: image: gcr.io/panoptes-survey/pocs-base privileged: true From 960a395272be9e98f4d043e5b1fe3486e1b335b9 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Sat, 9 Feb 2019 13:32:37 +1100 Subject: [PATCH 012/105] Run the environment sensors --- resources/docker_files/docker-compose.yml | 17 +++++++++++ scripts/docker/run_peas.py | 36 +++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100755 scripts/docker/run_peas.py diff --git a/resources/docker_files/docker-compose.yml b/resources/docker_files/docker-compose.yml index d4431a568..b910e4307 100644 --- a/resources/docker_files/docker-compose.yml +++ b/resources/docker_files/docker-compose.yml @@ -29,6 +29,23 @@ services: source: /var/panoptes/POCS target: /var/panoptes/POCS command: bash -c "python3 $POCS/scripts/docker/run_weather.py" + peas: + image: gcr.io/panoptes-survey/pocs-base + privileged: true + env_file: env_file + depends_on: + - "messaging" + volumes: + - type: bind + source: /var/panoptes/json_store + target: /var/panoptes/json_store + - type: bind + source: /var/panoptes/logs + target: /var/panoptes/logs + - type: bind + source: /var/panoptes/POCS + target: /var/panoptes/POCS + command: bash -c "python3 $POCS/scripts/docker/run_peas.py" paws: image: gcr.io/panoptes-survey/paws privileged: true diff --git a/scripts/docker/run_peas.py b/scripts/docker/run_peas.py new file mode 100755 index 000000000..e98311cca --- /dev/null +++ b/scripts/docker/run_peas.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 + +from peas.sensors import ArduinoSerialMonitor + +from pocs.utils.config import load_config +from pocs.utils import CountdownTimer +from pocs.utils import error + + +def main(config, loop_delay=60): + try: + print("Loading Arduino Serial Monitor") + environment = ArduinoSerialMonitor(auto_detect=True) + if environment is None: + print(f'Arduino Serial Monitor not set up properly, check ports') + return + print("Environmental sensors set up") + except error.PanError as e: + print('Problem setting up Arduino serial montiors: {}'.format(e)) + + timer = CountdownTimer(loop_delay) + + # Loop forever. + print(f'Starting weather station loop with {loop_delay}s delay') + while True: + timer.restart() + environment.capture(store_result=True, send_message=True) + + timer.sleep() + + # TODO: listen for messages and react to them + + +if __name__ == '__main__': + config = load_config('peas') + main(config) From 52523101cb00e2ce223a64d496b996594f0fb8f1 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Sat, 9 Feb 2019 13:36:55 +1100 Subject: [PATCH 013/105] Smaller delay on environmental sensors. Start POCS (but don't run) for messaigng --- resources/docker_files/docker-compose.yml | 3 ++- scripts/docker/run_peas.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/resources/docker_files/docker-compose.yml b/resources/docker_files/docker-compose.yml index b910e4307..2aadb4994 100644 --- a/resources/docker_files/docker-compose.yml +++ b/resources/docker_files/docker-compose.yml @@ -97,6 +97,7 @@ services: source: /var/panoptes/POCS target: /var/panoptes/POCS # No-op for now - connect via "docker exec -it panoptes_pocs_1 jupyter-console" - command: tail -f /dev/null + # command: tail -f /dev/null + command: bash -c "python3 $POCS/scripts/docker/run_pocs.py" volumes: astrometry_index: diff --git a/scripts/docker/run_peas.py b/scripts/docker/run_peas.py index e98311cca..9c1f6d0a1 100755 --- a/scripts/docker/run_peas.py +++ b/scripts/docker/run_peas.py @@ -7,7 +7,7 @@ from pocs.utils import error -def main(config, loop_delay=60): +def main(config, loop_delay=5): try: print("Loading Arduino Serial Monitor") environment = ArduinoSerialMonitor(auto_detect=True) @@ -21,7 +21,7 @@ def main(config, loop_delay=60): timer = CountdownTimer(loop_delay) # Loop forever. - print(f'Starting weather station loop with {loop_delay}s delay') + print(f'Starting environmental sensors loop with {loop_delay}s delay') while True: timer.restart() environment.capture(store_result=True, send_message=True) From ab1a0e8457e80a90f40b362cf86bdd820e05352e Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Sat, 9 Feb 2019 14:49:01 +1100 Subject: [PATCH 014/105] Fixing messaging ports for docker usage --- peas/sensors.py | 4 +++- peas/weather.py | 4 +++- requirements.txt | 3 ++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/peas/sensors.py b/peas/sensors.py index e7e475859..52ac0c4f5 100644 --- a/peas/sensors.py +++ b/peas/sensors.py @@ -75,7 +75,9 @@ def disconnect(self): def send_message(self, msg, topic='environment'): if self.messaging is None: - self.messaging = PanMessaging.create_publisher(6510) + msg_port = self.config['messaging']['msg_port'] + messaging_host = self.config['messaging']['messaging_host'] + self.messaging = PanMessaging.create_publisher(msg_port, host=messaging_host) self.messaging.send_message(topic, msg) diff --git a/peas/weather.py b/peas/weather.py index 6044cf382..d466620c3 100755 --- a/peas/weather.py +++ b/peas/weather.py @@ -606,7 +606,9 @@ def get_wind_speed(self, n=3): def send_message(self, msg, topic='weather'): if self.messaging is None: - self.messaging = PanMessaging.create_publisher(6510) + msg_port = self.config['messaging']['msg_port'] + messaging_host = self.config['messaging']['messaging_host'] + self.messaging = PanMessaging.create_publisher(msg_port, host=messaging_host) self.messaging.send_message(topic, msg) diff --git a/requirements.txt b/requirements.txt index c598230f5..9051e5dae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,8 @@ PyYAML >= 3.11 pyzmq >= 15.3.0 readline requests +scikit-image scipy >= 0.17.1 transitions >= 0.4.0 tweepy -wcsaxes +wcsaxes \ No newline at end of file From 9c00c56560974ad19a37b7fd8662e0690190aa73 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Sun, 10 Feb 2019 09:09:10 +1100 Subject: [PATCH 015/105] * Loading pocs config file so we have access to messaging info * Missing requirement for `convert` on cli. * Put weather plots in main images directory --- peas/sensors.py | 2 +- peas/weather.py | 2 +- scripts/install/apt-packages-list.txt | 2 ++ scripts/plot_weather.py | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/peas/sensors.py b/peas/sensors.py index 52ac0c4f5..29ed0bc59 100644 --- a/peas/sensors.py +++ b/peas/sensors.py @@ -20,7 +20,7 @@ class ArduinoSerialMonitor(object): """ def __init__(self, auto_detect=False, *args, **kwargs): - self.config = load_config(config_files='peas') + self.config = load_config(config_files=['pocs', 'peas']) self.logger = get_root_logger() assert 'environment' in self.config diff --git a/peas/weather.py b/peas/weather.py index d466620c3..6a38039c4 100755 --- a/peas/weather.py +++ b/peas/weather.py @@ -102,7 +102,7 @@ class AAGCloudSensor(object): """ def __init__(self, serial_address=None, store_result=True): - self.config = load_config(config_files='peas') + self.config = load_config(config_files=['pocs', 'peas']) self.logger = get_root_logger() # Read configuration diff --git a/scripts/install/apt-packages-list.txt b/scripts/install/apt-packages-list.txt index d8ba5747a..a749e0f70 100644 --- a/scripts/install/apt-packages-list.txt +++ b/scripts/install/apt-packages-list.txt @@ -4,6 +4,8 @@ # dcraw decodes Canon raw images dcraw +# For converting JPGs +imagemagick # gphoto2 enables remote control of cameras, including the Canon DSLRs that # Panoptes uses. This includes changing many settings and taking pictures. diff --git a/scripts/plot_weather.py b/scripts/plot_weather.py index 0a1754fdd..ec405c176 100755 --- a/scripts/plot_weather.py +++ b/scripts/plot_weather.py @@ -706,7 +706,7 @@ def save_plot(self, plot_filename=None): plot_filename = '{}.png'.format(self.date_string) plot_filename = os.path.join(os.path.expandvars( - '$PANDIR'), 'weather_plots', plot_filename) + '$PANDIR'), 'images', 'weather_plots', plot_filename) plot_filename = os.path.abspath(plot_filename) plot_dir = os.path.dirname(plot_filename) From 1ad2ae0565b6cb18fbaf7d37514813c400e076a4 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Sun, 10 Feb 2019 09:33:09 +1100 Subject: [PATCH 016/105] Simplify the bind volumes --- resources/docker_files/docker-compose.yml | 70 ++++------------------- 1 file changed, 12 insertions(+), 58 deletions(-) diff --git a/resources/docker_files/docker-compose.yml b/resources/docker_files/docker-compose.yml index 2aadb4994..b69692a48 100644 --- a/resources/docker_files/docker-compose.yml +++ b/resources/docker_files/docker-compose.yml @@ -5,12 +5,7 @@ services: privileged: true env_file: env_file volumes: - - type: bind - source: /var/panoptes/logs - target: /var/panoptes/logs - - type: bind - source: /var/panoptes/POCS - target: /var/panoptes/POCS + - pandir:/var/panoptes command: bash -c "python3 $POCS/scripts/run_messaging_hub.py --from_config" weather: image: gcr.io/panoptes-survey/pocs-base @@ -19,15 +14,7 @@ services: depends_on: - "messaging" volumes: - - type: bind - source: /var/panoptes/json_store - target: /var/panoptes/json_store - - type: bind - source: /var/panoptes/logs - target: /var/panoptes/logs - - type: bind - source: /var/panoptes/POCS - target: /var/panoptes/POCS + - pandir:/var/panoptes command: bash -c "python3 $POCS/scripts/docker/run_weather.py" peas: image: gcr.io/panoptes-survey/pocs-base @@ -36,15 +23,7 @@ services: depends_on: - "messaging" volumes: - - type: bind - source: /var/panoptes/json_store - target: /var/panoptes/json_store - - type: bind - source: /var/panoptes/logs - target: /var/panoptes/logs - - type: bind - source: /var/panoptes/POCS - target: /var/panoptes/POCS + - pandir:/var/panoptes command: bash -c "python3 $POCS/scripts/docker/run_peas.py" paws: image: gcr.io/panoptes-survey/paws @@ -58,21 +37,7 @@ services: protocol: tcp mode: host volumes: - - type: bind - source: /var/panoptes/json_store - target: /var/panoptes/json_store - - type: bind - source: /var/panoptes/logs - target: /var/panoptes/logs - - type: bind - source: /var/panoptes/POCS - target: /var/panoptes/POCS - - type: bind - source: /var/panoptes/PAWS - target: /var/panoptes/PAWS - - type: bind - source: /var/panoptes/images - target: /var/panoptes/images + - pandir:/var/panoptes pocs: image: gcr.io/panoptes-survey/pocs-base privileged: true @@ -80,24 +45,13 @@ services: depends_on: - "messaging" volumes: - - type: volume - source: astrometry_index - target: /var/panoptes/astrometry/data - read_only: true - - type: bind - source: /var/panoptes/images - target: /var/panoptes/images - - type: bind - source: /var/panoptes/json_store - target: /var/panoptes/json_store - - type: bind - source: /var/panoptes/logs - target: /var/panoptes/logs - - type: bind - source: /var/panoptes/POCS - target: /var/panoptes/POCS + - pandir:/var/panoptes # No-op for now - connect via "docker exec -it panoptes_pocs_1 jupyter-console" - # command: tail -f /dev/null - command: bash -c "python3 $POCS/scripts/docker/run_pocs.py" + command: tail -f /dev/null volumes: - astrometry_index: + pandir: + driver: local + driver_opts: + type: none + device: /var/panoptes + o: bind \ No newline at end of file From 7645c301b3a7732323540a42910e8c12a3d6a4a3 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Sun, 10 Feb 2019 09:43:15 +1100 Subject: [PATCH 017/105] Run separate container for messaging hub and expose the msg and cmd ports to host --- resources/docker_files/docker-compose.yml | 29 ++++++++--------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/resources/docker_files/docker-compose.yml b/resources/docker_files/docker-compose.yml index b69692a48..d8a9d79b6 100644 --- a/resources/docker_files/docker-compose.yml +++ b/resources/docker_files/docker-compose.yml @@ -4,27 +4,18 @@ services: image: gcr.io/panoptes-survey/pocs-base privileged: true env_file: env_file + ports: + - target: 6500 + published: 6500 + protocol: tcp + mode: host + - target: 6510 + published: 6510 + protocol: tcp + mode: host volumes: - pandir:/var/panoptes command: bash -c "python3 $POCS/scripts/run_messaging_hub.py --from_config" - weather: - image: gcr.io/panoptes-survey/pocs-base - privileged: true - env_file: env_file - depends_on: - - "messaging" - volumes: - - pandir:/var/panoptes - command: bash -c "python3 $POCS/scripts/docker/run_weather.py" - peas: - image: gcr.io/panoptes-survey/pocs-base - privileged: true - env_file: env_file - depends_on: - - "messaging" - volumes: - - pandir:/var/panoptes - command: bash -c "python3 $POCS/scripts/docker/run_peas.py" paws: image: gcr.io/panoptes-survey/paws privileged: true @@ -46,7 +37,7 @@ services: - "messaging" volumes: - pandir:/var/panoptes - # No-op for now - connect via "docker exec -it panoptes_pocs_1 jupyter-console" + # No-op to keep machine running command: tail -f /dev/null volumes: pandir: From e3a74b1354a4ebe8c936b69e32cb9a8a6bac37a1 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Sun, 10 Feb 2019 10:03:23 +1100 Subject: [PATCH 018/105] Removing old docker files, cleaing up startup scripts --- bin/peas | 3 + bin/pocs | 3 + scripts/docker/build-panuser-image.py | 186 ------------------------ scripts/docker/run_peas.py | 36 ----- scripts/docker/run_pocs.py | 21 --- scripts/docker/run_weather.py | 40 ----- bin/peas_shell => scripts/peas_shell.py | 2 +- bin/pocs_shell => scripts/pocs_shell.py | 0 8 files changed, 7 insertions(+), 284 deletions(-) create mode 100755 bin/peas create mode 100755 bin/pocs delete mode 100755 scripts/docker/build-panuser-image.py delete mode 100755 scripts/docker/run_peas.py delete mode 100755 scripts/docker/run_pocs.py delete mode 100755 scripts/docker/run_weather.py rename bin/peas_shell => scripts/peas_shell.py (99%) rename bin/pocs_shell => scripts/pocs_shell.py (100%) diff --git a/bin/peas b/bin/peas new file mode 100755 index 000000000..062416ab0 --- /dev/null +++ b/bin/peas @@ -0,0 +1,3 @@ +#!/bin/bash -e + +docker exec -it panoptes_pocs_1 bash -c "$POCS/scripts/peas_shell" \ No newline at end of file diff --git a/bin/pocs b/bin/pocs new file mode 100755 index 000000000..c292b3fc1 --- /dev/null +++ b/bin/pocs @@ -0,0 +1,3 @@ +#!/bin/bash -e + +docker exec -it panoptes_pocs_1 bash -c "$POCS/scripts/pocs_shell" \ No newline at end of file diff --git a/scripts/docker/build-panuser-image.py b/scripts/docker/build-panuser-image.py deleted file mode 100755 index 4312a5758..000000000 --- a/scripts/docker/build-panuser-image.py +++ /dev/null @@ -1,186 +0,0 @@ -#!/usr/bin/env python3 -# -# Script to create a docker image with the specified user and group -# created and set to the USER; also makes that user name the value of -# the PANUSER environment. This is typically either the current -# user (developer or CI case) or a user called panoptes for automated -# execution on a unit. The user must exist on the host computer. - -import argparse -import grp -import os -import pwd -import shutil -import subprocess -import sys -import tempfile - - -def generate_dockerfile(base_image='ubuntu:18.04', - user_name='panoptes', - user_id=None, - group_name='panoptes', - group_id=None, - pandir=None, - command='/bin/bash'): - if not isinstance(user_id, int): - user_id = '' - if not isinstance(group_id, int): - group_id = '' - - dockerfile = f""" -FROM {base_image} - -LABEL author="Developers for PANOPTES project" -LABEL url="https://github.com/panoptes/POCS" - -ENV LANG C.UTF-8 -ENV LC_ALL C.UTF-8 -ENV PANUSER {user_name} - -# Create user PANUSER ({user_name}). - -COPY create-panoptes-user.sh /workdir/ -RUN PANUSER={user_name} PANUSER_ID={user_id} \\ - PANGROUP={group_name or ''} PANGROUP_ID={group_id} \\ - NO_PASSWORD=true \\ - /workdir/create-panoptes-user.sh && \\ - rm -rf /workdir -""" - if pandir: - dockerfile += f""" - -# Set PANDIR and create the directory, owned by PANUSER. - -ENV PANDIR {pandir} -RUN mkdir -p {pandir} -RUN chown -R {user_name}:{group_name or user_name} {pandir} -WORKDIR {pandir} -""" - dockerfile += f""" - -USER {user_name} -CMD {command} -""" - return dockerfile - - -def getusername(): - uid = os.getuid() - entry = pwd.getpwuid(uid) - return entry.pw_name - - -def main(): - parser = argparse.ArgumentParser(description='Create docker image for the specified user.') - parser.add_argument( - '--user', - dest='user_name', - type=str, - default=(os.environ.get('PANUSER') or getusername()), - help=('Name of the PANUSER in the image. ' - 'If specified, the user must exist.')) - parser.add_argument( - '--group', - dest='group_name', - type=str, - help=('Primary group of the user PANUSER in the image. ' - 'If specified, the group must exist.')) - parser.add_argument( - '--pandir', - dest='pandir', - type=str, - default=(os.environ.get('PANDIR') or '/var/panoptes'), - help='Root directory of PANOPTES software and data.') - parser.add_argument( - '--base-image', - dest='base_image', - type=str, - default='ubuntu:18.04', - help='Base image from which to create this image.') - parser.add_argument( - '--command', - dest='command', - type=str, - default='/bin/bash', - help='Command to run when the image is launched in a container.') - parser.add_argument('--tag', dest='tag', type=str, help='Tag to apply to the created image.') - parser.add_argument('--verbose', '-v', action='store_true') - - args = parser.parse_args() - - def arg_error(msg): - print(msg, '\n', file=sys.stderr) - parser.print_help(file=sys.stderr) - sys.exit(1) - - try: - pwent = pwd.getpwnam(args.user_name) - user_name = pwent.pw_name - user_id = pwent.pw_uid - group_id = pwent.pw_gid - except KeyError: - arg_error(f'User with name "{args.user_name}" not found.') - - if args.group_name: - try: - grent = grp.getgrnam(args.group_name) - assert args.group_name == grent.gr_name - group_name = grent.gr_name - group_id = grent.gr_gid - except KeyError: - arg_error(f'Group with name "{args.group_name}" not found.') - else: - # Not putting a try-except here because a KeyError implies that - # something is wrong with the pwd and grp lookups, not with the - # user's input. - grent = grp.getgrgid(group_id) - assert group_id == grent.gr_gid - group_name = grent.gr_name - - if args.pandir: - if not os.path.isabs(args.pandir): - arg_error(f'PANOPTES directory must be an absolute path, not: {args.pandir}') - pandir = args.pandir - else: - pandir = None - - dockerfile = generate_dockerfile( - base_image=args.base_image, - user_name=user_name, - user_id=user_id, - group_name=group_name, - group_id=group_id, - pandir=pandir, - command=args.command) - if args.verbose: - print('Using this dockerfile:') - print('#' * 80) - print(dockerfile) - print('#' * 80) - - tag = args.tag or f'panuser-{user_name}' - - with tempfile.TemporaryDirectory() as tmpdir: - with open(os.path.join(tmpdir, 'Dockerfile'), 'w') as f: - f.write(dockerfile) - create_script = os.path.join(os.environ['POCS'], 'scripts', 'install', - 'create-panoptes-user.sh') - shutil.copy(create_script, tmpdir) - build_args = ['docker', 'build', '--tag', tag, tmpdir] - if args.verbose: - print('Running command:', build_args) - sys.exit(subprocess.call(build_args, timeout=3600)) - result = subprocess.run( - build_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=3600) - - if result.returncode == 0: - os.system(f'docker image ls {tag}') - sys.exit(0) - print('docker build failed with status', result.returncode) - print('docker build output:') - print(result.stdout) - - -if __name__ == '__main__': - main() diff --git a/scripts/docker/run_peas.py b/scripts/docker/run_peas.py deleted file mode 100755 index 9c1f6d0a1..000000000 --- a/scripts/docker/run_peas.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python3 - -from peas.sensors import ArduinoSerialMonitor - -from pocs.utils.config import load_config -from pocs.utils import CountdownTimer -from pocs.utils import error - - -def main(config, loop_delay=5): - try: - print("Loading Arduino Serial Monitor") - environment = ArduinoSerialMonitor(auto_detect=True) - if environment is None: - print(f'Arduino Serial Monitor not set up properly, check ports') - return - print("Environmental sensors set up") - except error.PanError as e: - print('Problem setting up Arduino serial montiors: {}'.format(e)) - - timer = CountdownTimer(loop_delay) - - # Loop forever. - print(f'Starting environmental sensors loop with {loop_delay}s delay') - while True: - timer.restart() - environment.capture(store_result=True, send_message=True) - - timer.sleep() - - # TODO: listen for messages and react to them - - -if __name__ == '__main__': - config = load_config('peas') - main(config) diff --git a/scripts/docker/run_pocs.py b/scripts/docker/run_pocs.py deleted file mode 100755 index 15605a36c..000000000 --- a/scripts/docker/run_pocs.py +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env python3 - -from pocs.observatory import Observatory -from pocs.core import POCS -from pocs.utils import error - - -def main(simulator='all'): - try: - print("Setting up POCS") - observatory = Observatory(simulator=simulator) - pocs = POCS(observatory, messaging=True) - pocs.initialize() - print("POCS initialized") - # pocs.run() - except error.PanError as e: - print('Problem setting up POCS: {}'.format(e)) - - -if __name__ == '__main__': - main() diff --git a/scripts/docker/run_weather.py b/scripts/docker/run_weather.py deleted file mode 100755 index 23fa4cc66..000000000 --- a/scripts/docker/run_weather.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python3 - -from peas.weather import AAGCloudSensor - -from pocs.utils.config import load_config -from pocs.utils import CountdownTimer -from pocs.utils import error - - -def main(config, loop_delay=60): - try: - try: - port = config['weather']['aag_cloud']['serial_port'] - except KeyError: - port = '/dev/ttyUSB0' - print("Loading AAG Cloud Sensor on {}".format(port)) - weather = AAGCloudSensor(serial_address=port, store_result=True) - if weather.AAG is None: - print(f'AAG not set up properly, check port') - return - print("Weather station set up") - except error.PanError as e: - print('Problem setting up PEAS: {}'.format(e)) - - timer = CountdownTimer(loop_delay) - - # Loop forever. - print(f'Starting weather station loop with {loop_delay}s delay') - while True: - timer.restart() - weather.capture(store_result=True, send_message=True) - - timer.sleep() - - # TODO: listen for messages and react to them - - -if __name__ == '__main__': - config = load_config('peas') - main(config) diff --git a/bin/peas_shell b/scripts/peas_shell.py similarity index 99% rename from bin/peas_shell rename to scripts/peas_shell.py index 704f12c75..e2679a89b 100755 --- a/bin/peas_shell +++ b/scripts/peas_shell.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import cmd import datetime import os diff --git a/bin/pocs_shell b/scripts/pocs_shell.py similarity index 100% rename from bin/pocs_shell rename to scripts/pocs_shell.py From 3d6fbd03046534b3d835eae015fdd8b577a4e6de Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Sun, 10 Feb 2019 10:06:40 +1100 Subject: [PATCH 019/105] Reverting the startup files as otherwise logger is broken for some reason --- bin/peas | 3 --- scripts/peas_shell.py => bin/peas_shell | 0 bin/pocs | 3 --- scripts/pocs_shell.py => bin/pocs_shell | 0 4 files changed, 6 deletions(-) delete mode 100755 bin/peas rename scripts/peas_shell.py => bin/peas_shell (100%) delete mode 100755 bin/pocs rename scripts/pocs_shell.py => bin/pocs_shell (100%) diff --git a/bin/peas b/bin/peas deleted file mode 100755 index 062416ab0..000000000 --- a/bin/peas +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -e - -docker exec -it panoptes_pocs_1 bash -c "$POCS/scripts/peas_shell" \ No newline at end of file diff --git a/scripts/peas_shell.py b/bin/peas_shell similarity index 100% rename from scripts/peas_shell.py rename to bin/peas_shell diff --git a/bin/pocs b/bin/pocs deleted file mode 100755 index c292b3fc1..000000000 --- a/bin/pocs +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -e - -docker exec -it panoptes_pocs_1 bash -c "$POCS/scripts/pocs_shell" \ No newline at end of file diff --git a/scripts/pocs_shell.py b/bin/pocs_shell similarity index 100% rename from scripts/pocs_shell.py rename to bin/pocs_shell From e3dd83cafc6044442ae8bd76006b5b1b0284aad9 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Sun, 10 Feb 2019 10:18:41 +1100 Subject: [PATCH 020/105] Cleaning bin files --- bin/peas | 3 +++ bin/pocs | 3 +++ {bin => scripts}/peas_shell | 0 {bin => scripts}/pocs_shell | 0 4 files changed, 6 insertions(+) create mode 100755 bin/peas create mode 100755 bin/pocs rename {bin => scripts}/peas_shell (100%) rename {bin => scripts}/pocs_shell (100%) diff --git a/bin/peas b/bin/peas new file mode 100755 index 000000000..5d619a0a2 --- /dev/null +++ b/bin/peas @@ -0,0 +1,3 @@ +#!/bin/bash + +docker exec -it panoptes_pocs_1 $POCS/scripts/peas_shell \ No newline at end of file diff --git a/bin/pocs b/bin/pocs new file mode 100755 index 000000000..d5f1dbd7f --- /dev/null +++ b/bin/pocs @@ -0,0 +1,3 @@ +#!/bin/bash + +docker exec -it panoptes_pocs_1 $POCS/scripts/pocs_shell \ No newline at end of file diff --git a/bin/peas_shell b/scripts/peas_shell similarity index 100% rename from bin/peas_shell rename to scripts/peas_shell diff --git a/bin/pocs_shell b/scripts/pocs_shell similarity index 100% rename from bin/pocs_shell rename to scripts/pocs_shell From 8d13d3bb600c397bd56f63ed0de38df9c3a75b1f Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Sun, 10 Feb 2019 10:53:00 +1100 Subject: [PATCH 021/105] First cut at a basic install script for docker items --- scripts/install/install-pocs.sh | 49 +++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 scripts/install/install-pocs.sh diff --git a/scripts/install/install-pocs.sh b/scripts/install/install-pocs.sh new file mode 100644 index 000000000..008d6cae8 --- /dev/null +++ b/scripts/install/install-pocs.sh @@ -0,0 +1,49 @@ +#!/bin/bash -e + +KEY_FILE=${1} +PANDIR=${2:-/var/panoptes} + +echo "Starting fresh install at `date`" > install.log + +echo "Using ${KEY_FILE}" + +echo "Creating ${PANDIR}" +mkdir -p ${PANDIR}/.key + +echo "Moving ${KEY_FILE} to hidden directory ${PANDIR}/.key" +mv ${KEY_FILE} ${PANDIR}/.key + +echo "Updating computer..." + +# Create environment variable for correct distribution +export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" + +# Add the Cloud SDK distribution URI as a package source +echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list + +# Import the Google Cloud Platform public key +curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - + +sudo apt update &>> install.log +sudo apt install -y docker google-cloud-sdk &>> install.log + +# Install miniconda and docker-compose +wget https://repo.continuum.io/miniconda/Miniconda3-3.7.0-Linux-x86_64.sh -O ~/miniconda.sh &>> install.log +bash ~/miniconda.sh -b -p ${PANDIR}/miniconda +export PATH="${PANDIR}/miniconda/bin:$PATH" +source ${PANDIR}/miniconda/bin/activate +rm ~/miniconda.sh + +conda create -n panoptes python=3.7 docker-compose &>> install.log + +conda activate panoptes +conda install -y docker-compose + +echo "Authenticating with google" +gcloud auth activate-service-account --key-file ${PANDIR}/.key/${KEY_FILE} +gcloud auth configure-docker + +echo "Pulling POCS files from cloud" +echo "WARNING: This is a large file that can take a long time!" +docker pull gcr.io/panoptes-survey/pocs-base +docker pull gcr.io/panoptes-survey/paws \ No newline at end of file From 40300ca08d5f12971d03c1ca58a4615f191f9471 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Sun, 10 Feb 2019 15:51:45 +1100 Subject: [PATCH 022/105] Updates to install file for docker. Still need to work out a few kinks, such as how to run easily --- scripts/install/install-pocs.sh | 48 ++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/scripts/install/install-pocs.sh b/scripts/install/install-pocs.sh index 008d6cae8..aad51e132 100644 --- a/scripts/install/install-pocs.sh +++ b/scripts/install/install-pocs.sh @@ -8,7 +8,11 @@ echo "Starting fresh install at `date`" > install.log echo "Using ${KEY_FILE}" echo "Creating ${PANDIR}" -mkdir -p ${PANDIR}/.key +sudo mkdir -p ${PANDIR}/.key + +sudo chown -R panoptes:panoptes ${PANDIR} +# Make sure time is correct or gcloud won't authenticate +sudo timedatectl set-ntp on echo "Moving ${KEY_FILE} to hidden directory ${PANDIR}/.key" mv ${KEY_FILE} ${PANDIR}/.key @@ -22,28 +26,46 @@ export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list # Import the Google Cloud Platform public key -curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - +wget -q -O- https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - &>> install.log + +# Get a copy of POCS +wget -q https://github.com/panoptes/POCS/archive/develop.zip +mv develop.zip ${PANDIR} && cd /var/panoptes +unzip develop.zip &>> install.log +mv POCS-develop POCS sudo apt update &>> install.log -sudo apt install -y docker google-cloud-sdk &>> install.log +sudo apt install -y docker.io &>> install.log +sudo adduser panoptes docker &>> install.log # Install miniconda and docker-compose -wget https://repo.continuum.io/miniconda/Miniconda3-3.7.0-Linux-x86_64.sh -O ~/miniconda.sh &>> install.log -bash ~/miniconda.sh -b -p ${PANDIR}/miniconda -export PATH="${PANDIR}/miniconda/bin:$PATH" -source ${PANDIR}/miniconda/bin/activate +wget -q https://repo.continuum.io/miniconda/Miniconda3-3.7.0-Linux-x86_64.sh -O ~/miniconda.sh &>> install.log +bash ~/miniconda.sh -b -p ${PANDIR}/miniconda &>> install.log rm ~/miniconda.sh -conda create -n panoptes python=3.7 docker-compose &>> install.log +# Add path to user's shell +echo "export PATH="/var/panoptes/miniconda/bin:$PATH"" >> ~/.bashrc +echo "export PANDIR=/var/panoptes" >> ~/.bashrc +echo "export POCS=/var/panoptes/POCS" >> ~/.bashrc + +# Add for this session +export PATH="/var/panoptes/miniconda/bin:$PATH" +#source ${PANDIR}/miniconda/bin/activate -conda activate panoptes -conda install -y docker-compose +echo "Creating new python environment for panoptes" +conda create -n panoptes --yes python=3 &>> install.log + +source activate panoptes &>> install.log +conda install --yes --quiet -c conda-forge pip google-cloud-sdk &>> install.log +pip install --quiet docker-compose &>> install.log echo "Authenticating with google" gcloud auth activate-service-account --key-file ${PANDIR}/.key/${KEY_FILE} -gcloud auth configure-docker +gcloud auth configure-docker --quiet echo "Pulling POCS files from cloud" echo "WARNING: This is a large file that can take a long time!" -docker pull gcr.io/panoptes-survey/pocs-base -docker pull gcr.io/panoptes-survey/paws \ No newline at end of file +sudo docker pull gcr.io/panoptes-survey/pocs-base +sudo docker pull gcr.io/panoptes-survey/paws + +echo "All done! Please reboot your system." \ No newline at end of file From 8ea84d022b04c6adbd897c7ee7d52a070a20b635 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Tue, 26 Feb 2019 07:44:49 +1100 Subject: [PATCH 023/105] * Using Ubuntu 18.10 in Docker images * FITS solving uses pipe * Removing unused packages --- Dockerfile | 3 ++- pocs/utils/images/fits.py | 2 +- scripts/install/apt-packages-list.txt | 7 ++----- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2a8ffc3ac..4fd20321d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # PANOPTES development container -FROM ubuntu:18.04 as build-env +FROM ubuntu:18.10 as build-env MAINTAINER Developers for PANOPTES project ARG pan_dir=/var/panoptes @@ -13,6 +13,7 @@ ENV PANLOG $PANDIR/logs ENV POCS $PANDIR/POCS ENV PANUSER root ENV SOLVE_FIELD=/usr/bin/solve-field +ENV DEBIAN_FRONTEND=noninteractive COPY . $POCS diff --git a/pocs/utils/images/fits.py b/pocs/utils/images/fits.py index ea6eeef52..c5513db5f 100644 --- a/pocs/utils/images/fits.py +++ b/pocs/utils/images/fits.py @@ -71,7 +71,7 @@ def solve_field(fname, timeout=15, solve_opts=None, **kwargs): try: proc = subprocess.Popen(cmd, universal_newlines=True, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + stdout=subprocess.PIPE, stderr=subprocess.PIPE) except OSError as e: raise error.InvalidCommand( "Can't send command to solve_field.sh: {} \t {}".format(e, cmd)) diff --git a/scripts/install/apt-packages-list.txt b/scripts/install/apt-packages-list.txt index a749e0f70..dad676e3b 100644 --- a/scripts/install/apt-packages-list.txt +++ b/scripts/install/apt-packages-list.txt @@ -55,11 +55,6 @@ gcc # Improves interaction with pocs_shell (via readline). libncurses5-dev -# These support the Python package "Shapely", used in PIAA (called from POCS). -libgeos-3.* -libgeos-c1v5 -libgeos-dev - # byobu and tmux provide a means to run multiple interactive shells in a way # that survives disconnection. byobu @@ -68,3 +63,5 @@ tmux # GNU coreutils provides basic commands that we depend on (echo, ls), and is # almost certainly already installed, but let's make sure. coreutils + +vim-nox \ No newline at end of file From 73b26616c862c5703552189595a6c46c01dd14ac Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Mon, 11 Mar 2019 07:51:53 +1100 Subject: [PATCH 024/105] Updates to uber-install script --- scripts/install/install-pocs.sh | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/scripts/install/install-pocs.sh b/scripts/install/install-pocs.sh index aad51e132..538ad1113 100644 --- a/scripts/install/install-pocs.sh +++ b/scripts/install/install-pocs.sh @@ -28,12 +28,6 @@ echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | sudo tee # Import the Google Cloud Platform public key wget -q -O- https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - &>> install.log -# Get a copy of POCS -wget -q https://github.com/panoptes/POCS/archive/develop.zip -mv develop.zip ${PANDIR} && cd /var/panoptes -unzip develop.zip &>> install.log -mv POCS-develop POCS - sudo apt update &>> install.log sudo apt install -y docker.io &>> install.log sudo adduser panoptes docker &>> install.log @@ -48,6 +42,20 @@ echo "export PATH="/var/panoptes/miniconda/bin:$PATH"" >> ~/.bashrc echo "export PANDIR=/var/panoptes" >> ~/.bashrc echo "export POCS=/var/panoptes/POCS" >> ~/.bashrc +cd $HOME +wget -q https://raw.githubusercontent.com/panoptes/POCS/40300ca08d5f12971d03c1ca58a4615f191f9471/resources/docker_files/docker-compose.yml +wget -q https://raw.githubusercontent.com/panoptes/POCS/40300ca08d5f12971d03c1ca58a4615f191f9471/resources/docker_files/env_file +wget -q https://raw.githubusercontent.com/panoptes/POCS/40300ca08d5f12971d03c1ca58a4615f191f9471/scripts/start_pocs_docker.sh +chmod +x start_pocs_docker.sh + +cat <> $HOME/Desktop/POCS.desktop +[Desktop Entry] +Exec=bash -c "cd $HOME && source activate panoptes && ./start_pocs_docker.sh" +Name=pocs +Terminal=true +Type=Application +EOT + # Add for this session export PATH="/var/panoptes/miniconda/bin:$PATH" #source ${PANDIR}/miniconda/bin/activate @@ -61,7 +69,7 @@ pip install --quiet docker-compose &>> install.log echo "Authenticating with google" gcloud auth activate-service-account --key-file ${PANDIR}/.key/${KEY_FILE} -gcloud auth configure-docker --quiet +gcloud auth configure-docker --quiet &>> install.log echo "Pulling POCS files from cloud" echo "WARNING: This is a large file that can take a long time!" From b601974fb9d9eaa38bb259bee2ffe16a0b4fc8c6 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Fri, 29 Mar 2019 12:23:05 +1100 Subject: [PATCH 025/105] I have no idea why this suddenly needs a change. --- pocs/tests/utils/test_fits_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pocs/tests/utils/test_fits_utils.py b/pocs/tests/utils/test_fits_utils.py index 6268c92bb..dab1ada7f 100644 --- a/pocs/tests/utils/test_fits_utils.py +++ b/pocs/tests/utils/test_fits_utils.py @@ -64,4 +64,4 @@ def test_solve_options(solved_fits_file): def test_solve_bad_field(solved_fits_file): proc = fits_utils.solve_field('Foo', verbose=True) outs, errs = proc.communicate() - assert 'ERROR' in outs + assert 'ERROR' in errs From adc68486a49a9a5be6d0da5f4aa4b2042115b9c4 Mon Sep 17 00:00:00 2001 From: Wilfred Gee Date: Tue, 2 Apr 2019 21:29:54 +1100 Subject: [PATCH 026/105] Inherit from the `gcr.io/panoptes-survey/panoptes-utils` image but build a few additional items. --- Dockerfile | 43 +++++++++---------------------------------- 1 file changed, 9 insertions(+), 34 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4fd20321d..bc675bc5a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,42 +1,17 @@ -# PANOPTES development container - -FROM ubuntu:18.10 as build-env +FROM gcr.io/panoptes-survey/panoptes-utils MAINTAINER Developers for PANOPTES project -ARG pan_dir=/var/panoptes - -ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 -ENV ENV /root/.bashrc -ENV SHELL /bin/bash -ENV PANDIR $pan_dir -ENV PANLOG $PANDIR/logs -ENV POCS $PANDIR/POCS -ENV PANUSER root -ENV SOLVE_FIELD=/usr/bin/solve-field -ENV DEBIAN_FRONTEND=noninteractive - -COPY . $POCS - -# Note that pocs-base has the default ubuntu environment, so -# we need to specify python3 so we don't get python2 +ENV PANDIR /var/panoptes -# Use "bash" as replacement for "sh" -# Note: I don't think this is the preferred way to do this anymore -RUN rm /bin/sh && ln -s /bin/bash /bin/sh \ - && apt-get update \ +COPY . ${POCS} +RUN apt-get update \ && apt-get --yes install \ - `cut '-d#' -f1 $POCS/scripts/install/apt-packages-list.txt | sort | uniq` \ - astrometry.net \ - python3-pip \ - wget \ + gcc \ + pkg-config \ + libncurses5-dev \ && rm -rf /var/lib/apt/lists/* \ - && mkdir -p $POCS \ - && mkdir -p $PANLOG \ - && mkdir -p ${PANDIR}/astrometry/data \ - && echo "add_path /var/panoptes/astrometry/data" >> /etc/astrometry.cfg \ - && cd $POCS \ - && pip3 install -Ur requirements.txt \ - && pip3 install -e . + && cd ${POCS} \ + && pip3 install --no-cache-dir -r requirements.txt WORKDIR ${POCS} From bd0287a79f457dc4bcf43c015e961254cf8111d3 Mon Sep 17 00:00:00 2001 From: Wilfred Gee Date: Tue, 2 Apr 2019 21:32:32 +1100 Subject: [PATCH 027/105] Clean up some dependencies --- requirements.txt | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/requirements.txt b/requirements.txt index 9051e5dae..f4fe4fe52 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,26 +1,18 @@ astroplan astropy >= 3.0.0 -codecov -coveralls dateparser -gcloud -google-cloud-storage jupyter-console jupyterlab -matplotlib >= 2.0.0,<3.0.0 +matplotlib > 3.0.0 mocket numpy >= 1.12.1, !=1.15.3 -pycodestyle == 2.3.1 -pymongo >= 3.2.2 -pyserial >= 3.1.1 pytest >= 3.4.0 python_dateutil >= 2.5.3 -PyYAML >= 3.11 +PyYAML >= 5.1 pyzmq >= 15.3.0 readline requests scikit-image scipy >= 0.17.1 transitions >= 0.4.0 -tweepy wcsaxes \ No newline at end of file From 580a409d66bdaefb3b8e0e9ca7c4b2490bb07094 Mon Sep 17 00:00:00 2001 From: Wilfred Gee Date: Tue, 2 Apr 2019 21:35:42 +1100 Subject: [PATCH 028/105] Run the tests from the docker container. --- .travis.yml | 96 +++--------------------------------- scripts/testing/run_tests.sh | 7 +++ 2 files changed, 13 insertions(+), 90 deletions(-) create mode 100755 scripts/testing/run_tests.sh diff --git a/.travis.yml b/.travis.yml index d2841278a..a9ad846d0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,93 +1,9 @@ -dist: trusty -sudo: required -language: python +language: minimal services: - - mongodb -python: - - "3.6" -env: - - PANDIR=$HOME POCS=$TRAVIS_BUILD_DIR PANUSER=$USER ARDUINO_VERSION=1.8.1 + - docker before_install: - - mkdir -p $PANDIR/logs - - mkdir -p $PANDIR/astrometry/data - - ln -s $POCS $PANDIR/POCS - - pip install -U pip - - pip install coveralls - - # Install cfitsio - - cd $PANDIR - - wget http://heasarc.gsfc.nasa.gov/FTP/software/fitsio/c/cfitsio_latest.tar.gz - - tar zxf cfitsio_latest.tar.gz - - cd cfitsio - - ./configure - - make - - make fpack - - make funpack - - sudo make install - - # Install arudino files - - cd $PANDIR - - export DISPLAY=:1.0 - - export - - wget http://downloads.arduino.cc/arduino-${ARDUINO_VERSION}-linux64.tar.xz - - tar xf arduino-${ARDUINO_VERSION}-linux64.tar.xz - - sudo mv arduino-${ARDUINO_VERSION} /usr/local/share/arduino - - sudo ln -s /usr/local/share/arduino/arduino /usr/local/bin/arduino -addons: - apt: - packages: - - gphoto2 - - libcairo2-dev - - libnetpbm10-dev - - netpbm - - libpng12-dev - - libjpeg-dev - - python-numpy - - python-pyfits - - python-dev - - zlib1g-dev - - libbz2-dev - - swig - - cfitsio-dev -install: - - cd $PANDIR - - # Install astrometry.net - - wget https://github.com/dstndstn/astrometry.net/releases/download/0.72/astrometry.net-0.72.tar.gz - - tar zxvf astrometry.net-0.72.tar.gz - - cd astrometry.net-0.72 && make && make py && make install INSTALL_DIR=$PANDIR/astrometry - - echo 'add_path $PANDIR/astrometry/data' | sudo tee --append $PANDIR/astrometry/etc/astrometry.cfg - - # Install miniconda - - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; - - bash miniconda.sh -b -p $HOME/miniconda - - export PATH="$HOME/miniconda/bin:$HOME/cfitsio/bin:$PANDIR/astrometry/bin:$PATH" - - hash -r - - - conda config --set always_yes yes --set changeps1 no - - conda update -q conda - - conda info -a # Useful for debugging any issues with conda - - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION - - source activate test-environment - - # install POCS and requirements - - cd $POCS - - pip install -r requirements.txt - - pip install -r docs/requirements.txt - - pip install -e . - - python pocs/utils/data.py --folder $PANDIR/astrometry/data + - ci_env=`bash <(curl -s https://codecov.io/env)` + - docker pull gcr.io/panoptes-survey/pocs +install: true # skips install script: - - export BOARD="arduino:avr:micro" - - arduino --verify --board $BOARD resources/arduino_files/camera_board/camera_board.ino - - arduino --verify --board $BOARD resources/arduino_files/power_board/power_board.ino - - arduino --verify --board $BOARD resources/arduino_files/telemetry_board/telemetry_board.ino - - export PYTHONPATH="$PYTHONPATH:$POCS/scripts/coverage" - - export COVERAGE_PROCESS_START=.coveragerc - - coverage run $(which pytest) -v --test-databases all - - coverage combine -cache: - pip: true - directories: - - $PANDIR/astrometry/ -after_success: - - bash <(curl -s https://codecov.io/bash) + - docker run -it $ci_env gcr.io/panoptes-survey/pocs scripts/testing/run_tests.sh \ No newline at end of file diff --git a/scripts/testing/run_tests.sh b/scripts/testing/run_tests.sh new file mode 100755 index 000000000..e7f6eada1 --- /dev/null +++ b/scripts/testing/run_tests.sh @@ -0,0 +1,7 @@ +#!/bin/bash -e + +export PYTHONPATH="$PYTHONPATH:$PANDIR/POCS/scripts/testing/coverage" +export COVERAGE_PROCESS_START=.coveragerc +coverage run $(which pytest) -v --test-databases all +coverage combine +bash <(curl -s https://codecov.io/bash) \ No newline at end of file From d6920de7b77e9d2400ae42a196f74c62dd176165 Mon Sep 17 00:00:00 2001 From: Wilfred Gee Date: Tue, 2 Apr 2019 21:43:38 +1100 Subject: [PATCH 029/105] Travis pulls latest from github but still tests against container --- .travis.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a9ad846d0..dfdbdc547 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,11 @@ services: before_install: - ci_env=`bash <(curl -s https://codecov.io/env)` - docker pull gcr.io/panoptes-survey/pocs +env: + - PANDIR=$HOME POCS=$TRAVIS_BUILD_DIR PANUSER=$USER ARDUINO_VERSION=1.8.1 install: true # skips install script: - - docker run -it $ci_env gcr.io/panoptes-survey/pocs scripts/testing/run_tests.sh \ No newline at end of file + - cd ${POCS} + - pip install -r requirements.txt + - pip install -e . + - docker run -it -v /var/panotpes:/var/panoptes $ci_env gcr.io/panoptes-survey/pocs scripts/testing/run_tests.sh \ No newline at end of file From ac53b768a653f81dc2497d690ce753dcb5d61dcb Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Wed, 3 Apr 2019 07:54:38 +1100 Subject: [PATCH 030/105] Switching language back to `python` to avoid `SNIMissingWarning` warning --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index dfdbdc547..53c4abe70 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,4 @@ -language: minimal +language: python services: - docker before_install: From 5626e2193388ffc75f08646dd5f6aef5f8b453c2 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Wed, 3 Apr 2019 08:00:18 +1100 Subject: [PATCH 031/105] Just pass the build directory as the shared volume. --- .travis.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 53c4abe70..15b79bc28 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,11 +4,6 @@ services: before_install: - ci_env=`bash <(curl -s https://codecov.io/env)` - docker pull gcr.io/panoptes-survey/pocs -env: - - PANDIR=$HOME POCS=$TRAVIS_BUILD_DIR PANUSER=$USER ARDUINO_VERSION=1.8.1 install: true # skips install script: - - cd ${POCS} - - pip install -r requirements.txt - - pip install -e . - - docker run -it -v /var/panotpes:/var/panoptes $ci_env gcr.io/panoptes-survey/pocs scripts/testing/run_tests.sh \ No newline at end of file + - docker run -it -v $TRAVIS_BUILD_DIR:/var/panoptes/POCS $ci_env gcr.io/panoptes-survey/pocs scripts/testing/run_tests.sh \ No newline at end of file From 343d95d4b559f74cdd669194cd9622b124194bef Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Wed, 3 Apr 2019 08:45:37 +1100 Subject: [PATCH 032/105] * Adding gphoto2 * Using cloudbuild config for building * Fix path for testing coverage --- Dockerfile | 4 ++++ cloudbuild.yaml | 21 +++++++++++++++++++++ scripts/testing/run_tests.sh | 2 +- 3 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 cloudbuild.yaml diff --git a/Dockerfile b/Dockerfile index bc675bc5a..165d4e2f5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,11 @@ RUN apt-get update \ gcc \ pkg-config \ libncurses5-dev \ + && wget https://raw.githubusercontent.com/gonzalo/gphoto2-updater/master/gphoto2-updater.sh \ + && chmod +x gphoto2-updater.sh \ + && ./gphoto2-updater.sh --stable \ && rm -rf /var/lib/apt/lists/* \ + && rm gphoto2-updater.sh \ && cd ${POCS} \ && pip3 install --no-cache-dir -r requirements.txt diff --git a/cloudbuild.yaml b/cloudbuild.yaml new file mode 100644 index 000000000..03e85263e --- /dev/null +++ b/cloudbuild.yaml @@ -0,0 +1,21 @@ +steps: +# Build +- name: 'gcr.io/cloud-builders/docker' + id: 'main-build' + args: + - 'build' + - '--tag=gcr.io/${PROJECT_ID}/pocs + :latest' + - '.' + waitFor: ['-'] + +# Push +- name: 'gcr.io/cloud-builders/docker' + id: 'push-latest' + args: + - 'push' + - 'gcr.io/${PROJECT_ID}/pocs:latest' + waitFor: ['main-build'] + +images: + - 'gcr.io/${PROJECT_ID}/pocs:latest' diff --git a/scripts/testing/run_tests.sh b/scripts/testing/run_tests.sh index e7f6eada1..9539e2ea2 100755 --- a/scripts/testing/run_tests.sh +++ b/scripts/testing/run_tests.sh @@ -1,6 +1,6 @@ #!/bin/bash -e -export PYTHONPATH="$PYTHONPATH:$PANDIR/POCS/scripts/testing/coverage" +export PYTHONPATH="$PYTHONPATH:$PANDIR/POCS/scripts/coverage" export COVERAGE_PROCESS_START=.coveragerc coverage run $(which pytest) -v --test-databases all coverage combine From 050d62a39028796080dd56f38e728ed41fee5bb1 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Wed, 3 Apr 2019 10:27:57 +1100 Subject: [PATCH 033/105] Locking pycodestyle at specific version. See #515. --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f4fe4fe52..b6d54bf80 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,6 +6,7 @@ jupyterlab matplotlib > 3.0.0 mocket numpy >= 1.12.1, !=1.15.3 +pycodestyle == 2.3.1 pytest >= 3.4.0 python_dateutil >= 2.5.3 PyYAML >= 5.1 @@ -15,4 +16,4 @@ requests scikit-image scipy >= 0.17.1 transitions >= 0.4.0 -wcsaxes \ No newline at end of file +wcsaxes From a91736568a2d72d1c7ff266522e24e5586ed11f2 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Thu, 4 Apr 2019 10:34:09 +1100 Subject: [PATCH 034/105] Update container building script. --- cloudbuild.yaml | 3 +-- scripts/build_container.sh | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) create mode 100755 scripts/build_container.sh diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 03e85263e..1d9dc7503 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -4,8 +4,7 @@ steps: id: 'main-build' args: - 'build' - - '--tag=gcr.io/${PROJECT_ID}/pocs - :latest' + - '--tag=gcr.io/${PROJECT_ID}/pocs:latest' - '.' waitFor: ['-'] diff --git a/scripts/build_container.sh b/scripts/build_container.sh new file mode 100755 index 000000000..2be5e9999 --- /dev/null +++ b/scripts/build_container.sh @@ -0,0 +1 @@ +gcloud builds submit --config $POCS/cloudbuild.yaml $POCS From 7beb4ca3600153a9dfec87a076f52b8458015967 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Sat, 6 Apr 2019 13:58:19 +1100 Subject: [PATCH 035/105] Rename docker compose file --- resources/docker_files/docker-compose.yaml | 22 ++++++++++ resources/docker_files/docker-compose.yml | 48 ---------------------- 2 files changed, 22 insertions(+), 48 deletions(-) create mode 100644 resources/docker_files/docker-compose.yaml delete mode 100644 resources/docker_files/docker-compose.yml diff --git a/resources/docker_files/docker-compose.yaml b/resources/docker_files/docker-compose.yaml new file mode 100644 index 000000000..d0e43605c --- /dev/null +++ b/resources/docker_files/docker-compose.yaml @@ -0,0 +1,22 @@ +version: '3.2' +services: + pocs: + image: gcr.io/panoptes-survey/pocs + container_name: pocs + privileged: true + env_file: + - ${PANDIR}/env_file + depends_on: + - "config-server" + - "messaging" + volumes: + - pandir:/var/panoptes + # No-op to keep machine running + command: tail -f /dev/null +volumes: + pandir: + driver: local + driver_opts: + type: none + device: /var/panoptes + o: bind \ No newline at end of file diff --git a/resources/docker_files/docker-compose.yml b/resources/docker_files/docker-compose.yml deleted file mode 100644 index d8a9d79b6..000000000 --- a/resources/docker_files/docker-compose.yml +++ /dev/null @@ -1,48 +0,0 @@ -version: '3.2' -services: - messaging: - image: gcr.io/panoptes-survey/pocs-base - privileged: true - env_file: env_file - ports: - - target: 6500 - published: 6500 - protocol: tcp - mode: host - - target: 6510 - published: 6510 - protocol: tcp - mode: host - volumes: - - pandir:/var/panoptes - command: bash -c "python3 $POCS/scripts/run_messaging_hub.py --from_config" - paws: - image: gcr.io/panoptes-survey/paws - privileged: true - env_file: env_file - depends_on: - - "messaging" - ports: - - target: 8888 - published: 8888 - protocol: tcp - mode: host - volumes: - - pandir:/var/panoptes - pocs: - image: gcr.io/panoptes-survey/pocs-base - privileged: true - env_file: env_file - depends_on: - - "messaging" - volumes: - - pandir:/var/panoptes - # No-op to keep machine running - command: tail -f /dev/null -volumes: - pandir: - driver: local - driver_opts: - type: none - device: /var/panoptes - o: bind \ No newline at end of file From d8783a3e73806ca23e7281fd8931648a140f69f3 Mon Sep 17 00:00:00 2001 From: Wilfred Gee Date: Wed, 10 Apr 2019 23:36:58 +1000 Subject: [PATCH 036/105] Cast Docker Spell Level 2 * Multi-arch for POCS. * Relaxing some requirements for now to make builds quicker. --- Dockerfile | 12 ++++++-- cloudbuild.yaml | 78 ++++++++++++++++++++++++++++++++++++++++++++---- requirements.txt | 18 +++++++---- 3 files changed, 93 insertions(+), 15 deletions(-) diff --git a/Dockerfile b/Dockerfile index 165d4e2f5..f2998a4aa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,6 @@ -FROM gcr.io/panoptes-survey/panoptes-utils +ARG arch=amd64 + +FROM gcr.io/panoptes-survey/panoptes-utils:$arch MAINTAINER Developers for PANOPTES project ENV PANDIR /var/panoptes @@ -9,14 +11,18 @@ RUN apt-get update \ gcc \ pkg-config \ libncurses5-dev \ + # GPhoto2 && wget https://raw.githubusercontent.com/gonzalo/gphoto2-updater/master/gphoto2-updater.sh \ && chmod +x gphoto2-updater.sh \ && ./gphoto2-updater.sh --stable \ + # Cleanup + && apt-get clean \ && rm -rf /var/lib/apt/lists/* \ && rm gphoto2-updater.sh \ + # POCS && cd ${POCS} \ - && pip3 install --no-cache-dir -r requirements.txt + && pip install --no-cache-dir -r requirements.txt WORKDIR ${POCS} -CMD ["/bin/bash"] \ No newline at end of file +CMD ["/bin/zsh"] \ No newline at end of file diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 1d9dc7503..4b7a63017 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -1,20 +1,86 @@ steps: +# Set up multiarch support +- name: 'gcr.io/cloud-builders/docker' + id: 'register-qemu' + args: + - 'run' + - '--privileged' + - 'multiarch/qemu-user-static:register' + - '--reset' + waitFor: ['-'] + # Build +# AMD Build - name: 'gcr.io/cloud-builders/docker' - id: 'main-build' + id: 'amd64' args: - 'build' - - '--tag=gcr.io/${PROJECT_ID}/pocs:latest' + - '--build-arg=arch=amd64' + - '--tag=gcr.io/${PROJECT_ID}/pocs:amd64' - '.' - waitFor: ['-'] + waitFor: ['register-qemu'] +# ARM Build (e.g. Raspberry Pi) +- name: 'gcr.io/cloud-builders/docker' + id: 'arm32v7' + args: + - 'build' + - '-f' + - 'Dockerfile.rpi' + - '--build-arg=arch=arm32v7' + - '--tag=gcr.io/${PROJECT_ID}/pocs:arm32v7' + - '.' + waitFor: ['register-qemu'] # Push - name: 'gcr.io/cloud-builders/docker' - id: 'push-latest' + id: 'push-amd64' args: - 'push' + - 'gcr.io/${PROJECT_ID}/pocs:amd64' + waitFor: ['amd64'] +- name: 'gcr.io/cloud-builders/docker' + id: 'push-arm' + args: + - 'push' + - 'gcr.io/${PROJECT_ID}/pocs:arm32v7' + waitFor: ['arm32v7'] + +# Manifest file for multiarch +- name: 'gcr.io/cloud-builders/docker' + id: 'manifest' + env: + - 'DOCKER_CLI_EXPERIMENTAL=enabled' + args: + - 'manifest' + - 'create' - 'gcr.io/${PROJECT_ID}/pocs:latest' - waitFor: ['main-build'] + - 'gcr.io/${PROJECT_ID}/pocs:arm32v7' + - 'gcr.io/${PROJECT_ID}/pocs:amd64' + waitFor: ['push-amd64', 'push-arm'] -images: +- name: 'gcr.io/cloud-builders/docker' + id: 'annotate-manifest' + env: + - 'DOCKER_CLI_EXPERIMENTAL=enabled' + args: + - 'manifest' + - 'annotate' - 'gcr.io/${PROJECT_ID}/pocs:latest' + - 'gcr.io/${PROJECT_ID}/pocs:arm32v7' + - '--os=linux' + - '--arch=arm' + waitFor: ['manifest'] + +# Push manifest file +- name: 'gcr.io/cloud-builders/docker' + id: 'push-manifest' + env: + - 'DOCKER_CLI_EXPERIMENTAL=enabled' + args: + - 'manifest' + - 'push' + - 'gcr.io/${PROJECT_ID}/pocs:latest' + waitFor: ['annotate-manifest'] +images: + - 'gcr.io/${PROJECT_ID}/pocs:amd64' + - 'gcr.io/${PROJECT_ID}/pocs:arm32v7' diff --git a/requirements.txt b/requirements.txt index b6d54bf80..c1b1339d4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,19 +1,25 @@ astroplan -astropy >= 3.0.0 +astropy dateparser +docker-compose +Flask jupyter-console jupyterlab -matplotlib > 3.0.0 +matplotlib mocket -numpy >= 1.12.1, !=1.15.3 +numpy +psycopg2-binary pycodestyle == 2.3.1 +pyserial pytest >= 3.4.0 -python_dateutil >= 2.5.3 -PyYAML >= 5.1 -pyzmq >= 15.3.0 +python_dateutil +PyYAML >= 3.11 +pyzmq readline requests +scalpl scikit-image +scipy scipy >= 0.17.1 transitions >= 0.4.0 wcsaxes From c06ca593b1d14f8c4647be9350719f92a51307cd Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Thu, 11 Apr 2019 13:05:53 +1000 Subject: [PATCH 037/105] Trying to build for rpi and amd64 --- Dockerfile | 3 ++- cloudbuild.yaml | 2 -- requirements.txt | 7 +++---- scripts/build_container.sh | 6 +++++- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index f2998a4aa..aa86eec8f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,8 @@ MAINTAINER Developers for PANOPTES project ENV PANDIR /var/panoptes COPY . ${POCS} -RUN apt-get update \ +RUN /opt/conda/bin/conda activate panoptes-env \ + && apt-get update \ && apt-get --yes install \ gcc \ pkg-config \ diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 4b7a63017..121b6b807 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -24,8 +24,6 @@ steps: id: 'arm32v7' args: - 'build' - - '-f' - - 'Dockerfile.rpi' - '--build-arg=arch=arm32v7' - '--tag=gcr.io/${PROJECT_ID}/pocs:arm32v7' - '.' diff --git a/requirements.txt b/requirements.txt index c1b1339d4..3b3882966 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,17 +9,16 @@ matplotlib mocket numpy psycopg2-binary -pycodestyle == 2.3.1 +pycodestyle pyserial -pytest >= 3.4.0 +pytest python_dateutil -PyYAML >= 3.11 +PyYAML pyzmq readline requests scalpl scikit-image scipy -scipy >= 0.17.1 transitions >= 0.4.0 wcsaxes diff --git a/scripts/build_container.sh b/scripts/build_container.sh index 2be5e9999..7380878ab 100755 --- a/scripts/build_container.sh +++ b/scripts/build_container.sh @@ -1 +1,5 @@ -gcloud builds submit --config $POCS/cloudbuild.yaml $POCS +#!/bin/bash -e +SOURCE_DIR=${POCS} + +gcloud builds submit --timeout="5h" --config ${SOURCE_DIR}/cloudbuild.yaml --async ${SOURCE_DIR} + From c5293a02be063849b34338bb0886d3918a425cb7 Mon Sep 17 00:00:00 2001 From: Wilfred Gee Date: Thu, 11 Apr 2019 21:25:13 +1000 Subject: [PATCH 038/105] Renaming script for consistency --- scripts/{build_container.sh => build_containers.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/{build_container.sh => build_containers.sh} (100%) diff --git a/scripts/build_container.sh b/scripts/build_containers.sh similarity index 100% rename from scripts/build_container.sh rename to scripts/build_containers.sh From f6cf88e152a9f9a6fe9ad6f736dfe8e56ccc4826 Mon Sep 17 00:00:00 2001 From: Wilfred Gee Date: Thu, 11 Apr 2019 21:26:44 +1000 Subject: [PATCH 039/105] Fixnig install of gphoto2 and using full path for pip --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index aa86eec8f..789462877 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,8 +6,8 @@ MAINTAINER Developers for PANOPTES project ENV PANDIR /var/panoptes COPY . ${POCS} -RUN /opt/conda/bin/conda activate panoptes-env \ - && apt-get update \ + +RUN apt-get update \ && apt-get --yes install \ gcc \ pkg-config \ @@ -15,14 +15,14 @@ RUN /opt/conda/bin/conda activate panoptes-env \ # GPhoto2 && wget https://raw.githubusercontent.com/gonzalo/gphoto2-updater/master/gphoto2-updater.sh \ && chmod +x gphoto2-updater.sh \ - && ./gphoto2-updater.sh --stable \ + && /bin/bash gphoto2-updater.sh --stable \ # Cleanup && apt-get clean \ && rm -rf /var/lib/apt/lists/* \ && rm gphoto2-updater.sh \ # POCS && cd ${POCS} \ - && pip install --no-cache-dir -r requirements.txt + && /opt/conda/envs/panoptes-env/bin/pip install --no-cache-dir -r requirements.txt WORKDIR ${POCS} From 3396dd41f16130318bba0277a3fcf7230caeebe8 Mon Sep 17 00:00:00 2001 From: Wilfred Gee Date: Sat, 13 Apr 2019 18:35:00 +1000 Subject: [PATCH 040/105] * POCS dir is flexible via `pocs_dir` arg to docker build * Adding vim-nox to POCS image * Symlink the POCS conf_files dir to $PANDIR/conf_files --- Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Dockerfile b/Dockerfile index 789462877..0f05d2b3a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,10 @@ ARG arch=amd64 FROM gcr.io/panoptes-survey/panoptes-utils:$arch MAINTAINER Developers for PANOPTES project +ARG pocs_dir=POCS + ENV PANDIR /var/panoptes +ENV POCS ${PANDIR}/${pocs_dir} COPY . ${POCS} @@ -12,6 +15,7 @@ RUN apt-get update \ gcc \ pkg-config \ libncurses5-dev \ + vim-nox \ # GPhoto2 && wget https://raw.githubusercontent.com/gonzalo/gphoto2-updater/master/gphoto2-updater.sh \ && chmod +x gphoto2-updater.sh \ @@ -22,6 +26,7 @@ RUN apt-get update \ && rm gphoto2-updater.sh \ # POCS && cd ${POCS} \ + && ln -s ${POCS}/conf_files/ ${PANDIR}/ \ && /opt/conda/envs/panoptes-env/bin/pip install --no-cache-dir -r requirements.txt WORKDIR ${POCS} From 63695e3d686dc52fbe91d430e13c60a25c55f553 Mon Sep 17 00:00:00 2001 From: Wilfred Gee Date: Sun, 14 Apr 2019 19:31:53 +1000 Subject: [PATCH 041/105] Adding byobu to allow some Docker processes to run in background easily. --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 0f05d2b3a..6ff7e3895 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,6 +12,7 @@ COPY . ${POCS} RUN apt-get update \ && apt-get --yes install \ + byobu \ gcc \ pkg-config \ libncurses5-dev \ From 1b242091fc7408ae90d16fc80e7951033dabd9ae Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Mon, 15 Apr 2019 07:22:03 +1000 Subject: [PATCH 042/105] Removing old `install` scripts --- scripts/install/README.md | 114 --- scripts/install/apt-packages-list.txt | 67 -- scripts/install/conda-packages-list.txt | 2 - scripts/install/configure-apt-cache.sh | 57 -- scripts/install/create-panoptes-user.sh | 63 -- scripts/install/default-env-vars.sh | 7 - scripts/install/install-apt-packages.sh | 58 -- scripts/install/install-dependencies.sh | 813 ------------------ scripts/install/install-helper-functions.sh | 225 ----- scripts/install/install-pocs.sh | 79 -- .../install/run-apt-cacher-ng-in-docker.sh | 29 - 11 files changed, 1514 deletions(-) delete mode 100644 scripts/install/README.md delete mode 100644 scripts/install/apt-packages-list.txt delete mode 100644 scripts/install/conda-packages-list.txt delete mode 100755 scripts/install/configure-apt-cache.sh delete mode 100755 scripts/install/create-panoptes-user.sh delete mode 100644 scripts/install/default-env-vars.sh delete mode 100755 scripts/install/install-apt-packages.sh delete mode 100755 scripts/install/install-dependencies.sh delete mode 100644 scripts/install/install-helper-functions.sh delete mode 100644 scripts/install/install-pocs.sh delete mode 100755 scripts/install/run-apt-cacher-ng-in-docker.sh diff --git a/scripts/install/README.md b/scripts/install/README.md deleted file mode 100644 index 0f46fe214..000000000 --- a/scripts/install/README.md +++ /dev/null @@ -1,114 +0,0 @@ -# PANOPTES Software Installation Tools - -## Installation Procedure - -__MUCH WORK IS NEEDED HERE__ - -### Initial Setup of Bare Computer - -#### Install Ubuntu 18.04 Desktop -#### Create Primary Account - -When prompted to create the primary account during the install of Ubuntu, -create an account with your name (e.g. james)... - -### Install Linux Packages (Software) Needed to Install PANOPTES - -``` -sudo apt-get update -sudo apt-get install --yes git -``` - -### Install Linux Packages (Software) Needed by PANOPTES - -*This is where we need a single script that we can fetch from github -rather than use git clone.* - -#### Create Secondary (PANOPTES) Account - -Create an account for user `panoptes` with all the privileges of the -primary user, especially plugdev (needed for access to serial ports). - -> If using docker, then also add the user to the docker group. -> If you don't know what that means, then skip adding to the docker group. - -## Script Descriptions - -### default-env-vars.sh - -Sets the PANOPTES related environment variables to their default values. -Once you've run install-dependencies.sh you won't need this script anymore. - -### run-apt-cacher-ng-in-docker.sh - -This is a tool primarily for developers and testers of the PANOPTES -software. It helps speed up the install-apt-packages.sh in the case -that you need to run that command many times, as when testing changes -to the install scripts. In particular, it starts a docker container -running a caching proxy that listens on port 3142 for requests for -apt packages. If it hasn't seen a request for that package before, -it pass the request on to the "real" package repository, stores the -fetched package in a directory and returns the package to the caller -(e.g. to `apt-get`). - -Requires that `docker` be installed. Learn more on the -(docker site)[https://docs.docker.com/install/linux/docker-ce/ubuntu/]. - -### configure-apt-cache.sh - -If you've run run-apt-cacher-ng-in-docker.sh, then you can either -tell apt-get where to find the caching proxy each time or you can -run this script which permanently records the fact that you have -a caching proxy running on port 3142. - -``` - $ $POCS/scripts/install/configure-apt-cache.sh 3142 -``` - -### install-apt-packages.sh - -Installs the required Linux open source software onto the machines. -The user that executes this must be a member of the sudo group -(i.e. allowed to run commands as `root`). -If you're running the caching proxy (see above), and have NOT -run configure-apt-cache.sh, then you'll want to tell this script -where to find the proxy: - -``` - $ APT_PROXY_PORT=3142 $POCS/scripts/install/install-apt-packages.sh -``` - -### install-dependencies.sh - -Installs the open source software that the PANOPTES software depends upon, -including running install-apt-packages.sh. -Writes a script ($PANDIR/set-panoptes-env.sh) that sets up your shell with -the correct environment variables for running the PANOPTES software, and -adds a call to that script into your shell's initialization script. - -### install-helper-functions.sh - -This isn't intended to be executed directly; instead it provides a bunch of -functions that are used by several of the scripts described above. - -## Resource (List) Descriptions - -These files list packages to be installed, and are read by the scripts -described above. - -### apt-packages-list.txt - -List of Linux packages to be installed; these are "native" packages (i.e. -compiled software). Installing such packages requires `root` privileges -in order to execute `apt-get install` (used by `install-apt-packages.sh` -and `install-dependencies.sh`). - -### conda-packages-list.txt - -List of packages in the Anaconda format, many of which are implemented in -Python, but not all. These do not require `root` privileges to install. - -### $POCS/requirements.txt - -List of packages in the Wheel format, installed using the `pip` tool. -These do not require `root` privileges to install. diff --git a/scripts/install/apt-packages-list.txt b/scripts/install/apt-packages-list.txt deleted file mode 100644 index dad676e3b..000000000 --- a/scripts/install/apt-packages-list.txt +++ /dev/null @@ -1,67 +0,0 @@ -# This file contains a list of apt packages to install. -# Lines starting with # are comments as are blank lines, -# and # starts trailing comments. - -# dcraw decodes Canon raw images -dcraw -# For converting JPGs -imagemagick - -# gphoto2 enables remote control of cameras, including the Canon DSLRs that -# Panoptes uses. This includes changing many settings and taking pictures. -# We are currently using the published linux package, but that can be quite -# old relative to the latest stable version. gphoto2-updater is an install -# script that we might want to use if we need a newer version. For more info, -# see: https://github.com/gonzalo/gphoto2-updater -gphoto2 - -# Packages needed according to http://astrometry.net/doc/build.html -libbz2-dev -libbz2-dev -libcfitsio-dev -libjpeg-dev -libnetpbm10-dev -libpng-dev -netpbm -python-dev -python-numpy -python-pyfits -swig -zlib1g-dev - -# Cairo is a graphics library, and matplotlib can use it as a backend -# for rendering. -libcairo2-dev - -# ZMQ is used for our messaging system. -libzmq3-dev - -# exiftool is used in a couple of places to extract info from the .cr2 file -exiftool - -# We use cfitsio command line tools (e.g. fpack) from POCS, in addition to the -# uses of the cfitsio libraries by astrometry above. -libcfitsio-bin - -# These are used if you want to setup SSH access into the computer (pretty -# darn likely). -openssh-client -openssh-server - -# Linux tool used during building and installing software (e.g. astrometry). -pkg-config -gcc - -# Improves interaction with pocs_shell (via readline). -libncurses5-dev - -# byobu and tmux provide a means to run multiple interactive shells in a way -# that survives disconnection. -byobu -tmux - -# GNU coreutils provides basic commands that we depend on (echo, ls), and is -# almost certainly already installed, but let's make sure. -coreutils - -vim-nox \ No newline at end of file diff --git a/scripts/install/conda-packages-list.txt b/scripts/install/conda-packages-list.txt deleted file mode 100644 index d820bb9bb..000000000 --- a/scripts/install/conda-packages-list.txt +++ /dev/null @@ -1,2 +0,0 @@ -pandas - diff --git a/scripts/install/configure-apt-cache.sh b/scripts/install/configure-apt-cache.sh deleted file mode 100755 index e46730a91..000000000 --- a/scripts/install/configure-apt-cache.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash -e - -# Tell apt where to find the local (on-host) caching proxy for APT packages, -# if one has been setup. Pass in the port as the first (and only) argument -# to this script, else set the APT_PROXY_PORT environment variable to the -# port number. Records the URL of the proxy "permanently" in -# /etc/apt/apt.conf.d/01proxy; root access is required to write to -# that file. -# -# Using this script is appropriate if it isn't practical to pass the -# URL of the proxy on the apt-get command line, such as this way: -# -# apt-get -o "Acquire::HTTP::Proxy=${APT_PROXY_URL}" install ... -# -# The script install-apt-packages.sh uses this latter approach. -# -# To learn more, see: -# https://github.com/sameersbn/docker-apt-cacher-ng -# https://gist.github.com/dergachev/8441335 -# https://chandrusoft.wordpress.com/2012/09/30/disable-proxy-when-using-apt-get/ - -THIS_DIR="$(dirname "$(readlink -f "${0}")")" -# shellcheck source=/var/panoptes/POCS/scripts/install/install-helper-functions.sh -source "${THIS_DIR}/install-helper-functions.sh" - -# shellcheck disable=SC2086 -APT_PROXY_URL="$(get_apt_proxy_url ${1})" -if [ -z "${APT_PROXY_URL}" ] ; then - echo "Unable to find operating APT proxy!" 1>&2 - exit 1 -fi - -APT_PROXY_FILE=/etc/apt/apt.conf.d/01proxy - -if [ 0 == "$(id -u)" ]; then - # Running as root already. - cat >> "${APT_PROXY_FILE}" < /dev/null -Acquire::HTTP::Proxy "${APT_PROXY_URL}"; -Acquire::HTTPS::Proxy "false"; -EOL -) -fi - -echo -echo "Appended to ${APT_PROXY_FILE}" -echo_bar -cat /etc/apt/apt.conf.d/01proxy -echo_bar -echo -echo "Using apt proxy at: ${APT_PROXY_URL}" diff --git a/scripts/install/create-panoptes-user.sh b/scripts/install/create-panoptes-user.sh deleted file mode 100755 index e11a12511..000000000 --- a/scripts/install/create-panoptes-user.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash -ex -# -# Create the user that will own /var/panoptes and will execute the -# panoptes software. Minimal usage: -# -# PANUSER=panoptes ./create-panoptes-user.sh -# -# Looks for these environment variables: -# -# PANUSER: Name of the user to create. Required -# PANUSER_ID: Id (integer) of the user to create. Not required. -# PANGROUP: Name of the group to to create for the user. Defaults to PANUSER. -# PANGROUP_ID: Id (integer) of the group to create. Not required. -# NO_PROMPTING: If not empty, then no prompting for a password or GECOS info is done. - -if [ -z "${PANUSER}" ] ; then - echo "Environment variable PANUSER is not set" - exit 1 -fi - -if [ "${PANUSER}" == "$(id -u -n)" ] ; then - echo "PANUSER (${PANUSER}) is the current user, so already exists." - exit 0 -fi - -user_id=$(id -u "${PANUSER}" 2> /dev/null || /bin/true) - -if [ -n "$user_id" ] ; then - echo "PANUSER (${PANUSER}) already exists with UID $user_id." - exit 1 -fi - -if [ -n "${PANGROUP}" ] ; then - # NOT testing if the group exists. - if [ -n "${PANGROUP_ID}" ] ; then - # The caller provided a group id. - addgroup --debug --gid "${PANGROUP_ID}" "${PANGROUP}" - else - addgroup --debug "${PANGROUP}" - PANGROUP_ID=$(id -g "${PANGROUP}") - fi -elif [ -n "${PANGROUP_ID}" ] ; then - echo "Specifying only the group id and not group name is not supported." - exit 1 -fi - -CMD="adduser --debug" -CMD+=" --home /home/${PANUSER}" -CMD+=" --shell /bin/bash" -CMD+=" --add_extra_groups" -if [ -n "${PANUSER_ID}" ] ; then - CMD+=" --uid ${PANUSER_ID}" -fi -if [ -n "${PANGROUP}" ] ; then - CMD+=" --ingroup ${PANGROUP}" -fi -if [ -n "${NO_PASSWORD}" ] ; then - CMD+=" --disabled-password --gecos ''" -fi - -CMD+=" ${PANUSER}" - -$CMD \ No newline at end of file diff --git a/scripts/install/default-env-vars.sh b/scripts/install/default-env-vars.sh deleted file mode 100644 index 7851d3326..000000000 --- a/scripts/install/default-env-vars.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -[[ -z "$PANUSER" ]] && export PANUSER=panoptes # Default user -[[ -z "$PANDIR" ]] && export PANDIR=/var/panoptes # Main Dir -[[ -z "$PANLOG" ]] && export PANLOG=$PANDIR/logs # Log files -[[ -z "$POCS" ]] && export POCS=$PANDIR/POCS # Main Observatory Control -[[ -z "$PAWS" ]] && export PAWS=$PANDIR/PAWS # Web Interface diff --git a/scripts/install/install-apt-packages.sh b/scripts/install/install-apt-packages.sh deleted file mode 100755 index 623d3c104..000000000 --- a/scripts/install/install-apt-packages.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -ex - -# Installs APT packages (e.g. debian/ubuntu packages installed for all users). -# Requires being logged in as root or the ability to successfully execute sudo. - -THIS_DIR="$(dirname "$(readlink -f "${0}")")" -# shellcheck source=/var/panoptes/POCS/scripts/install/install-helper-functions.sh -source "${THIS_DIR}/install-helper-functions.sh" - -# Suppress prompting for input during package processing, unless DEBIAN_FRONTEND -# is already set. The tzdata package prompts the user to pick a timezone, -# annoying when you're trying to automate things and don't care which time -# zone is selected (the default is UTC). -if [ -z "${DEBIAN_FRONTEND}" ] ; then - export DEBIAN_FRONTEND=noninteractive -fi - -# Generate the basic apt-get install command, minus the list of packages. -# We store into a shell array, i.e. an array of strings. -declare -a apt_get_install=(apt-get install --no-install-recommends --yes) -# shellcheck disable=SC2119 -apt_proxy_url="$(get_apt_proxy_url)" -if [ -n "${apt_proxy_url}" ] ; then - apt_get_install+=(-o "Acquire::HTTP::Proxy=${apt_proxy_url}") -fi - -# Install all of the packages specified in the files in the args. -function install_apt_packages() { - echo - echo_running_sudo "apt-get update" - echo - my_sudo apt-get update - - for APT_PKGS_FILE in "$@" - do - # Remove all the comments from the package list and install the packages whose - # names are left. - APT_PKGS="$(cut '-d#' -f1 "${APT_PKGS_FILE}" | sort | uniq)" - echo - echo_running_sudo "apt-get install for the files in ${APT_PKGS_FILE}" - echo - # A note on syntax: ${array_variable} expands to just the first element - # of the array. ${array_variable[@]} expands to all of the elements. - # Putting quotes around that has the affect of expanding the array as - # one quoted string PER element in the array, and thus spaces in - # a single element (e.g. "a b") doesn't result in multiple 'words'. - # shellcheck disable=SC2086 - my_sudo "${apt_get_install[@]}" ${APT_PKGS} - done -} - -cd "${THIS_DIR}" - -if [ $# -eq 0 ] ; then - install_apt_packages apt-packages-list.txt -else - install_apt_packages "$@" -fi diff --git a/scripts/install/install-dependencies.sh b/scripts/install/install-dependencies.sh deleted file mode 100755 index 76e38b639..000000000 --- a/scripts/install/install-dependencies.sh +++ /dev/null @@ -1,813 +0,0 @@ -#!/bin/bash -e - -# Run with --help to see your options. With no options, does a complete install -# of dependencies, though attempts to reuse existing installs. - -THIS_DIR="$(dirname "$(readlink -f "${0}")")" -THIS_PROGRAM="$(basename "${0}")" - -# shellcheck source=/var/panoptes/POCS/scripts/install/install-helper-functions.sh -source "${THIS_DIR}/install-helper-functions.sh" - -# We try to figure out where things are so that minimal work is required -# by the user. -VARS_ARE_OK=1 -GUESSED_A_VAR=0 -if [[ -z "${PANDIR}" ]] ; then - if [[ -d /var/panoptes ]] ; then - PANDIR=/var/panoptes - GUESSED_A_VAR=1 - else - echo "PANDIR variable not set to a directory; usually /var/panoptes." - VARS_ARE_OK=0 - fi -elif [[ -e "${PANDIR}" && ! -d "${PANDIR}" ]] ; then - echo "PANDIR (${PANDIR}) is not a directory." - VARS_ARE_OK=0 -fi - -if [[ -z "${POCS}" && ! -z "${PANDIR}" ]] ; then - POCS="${PANDIR}/POCS" - GUESSED_A_VAR=1 -fi -if [[ -z "${POCS}" ]] ; then - echo "Unable to find the POCS directory; usually /var/panoptes/POCS." - VARS_ARE_OK=0 -elif [[ ! -d "${POCS}" ]] ; then - echo '$'"POCS (${POCS}) is not a directory." - VARS_ARE_OK=0 -fi - -if [[ -z "${PAWS}" && ! -z "${PANDIR}" ]] ; then - PAWS="${PANDIR}/PAWS" - GUESSED_A_VAR=1 -fi -if [[ -z "${PAWS}" ]] ; then - echo "Unable to find the PAWS directory; usually /var/panoptes/PAWS." - VARS_ARE_OK=0 -elif [[ ! -d "${PAWS}" ]] ; then - echo '$'"PAWS (${PAWS}) is not a directory." - VARS_ARE_OK=0 -fi - -if [[ -z "${PANLOG}" && ! -z "${PANDIR}" ]] ; then - PANLOG="${PANDIR}/logs" - GUESSED_A_VAR=1 -fi -if [[ -z "${PANLOG}" ]] ; then - echo "Unable to find the PANLOG directory; usually /var/panoptes/logs." - VARS_ARE_OK=0 -elif [[ -e "${PANLOG}" && ! -d "${PANLOG}" ]] ; then - echo '$'"PANLOG (${PANLOG}) is not a directory." - VARS_ARE_OK=0 -fi - -if [[ -z "${PANUSER}" && "$(whoami)" == "panoptes" ]] ; then - PANUSER="panoptes" - GUESSED_A_VAR=1 -elif [[ -z "${PANUSER}" ]] ; then - echo "Unable to determine the PANOPTES user name (PANUSER); usually panoptes." - VARS_ARE_OK=0 -fi - -if [[ "${VARS_ARE_OK}" == "0" ]] ; then - echo "Please set the Panoptes environment variables, then re-run this script." - exit 1 -fi - -if [[ "${GUESSED_A_VAR}" == "1" ]] ; then - echo " - Using these variable assignments: - - PANDIR = ${PANDIR} - POCS = ${POCS} - PAWS = ${PAWS} - PANLOG = ${PANLOG} - PANUSER = ${PANUSER} -" -fi - -PYTHON_VERSION="3.6" -ASTROMETRY_VERSION="0.76" -ASTROMETRY_DIR="${PANDIR}/astrometry" -# CONDA_INSTALL_DIR="${PANDIR}/miniconda" -CONDA_INSTALL_DIR="/opt/conda" -CONDA_SH="${CONDA_INSTALL_DIR}/etc/profile.d/conda.sh" -TIMESTAMP="$(date "+%Y%m%d.%H%M%S")" -INSTALL_LOGS_DIR="${PANDIR}/logs/install/${TIMESTAMP}" -# To which shell file do we need to prepend our additions? -SHELL_RC="${HOME}/.bashrc" -PANOPTES_ENV_SH="${PANDIR}/set-panoptes-env.sh" - -# Export these so they can be accessed in by install scripts executed -# by this script. -export PANDIR POCS PAWS PANLOG PANUSER -export DO_APT_GET=1 -export DO_MONGODB=1 -export DO_CONDA=1 -export DO_REBUILD_CONDA_ENV=0 -export DO_INSTALL_CONDA_PACKAGES=1 -export DO_ASTROMETRY=1 -export DO_ASTROMETRY_INDICES=1 -export DO_PIP_REQUIREMENTS=1 - -DO_RUN_ONE_FUNCTION="" - -#------------------------------------------------------------------------------- - -function show_help() { - echo "${THIS_PROGRAM} - Install software needed for PANOPTES. - -${THIS_PROGRAM} [options] - -options: --h, --help show brief help --x turn on bash debug output ---run run the named function and exit, for debugging ---no-apt-get don't run apt-get to install Linux packages ---no-mongodb don't install and start mongodb server ---no-conda don't install the latest version of Anaconda ---no-conda-packages don't install packages into Anaconda ---rebuild-conda-env rebuild the panoptes-env ---no-astrometry don't install astrometry.net software ---no-astrometry-indices don't install astrometry.net indices ---no-pip-requirements don't install python packages -" -} - -# Parse the command line args. -while test ${#} -gt 0; do - case "${1}" in - -h|--help) - show_help - exit 0 - ;; - -x) - set -x - shift - ;; - --run) - if [ $# -lt 2 ]; then - echo "Function name missing after ${1}." - show_help - exit 1 - fi - shift - DO_RUN_ONE_FUNCTION="${1}" - shift - ;; - --no-apt-get) - DO_APT_GET=0 - shift - ;; - --no-mongodb) - DO_MONGODB=0 - shift - ;; - --no-conda) - DO_CONDA=0 - shift - ;; - --rebuild-conda-env) - DO_REBUILD_CONDA_ENV=1 - shift - ;; - --no-conda-packages) - DO_INSTALL_CONDA_PACKAGES=0 - shift - ;; - --no-pip-requirements) - DO_PIP_REQUIREMENTS=0 - shift - ;; - --no-astrometry) - DO_ASTROMETRY=0 - shift - ;; - --no-astrometry-ind*) - DO_ASTROMETRY_INDICES=0 - shift - ;; - *) - echo "Unknown parameter: ${1}" - echo - show_help - exit 1 - ;; - esac -done - -#------------------------------------------------------------------------------- -# Misc helper functions. - -# Backup the file whose path is the first arg, and print the path of the -# backup file. If the file doesn't exist, no path is output. -function backup_file() { - local -r the_original="${1}" - if [[ ! -e "${the_original}" ]] ; then - return - fi - if [[ ! -f "${the_original}" ]] ; then - echo 1>2 " -${the_original} is not a regular file, can't copy! -" - exit 1 - fi - local -r the_backup="$(mktemp "${the_original}.backup.XXXXXXX")" - cp -p "${the_original}" "${the_backup}" - echo "${the_backup}" -} - -function diff_backup_and_file_then_cleanup() { - local -r the_backup="${1}" - local -r the_file="${2}" - if [[ -z "${the_backup}" ]] ; then - echo_bar - echo - echo "Created ${the_file}:" - echo - cat "${the_file}" - echo - return - fi - if ! cmp "${the_backup}" "${the_file}" ; then - echo_bar - echo - echo "Modified ${the_file}:" - echo - diff -u "${the_backup}" "${the_file}" || /bin/true - echo - fi - rm -f "${the_backup}" -} - -#------------------------------------------------------------------------------- -# Functions for creating the file in which we record the PANOPTES environment -# variables and shell setup commands, and for inserting a 'source ' -# into the rc file of the user's shell (e.g. $HOME/.bashrc for bash). - -# If the desired shell rc file doesn't exist, create it. -function ensure_shell_rc_exists() { - if [[ ! -f "${SHELL_RC}" ]] ; then - touch "${SHELL_RC}" - fi -} - -# Return a status code indicating whether the profile file contains the -# text in the first arg to this function (true if contains, false otherwise). -function profile_contains_text() { - local -r target_text="${1}" - if grep -F -q -- "${target_text}" "${SHELL_RC}" ; then - return 0 - else - return 1 - fi -} - -# Add the text of the first arg to the PANOPTES environment setup. -function add_to_panoptes_env_setup() { - PANOPTES_ENV_SETUP+=("${1}") -} - -# Append $1 to PATH and write command to do the same to the -# PANOPTES environment setup. -function add_to_PATH() { - local -r the_dir="$(readlink -f "${1}")" - PATH="${the_dir}:${PATH}" - PANOPTES_ENV_SETUP+=("PATH=\"${the_dir}:\${PATH}\"") -} - -# Create (or overwrite) the PANOPTES environment setup file, -# and print a diff showing the changes, if there are any. -function update_panoptes_env_file() { - local -r the_backup="$(backup_file "${PANOPTES_ENV_SH}")" - cat >"${PANOPTES_ENV_SH}" <>"${PANOPTES_ENV_SH}" - diff_backup_and_file_then_cleanup "${the_backup}" "${PANOPTES_ENV_SH}" -} - -# Arrange for the PANOPTES environment setup file to be used -# when the rc file of the user's shell is executed. -function update_shell_rc_file() { - if [[ ! -f "${SHELL_RC}" ]] ; then - cat >"${SHELL_RC}" < "${SHELL_RC}" -# Added by PANOPTES install-dependencies.sh -source ${PANOPTES_ENV_SH} - -END_OF_FILE - diff_backup_and_file_then_cleanup "${the_backup}" "${SHELL_RC}" -} - -#------------------------------------------------------------------------------- - -# Given the path to a pkg-config file (.pc), extract the version number. -function extract_version_from_pkg_config() { - if [[ -f "${1}" ]] ; then - grep -E '^Version:' "${1}" | cut '-d:' -f2 - else - echo "" - fi -} - -function install_mongodb() { - # This is based on https://www.howtoforge.com/tutorial/install-mongodb-on-ubuntu/ - # Note this function does not configure mongodb itself, i.e. no users or - # security settings. - echo_bar - local MONGO_KEY="" - local MONGO_URL="http://repo.mongodb.org/apt/ubuntu" - local MONGO_VERSION="" - local MONGO_SOURCE_PATH="" - local LSB_RELEASE="" - # lsb_release is deprecated, removed from many debian distributions, - # and not present in the official Ubuntu docker images. /etc/os-release - # is recommended instead. - if [ -f /etc/os-release ] ; then - LSB_RELEASE="$(grep VERSION_CODENAME= /etc/os-release | cut -d= -f2)" - elif [[ -n "$(safe_which lsb_release)" ]] ; then - LSB_RELEASE="$(lsb_release -sc)" - fi - if [[ "${LSB_RELEASE}" = "xenial" ]] ; then - MONGO_KEY=2930ADAE8CAF5059EE73BB4B58712A2291FA4AD5 - MONGO_VERSION=3.6 - elif [[ "${LSB_RELEASE}" = "bionic" ]] ; then - MONGO_KEY=9DA31620334BD75D9DCB49F368818C72E52529D4 - MONGO_VERSION=4.0 - else - echo "ERROR: don't know which version of MongoDB to install." - return 1 - fi - MONGO_URL+=" ${LSB_RELEASE}/mongodb-org/${MONGO_VERSION}" - MONGO_SOURCE_PATH="/etc/apt/sources.list.d/mongodb-org-${MONGO_VERSION}.list" - - echo " -Installing MongoDB ${MONGO_VERSION}, for which several commands require sudo, -so you may be prompted for you password. Starting by telling APT where to find -the MongoDB packages. -" - sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv ${MONGO_KEY} - echo "deb ${MONGO_URL} multiverse" | sudo tee "${MONGO_SOURCE_PATH}" - echo " -Updating the list of packages so APT finds the MongoDB packages, -then installing MongoDB. -" - sudo apt-get update - sudo apt-get install -y mongodb-org - echo " -MongoDB is installed, now updating the config and starting mongod. -" - echo "[Unit] -Description=High-performance, schema-free document-oriented database -After=network.target -Documentation=https://docs.mongodb.org/manual - -[Service] -User=mongodb -Group=mongodb -ExecStart=/usr/bin/mongod --quiet --config /etc/mongod.conf - -[Install] -WantedBy=multi-user.target" | sudo tee /lib/systemd/system/mongod.service - - sudo systemctl daemon-reload - sudo systemctl start mongod - sudo systemctl enable mongod -} - -function maybe_install_mongodb() { - if [[ -z "$(safe_which mongo)" ]] || ! (systemctl | grep -q -F mongo) ; then - install_mongodb - fi -} - -#------------------------------------------------------------------------------- -# Anaconda (miniconda) support. - -# Did the user source conda's etc/profile.d/conda.sh? -function conda_is_present() { - if [[ -z "${CONDA_SHLVL}" ]] ; then - return 1 # No - fi - return 0 # Probably -} - -# Get the location of the panoptes environment. -function get_panoptes_env_location() { - if ! conda_is_present ; then - return - fi - # Do we have a conda executable? We don't require CONDA_EXE to already be set - # because conda doesn't set it until the first environment is activated. - local -r conda_exe="${CONDA_EXE:-${CONDA_INSTALL_DIR}/bin/conda}" - if [[ -z "${conda_exe}" || ! -x "${conda_exe}" ]] ; then - return # No - fi - # Get the list of environments. - local -r conda_envs="$("${conda_exe}" info --envs 2>/dev/null || /bin/true)" - if [[ -z "${conda_envs}" ]] ; then - return # Cound't find any environments - fi - # Is there a panoptes-env? - local -r panoptes_env="$( - echo "${conda_envs}" | - (grep -E '^panoptes-env[[:space:]]' || /bin/true))" - if [[ -z "${panoptes_env}" ]] ; then - return # No panoptes environment. - fi - # Extract the panoptes-env line and print the path that comes at - # the end of that line. - echo "${panoptes_env}" | sed 's_.* /_/_' -} - -# Install latest version of Miniconda (Anaconda with very few packages; any that -# are needed can then be installed). -function install_conda() { - if [[ -d "${CONDA_INSTALL_DIR}" ]] ; then - echo_bar - echo - echo "Removing previous miniconda installation from ${CONDA_INSTALL_DIR}" - rm -rf "${CONDA_INSTALL_DIR}" - fi - - echo_bar - echo - echo "Installing miniconda. License at: https://conda.io/docs/license.html" - local -r the_script="${PANDIR}/tmp/miniconda.sh" - mkdir -p "$(dirname "${the_script}")" - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \ - -O "${the_script}" - bash "${the_script}" -b -p "${CONDA_INSTALL_DIR}" - rm "${the_script}" - - # As per the Anaconda 4.4 release notes, one is supposed to add the following - # to .bash_profile, .bashrc or wherever is appropriate: - # . $CONDA_INSTALL_DIR/etc/profile.d/conda.sh - # conda activate - # Where CONDA_INSTALL_DIR is where Anaconda or miniconda was installed. - # We do the first step here. Later we'll activate the PANOPTES environment. - - # shellcheck disable=SC1090 - . "${CONDA_SH}" -} - -# Add additional repositories in which conda should search for packages. -function add_conda_channels() { - # Add the astropy channel, i.e. an additional repository in which to - # look for packages. With conda 4.1.0 and later, by default the highest - # priority repository that contains a package is used as the source for - # that package, even if there is a newer version in a lower priority - # package. And by default the most recently added repository is treated - # as the highest priority repository. Here we use prepend to be clear - # that we want astropy to be highest priority. - # shellcheck disable=SC1090 - . "${CONDA_SH}" - if (conda config --show channels | grep -F -q astropy) ; then - conda config --remove channels astropy - fi - conda config --prepend channels astropy - - # And put conda-forge at the back of the line. - if (conda config --show channels | grep -F -q conda-forge) ; then - conda config --remove channels conda-forge - fi - conda config --append channels conda-forge -} - -# Prepare the conda environment for PANOPTES. -function prepare_panoptes_conda_env() { - # Use the base Anaconda environment until we're ready to - # work with the PANOPTES environment. - # shellcheck disable=SC1090 - . "${CONDA_SH}" - conda activate base - - # Determine if the PANOPTES environment already exists. - local do_create_conda_env=0 - if ! (conda info --envs | grep -q panoptes-env) ; then - do_create_conda_env=1 - elif [[ "${DO_REBUILD_CONDA_ENV}" -eq 1 ]] ; then - conda remove --all --yes --quiet -n panoptes-env - do_create_conda_env=1 - fi - - # Create the PANOPTES environment if necessary. - if [[ "${do_create_conda_env}" -eq 1 ]] ; then - echo_bar - echo - echo "Creating conda environment 'panoptes-env' with Python ${PYTHON_VERSION}" - conda create -n panoptes-env --yes --quiet "python=${PYTHON_VERSION}" - fi - - if [[ "${DO_INSTALL_CONDA_PACKAGES}" -eq 1 ]] ; then - echo_bar - echo - echo "Installing panoptes-env packages." - conda install -n panoptes-env --yes --quiet "--file=${THIS_DIR}/conda-packages-list.txt" - - echo_bar - echo - echo "Updating all panoptes-env packages." - conda update -n panoptes-env --yes --quiet --all - fi -} - -# Install conda if we can't find it. Note that starting with version -# 4.4, conda's bin directory is not on the path, so 'which conda' -# can't be used to determine if it is present. -# -# TODO(jamessynge): Stop allowing for an existing conda to be -# located elsewhere. -function maybe_install_conda() { - # Just in case conda isn't setup, but exists... - if [[ -z "$(safe_type conda)" && -f "${CONDA_SH}" ]] ; then - # shellcheck disable=SC1090 - . "${CONDA_SH}" - fi - if [[ -z "$(safe_type conda)" ]] ; then - install_conda - else - echo_bar - echo - echo "Reusing existing conda installation ($(conda --version)):" "${_CONDA_ROOT}" - fi - echo_bar - echo - echo "Updating base conda." - conda update -n base --yes --quiet conda -} - -function get_installed_astrometry_version() { - local -r solve_field="${ASTROMETRY_DIR}/bin/solve-field" - if [[ -x "${solve_field}" ]] ; then - "${solve_field}" --help|(grep -E '^Revision [0-9.]+,' || /bin/true)|cut -c10-|cut -d, -f1 - fi -} - -function test_installed_astrometry_version() { - local -r installed_version="$(get_installed_astrometry_version)" - if [[ -n "${installed_version}" && \ - "${ASTROMETRY_VERSION}" == "${installed_version}" ]] ; then - echo_bar - echo - echo "Reusing existing astrometry ${installed_version} installation." - return 0 - else - return 1 - fi -} - -# Downloads astrometry version ${ASTROMETRY_VERSION} into a temp directory, -# then builds and installs into ${ASTROMETRY_DIR}. Skips as much as -# possible if able to determine that the version hasn't changed. -function install_astrometry() { - local -r FN="astrometry.net-${ASTROMETRY_VERSION}.tar.gz" - local -r URL="http://astrometry.net/downloads/${FN}" - local -r SCRATCH_DIR="$(mktemp -d "${PANDIR}/tmp/install-astrometry.XXXXXXXXXXXX")" - local -r INSTALL_TO="${ASTROMETRY_DIR}" - local -r md5sum_file="${INSTALL_TO}/TAR_MD5SUM.txt" - local -r make_log="${INSTALL_LOGS_DIR}/make-astrometry.log" - local -r make_py_log="${INSTALL_LOGS_DIR}/make-astrometry-py.log" - local -r make_install_log="${INSTALL_LOGS_DIR}/make-astrometry-install.log" - cd "${SCRATCH_DIR}" - - echo_bar - echo - echo "Fetching astrometry into directory $(pwd)" - wget "${URL}" - - # Is the file the same as the one used for the current install? - if [[ -f "${md5sum_file}" ]] && md5sum --check --status "${md5sum_file}" && \ - [[ "$(get_installed_astrometry_version)" != "" ]] ; then - echo_bar - echo - echo -n "Checksum matches already installed astrometry version " - echo "$(get_installed_astrometry_version), not replacing." - cd "${PANDIR}" - rm -rf "${SCRATCH_DIR}" - return - fi - local -r tar_md5sum="$(md5sum "${FN}")" - - echo_bar - echo - echo "Unpacking ${FN}." - tar zxf "${FN}" - - if [[ "${ASTROMETRY_VERSION}" == "latest" ]] ; then - # We need to know the version that is in the tar file in order to - # enter that directory. - ASTROMETRY_VERSION="" - for version in $(find astrometry.net-* -maxdepth 0 -type d | sed "s/astrometry.net-//") - do - if [[ "${ASTROMETRY_VERSION}" == "" ]] ; then - ASTROMETRY_VERSION="${version}" - else - echo_bar - echo - echo "ERROR: Found two version number directories in the tar file!" - find astrometry.net-* -maxdepth 0 -type d - exit 1 - fi - done - - if [[ ! -f "${md5sum_file}" ]] ; then - # We've not yet been able to check if the installed version should - # be replaced. - if test_installed_astrometry_version ; then - return - fi - fi - fi - - echo_bar - echo - echo "Building astrometry ${ASTROMETRY_VERSION}, logging to ${make_log}" - - cd "astrometry.net-${ASTROMETRY_VERSION}" - make >"${make_log}" 2>&1 - - echo_bar - echo - echo "Building astrometry python support, logging to ${make_py_log}" - make py >"${make_py_log}" 2>&1 - - echo_bar - echo - echo "Installing built astrometry into directory ${INSTALL_TO}," - echo "and logging to ${make_install_log}" - if [[ -d "${INSTALL_TO}" ]] ; then - rm -rf "${INSTALL_TO}" - fi - mkdir -p "${INSTALL_TO}" - make install "INSTALL_DIR=${INSTALL_TO}" >"${make_install_log}" 2>&1 - - cd "${PANDIR}" - rm -rf "${SCRATCH_DIR}" - - echo "${tar_md5sum}" >"${md5sum_file}" - local -r installed_version="$(get_installed_astrometry_version)" - if [[ "${ASTROMETRY_VERSION}" != "${installed_version}" ]] ; then - echo - echo "ERROR: version mismatch after installing astrometry!" - echo "Expected: ${ASTROMETRY_VERSION}" - echo " Actual: ${installed_version}" - exit 1 - fi -} - -# Installs astrometry if missing or if the wrong version. -function maybe_install_astrometry() { - if ! test_installed_astrometry_version ; then - install_astrometry - fi -} - -# Run a POCS script for installing the indices used by astrometry -# to solve fields. -function install_astrometry_indices() { - echo_bar - echo - echo "Fetching astrometry and other indices." - mkdir -p "${ASTROMETRY_DIR}/data" - python "${POCS}/pocs/utils/data.py" -} - -#------------------------------------------------------------------------------- -# Do what the user asked us to do... - -mkdir -p "${PANDIR}" -cd "${PANDIR}" -mkdir -p tmp -mkdir -p "${INSTALL_LOGS_DIR}" - -# For testing, run the named function in a sub-shell, then exit. -if [[ -n "${DO_RUN_ONE_FUNCTION}" ]] ; then - if ("${DO_RUN_ONE_FUNCTION}") ; then - echo "Function was successful." - else - echo "Function failed with status $?" - fi - exit -fi - -# Install packages using the APT package manager. Works on Debian based systems -# such as Ubuntu. -if [[ "${DO_APT_GET}" -eq 1 ]] ; then - "${THIS_DIR}/install-apt-packages.sh" -fi - -# Install Mongodb, a document database. Used by POCS for storing observation -# metadata and the environment readings from which weather plots are generated. -if [[ "${DO_MONGODB}" -eq 1 ]] ; then - maybe_install_mongodb -fi - -# Install Conda, a Python package manager from Anaconda, Inc. Supports both -# pure Python packages (just as pip does) and packages with non-Python parts -# (e.g. native libraries). -if [[ "${DO_CONDA}" -eq 1 ]] ; then - maybe_install_conda -fi - -if [[ -f "${CONDA_SH}" ]] ; then - # shellcheck disable=SC1090 - . "${CONDA_SH}" -else - echo_bar - echo " -Error: conda is not installed, but remaining installation steps make -use of the conda environment for PANOPTES (panoptes-env). -" - exit 1 -fi - -if [[ "${DO_CONDA}" -eq 1 || \ - "${DO_REBUILD_CONDA_ENV}" -eq 1 || \ - "${DO_INSTALL_CONDA_PACKAGES}" -eq 1 ]] ; then - prepare_panoptes_conda_env -fi - -# Activate the PANOPTES environment in this shell. -conda activate panoptes-env - -# Install pure Python packages using pip; note that we prefer to -# install these with conda if they are available that way, as we -# try to reduce the number of package managers and repositories -# we have to deal with. -if [[ "${DO_PIP_REQUIREMENTS}" -eq 1 ]] ; then - echo_bar - echo - echo "Upgrading pip before installing other python packages." - pip install --quiet --upgrade pip - - echo_bar - echo - echo "Installing python packages using pip." - pip install --quiet --requirement "${POCS}/requirements.txt" -fi - -# Install the astrometry.net software package, enabling plate-solving -# of images captured by a PANOPTES unit. -if [[ "${DO_ASTROMETRY}" -eq 1 ]] ; then - (maybe_install_astrometry) -fi -if [[ "${DO_ASTROMETRY_INDICES}" -eq 1 ]] ; then - (install_astrometry_indices) -fi - -update_panoptes_env_file -update_shell_rc_file - -# Cleanup the tmp directory, but only if it is empty. -rmdir --ignore-fail-on-non-empty "${PANDIR}/tmp" - -set +x - -echo -echo_bar -echo_bar -echo -echo " -Installation complete. Please run these commands: - source ${PANOPTES_ENV_SH} - cd \$POCS - python setup.py install ; pytest -None of the tests should fail. -" - -exit diff --git a/scripts/install/install-helper-functions.sh b/scripts/install/install-helper-functions.sh deleted file mode 100644 index 01671b801..000000000 --- a/scripts/install/install-helper-functions.sh +++ /dev/null @@ -1,225 +0,0 @@ -#!/bin/bash -# This file is to be sourced into another to add helper functions, rather than -# executed directly. - -#------------------------------------------------------------------------------- -# Logging support (nascent; I want to add more control and a log file). - -# Print a separator bar of # characters. -function echo_bar() { - local terminal_width - if [ -n "${TERM}" ] && [ -t 0 ] ; then - if [[ -n "$(which resize)" ]] ; then - terminal_width="$(resize 2>/dev/null | grep COLUMNS= | cut -d= -f2)" - elif [[ -n "$(which stty)" ]] ; then - terminal_width="$(stty size 2>/dev/null | cut '-d ' -f2)" - fi - fi - printf "%${terminal_width:-80}s\n" | tr ' ' '#' -} - -function echo_running_sudo() { - if [ "$(id -u -n)" == "root" ] ; then - echo "Running $1" - else - echo "Running sudo $1; you may be prompted for your password." - fi -} - -#------------------------------------------------------------------------------- -# Misc helper functions. - -# Execute the args (a command) as root, either using sudo or directly if already -# running as root. -function my_sudo() { - if [ "$(id -u -n)" == "root" ] ; then - "$@" - else - (set -x ; sudo "$@") - fi -} - -# Get the type of the first arg, i.e. shell function, executable, etc. -# For more info: https://ss64.com/bash/type.html -# or: https://bash.cyberciti.biz/guide/Type_command -function safe_type() { - type -t "${1}" || /bin/true -} - -# Print the disk location of the first arg. -function safe_which() { - type -p "${1}" || /bin/true -} - -# Does the first arg start with the second arg? -function beginswith() { case "${1}" in "${2}"*) true;; *) false;; esac; } - -# Match a line that looks like this: -# 8.8.8.8 via 192.168.86.1 dev wlp3s0 src 192.168.86.36 uid 1000 -# And extract either the "via" or the "src" address. -# $1 (first and only arge) should be "via" or "src". -function sbin_ip_route_addr() { - local -r pattern="${1}" - local cmd - if [ -x "$(safe_which ip)" ] ; then - cmd='ip' - elif [ -x /sbin/ip ] ; then - cmd='/sbin/ip' - else - return 0 - fi - "${cmd}" route get 8.8.8.8 | awk -F" ${pattern} " 'NR==1{split($2,a," ");print a[1]}' -} - -#------------------------------------------------------------------------------- -# Helpers for finding the IP gateway of this device. - -# Get the address of the gateway stored in /proc/net/route. Available inside -# a docker container or in Ubuntu 18.04. When executed inside a container, -# the address is that of the host; when executed on the host, the address is -# that of the IP gateway on the lan segment (e.g. the adjacent router). -function proc_net_route_gateway() { - if [ ! -e /proc/net/route ] ; then - return 0 - fi - # This can be done with awk, but that is then much harder to debug. - # /proc/net/route has contents like this: - # Iface Destination Gateway Flags RefCnt Use Metric Mask MTU Window IRTT - # eth0 00000000 010011AC 0003 0 0 0 00000000 0 0 0 - # We're looking to find a line like the second one, with zero for the destination - # and the Gateway is the 32-bit hex value for the IPv4 address of the docker host, - # where the last two characters represent the leading (high) bits of the address. - local -r -a words=( $(grep -E -i '^[a-z][a-z0-9]+\s00000000\s[0-9a-f][0-9a-f][0-9a-f][0-9a-f]' /proc/net/route ) ) - local -r gateway="${words[2]}" - local -r a="0x${gateway:6:2}" - local -r b="0x${gateway:4:2}" - local -r c="0x${gateway:2:2}" - local -r d="0x${gateway:0:2}" - printf "%d.%d.%d.%d\n" "${a}" "${b}" "${c}" "${d}" -} - -# Print the address of the first gateway on the route to 8.8.8.8, i.e. to -# Google's public DNS server. Inside a docker container, this will be the -# address of the host. Requires the iproute2 package be installed, which it -# is not by default in an ubuntu-18.04 docker image. -function sbin_ip_gateway() { - sbin_ip_route_addr via -} - -# Try all methods, output all gateway addresses (there could be duplicates). -function get_gateway_ips() { - proc_net_route_gateway - sbin_ip_gateway -} - -#------------------------------------------------------------------------------- -# Helpers for finding the IP address of this device. - -# Get the IP addresses of the host, of which there may be multiple or none. -function hostname_addresses() { - if [ -x "$(safe_which hostname)" ] ; then - hostname --all-ip-addresses - fi -} - -# Print the first IP on the route to 8.8.8.8, i.e. to Google's public DNS -# server. Inside a docker container, this will be the address of the container, -# and on a host, this will be the address of one of the interfaces or nothing -# if 'ip' is not installed or there is no interfaces enabled. -# Requires the iproute2 package be installed, which it is not by default in an -# ubuntu-18.04 docker image. -function sbin_ip_host() { - sbin_ip_route_addr src -} - -# Try all methods, output all host addresses (there could be duplicates). -function get_host_ips() { - hostname_addresses - sbin_ip_host -} - -#------------------------------------------------------------------------------- -# Helpers for finding the APT caching proxy. - -# Checks whether $1 (a URL) is for a server that will respond. -# Returns status code: -# 1 if the URL ($1) is invalid -# $2 if no tool available to check the validity -# 0 otherwise -function check_is_url() { - local -r url="${1}" - local -r default_status="${2}" - - # curl supports head requests, exactly what we want. - if [ -x "$(safe_which curl)" ] ; then - local -r response_head="$(curl --silent --fail-early --head --max-time 3 "${url}" || /bin/true)" - if beginswith "${response_head}" "HTTP/" ; then - return 0 - fi - echo "curl unable to fetch headers for ${url}" 1>&2 - return 1 - fi - - if [ -x "$(safe_which wget)" ] ; then - (set +e ; wget --quiet --spider --tries 1 "${url}") - case $? in - 0|6|8) # No error, auth failure or HTTP error. - return 0 - ;; - *) - echo "wget unable to fetch headers for ${url}" 1>&2 - return 1 - esac - fi - - return "${default_status}" -} - -# Print the URL for the APT caching proxy. Looks for the port to use as -# the first arg (if any are provided), else the APT_PROXY_PORT environment -# variable; if neither is set, then no URL is printed. -# If curl is available, it is used to test whether the URL is valid, but if -# not then the URL is assumed valid. -function get_apt_proxy_url() { - local -r apt_proxy_port="${1:-${APT_PROXY_PORT}}" - if [ -z "${apt_proxy_port}" ] ; then - cat <&2 -No apt cache proxy declared via APT_PROXY_PORT or script argument. -EOF - return 0 - fi - # echo "APT_PROXY_PORT: ${apt_proxy_port}" 1>&2 - - local -a addresses - if grep ':/docker/' /proc/1/cgroup -qa; then - # Running inside docker, so look for the proxy on the docker host first, - # then inside the container. - addresses+=($(get_gateway_ips)) - addresses+=(localhost) - addresses+=($(get_host_ips)) - else - # Not running inside docker, so look on this host only. - addresses+=($(get_host_ips)) - addresses+=(localhost) - fi - - # See if we can connect to a webserver at our address and the apt proxy port. - local -A tested_addresses - for address in "${addresses[@]}" ; do - if [ -z "${address}" ] ; then - continue - fi - # Don't waste time trying to connect to an address we already checked. - if [ -n "${tested_addresses["${address}"]}" ] ; then - # This address is a duplicate. - continue - fi - tested_addresses["${address}"]="seen" - - local apt_proxy_url="http://${address}:${apt_proxy_port}" - if check_is_url "${apt_proxy_url}" 0 ; then - echo "${apt_proxy_url}" - return 0 - fi - done -} diff --git a/scripts/install/install-pocs.sh b/scripts/install/install-pocs.sh deleted file mode 100644 index 538ad1113..000000000 --- a/scripts/install/install-pocs.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/bin/bash -e - -KEY_FILE=${1} -PANDIR=${2:-/var/panoptes} - -echo "Starting fresh install at `date`" > install.log - -echo "Using ${KEY_FILE}" - -echo "Creating ${PANDIR}" -sudo mkdir -p ${PANDIR}/.key - -sudo chown -R panoptes:panoptes ${PANDIR} -# Make sure time is correct or gcloud won't authenticate -sudo timedatectl set-ntp on - -echo "Moving ${KEY_FILE} to hidden directory ${PANDIR}/.key" -mv ${KEY_FILE} ${PANDIR}/.key - -echo "Updating computer..." - -# Create environment variable for correct distribution -export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" - -# Add the Cloud SDK distribution URI as a package source -echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list - -# Import the Google Cloud Platform public key -wget -q -O- https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - &>> install.log - -sudo apt update &>> install.log -sudo apt install -y docker.io &>> install.log -sudo adduser panoptes docker &>> install.log - -# Install miniconda and docker-compose -wget -q https://repo.continuum.io/miniconda/Miniconda3-3.7.0-Linux-x86_64.sh -O ~/miniconda.sh &>> install.log -bash ~/miniconda.sh -b -p ${PANDIR}/miniconda &>> install.log -rm ~/miniconda.sh - -# Add path to user's shell -echo "export PATH="/var/panoptes/miniconda/bin:$PATH"" >> ~/.bashrc -echo "export PANDIR=/var/panoptes" >> ~/.bashrc -echo "export POCS=/var/panoptes/POCS" >> ~/.bashrc - -cd $HOME -wget -q https://raw.githubusercontent.com/panoptes/POCS/40300ca08d5f12971d03c1ca58a4615f191f9471/resources/docker_files/docker-compose.yml -wget -q https://raw.githubusercontent.com/panoptes/POCS/40300ca08d5f12971d03c1ca58a4615f191f9471/resources/docker_files/env_file -wget -q https://raw.githubusercontent.com/panoptes/POCS/40300ca08d5f12971d03c1ca58a4615f191f9471/scripts/start_pocs_docker.sh -chmod +x start_pocs_docker.sh - -cat <> $HOME/Desktop/POCS.desktop -[Desktop Entry] -Exec=bash -c "cd $HOME && source activate panoptes && ./start_pocs_docker.sh" -Name=pocs -Terminal=true -Type=Application -EOT - -# Add for this session -export PATH="/var/panoptes/miniconda/bin:$PATH" -#source ${PANDIR}/miniconda/bin/activate - -echo "Creating new python environment for panoptes" -conda create -n panoptes --yes python=3 &>> install.log - -source activate panoptes &>> install.log -conda install --yes --quiet -c conda-forge pip google-cloud-sdk &>> install.log -pip install --quiet docker-compose &>> install.log - -echo "Authenticating with google" -gcloud auth activate-service-account --key-file ${PANDIR}/.key/${KEY_FILE} -gcloud auth configure-docker --quiet &>> install.log - -echo "Pulling POCS files from cloud" -echo "WARNING: This is a large file that can take a long time!" -sudo docker pull gcr.io/panoptes-survey/pocs-base -sudo docker pull gcr.io/panoptes-survey/paws - -echo "All done! Please reboot your system." \ No newline at end of file diff --git a/scripts/install/run-apt-cacher-ng-in-docker.sh b/scripts/install/run-apt-cacher-ng-in-docker.sh deleted file mode 100755 index 4c16f96fb..000000000 --- a/scripts/install/run-apt-cacher-ng-in-docker.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -e - -# Runs apt-cacher-ng in a docker container, exposing port 3142 which may -# be used by apt. See configure-apt-cache.sh for the consumer portion -# (intended to run inside of containers). - -# Note that this caching proxy is limited to caching linux packages, -# and won't cache other items (e.g. miniconda packages). A more general -# caching solution could be useful for PANOPTES application of docker. - -CONTAINER_NAME=apt-cacher-ng -CACHE_DIRECTORY="/tmp/$USER-apt-cacher-ng" - -# If the container already exists, start it. Exit code is zero (OK) from -# docker start if it restarted a paused container, or if the container is -# already running. -if (docker start "${CONTAINER_NAME}" >/dev/null 2>&1) ; then - echo "${CONTAINER_NAME} is running." -else - echo "${CONTAINER_NAME} is not running, so starting it." - docker run --name "${CONTAINER_NAME}" --init -d --restart=always \ - --publish 3142:3142 \ - --volume "${CACHE_DIRECTORY}:/var/cache/apt-cacher-ng" \ - sameersbn/apt-cacher-ng:3.1-1 -fi - -if [ -d "${CACHE_DIRECTORY}" ] ; then - echo "Cache size:" $(du -s -h "${CACHE_DIRECTORY}") -fi From 12fcf29a4acc3811ddd741cc0661f0385dd0993f Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Mon, 15 Apr 2019 08:52:14 +1000 Subject: [PATCH 043/105] First attempt at a script to install POCS environment: * Make /var/panoptes directories for host * Install git, wget * Clone git repos as specified user (POCS, PAWS, panoptes-utils) * Install Docker * Pull Docker images (pocs, panoptes-utils) --- scripts/install/install-pocs.sh | 71 +++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100755 scripts/install/install-pocs.sh diff --git a/scripts/install/install-pocs.sh b/scripts/install/install-pocs.sh new file mode 100755 index 000000000..04d4842ff --- /dev/null +++ b/scripts/install/install-pocs.sh @@ -0,0 +1,71 @@ +#!/bin/bash -e + +PANUSER='panoptes' +PANDIR='/var/panoptes' + +while [[ $# -gt 0 ]] +do +key="$1" +case $key in + -u|--user) + PANUSER="$2" + shift # past argument + shift # past value + ;; + -d|--dir) + PANDIR="$2" + shift # past argument + shift # past value + ;; +esac +done + +echo "USER: ${PANUSER}" +echo "DIR: ${PANDIR}" + +echo "Creating directories" +# Make directories +sudo mkdir -p ${PANDIR} +sudo chown -R ${PANUSER}:${PANUSER} ${PANDIR} + +mkdir -p ${PANDIR}/logs +mkdir -p ${PANDIR}/conf_files +mkdir -p ${PANDIR}/images + +echo "Log files will be stored in ${PANDIR}/logs/install-pocs.log" + +# apt: git, wget +echo "Installing system dependencies" +sudo apt update &>> ${PANDIR}/logs/install-pocs.log +sudo apt --yes install wget git &>> ${PANDIR}/logs/install-pocs.log + +echo "Cloning PANOPTES source code" +echo "Github user for PANOPTES repos (POCS, PAWS, panoptes-utils)" +read -p "Github User: [panoptes] " github_user + +cd ${PANDIR} + +declare -a repos=("POCS" "PAWS" "panoptes-utils") + +for repo in "${repos[@]}"; do + if [ ! -d "${PANDIR}/${repo}" ]; then + echo "Cloning ${repo}" + git clone https://github.com/${github_user}/${repo}.git &>> ${PANDIR}/logs/install-pocs.log + else + echo "Repo ${repo} already exists on system" + fi +done + +# Get Docker +if ! hash docker; then + echo "Installing Docker" + sh -c "$(wget https://get.docker.com -O -)" + sudo usermod -aG docker ${PANUSER} +fi + +echo "Pulling POCS docker images" +sudo docker pull gcr.io/panoptes-survey/pocs +sudo docker pull gcr.io/panoptes-survey/paws + + +echo "Please logout and log back in to being using Docker as ${PANUSER}" From 2056057cf8b76782e4df0b2f0897722efdabe1df Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Mon, 15 Apr 2019 09:06:48 +1000 Subject: [PATCH 044/105] Only submit coverage if running from travis --- scripts/testing/run_tests.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/testing/run_tests.sh b/scripts/testing/run_tests.sh index 9539e2ea2..304db4120 100755 --- a/scripts/testing/run_tests.sh +++ b/scripts/testing/run_tests.sh @@ -4,4 +4,9 @@ export PYTHONPATH="$PYTHONPATH:$PANDIR/POCS/scripts/coverage" export COVERAGE_PROCESS_START=.coveragerc coverage run $(which pytest) -v --test-databases all coverage combine -bash <(curl -s https://codecov.io/bash) \ No newline at end of file + +# Only worry about coverage if on travis. +if [[ $TRAVIS ]]; then + coverage combine + bash <(curl -s https://codecov.io/bash) +fi \ No newline at end of file From b7e08576373fd324ecbe577a1469eb7ff5bc9d27 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Mon, 15 Apr 2019 09:10:16 +1000 Subject: [PATCH 045/105] Excplicitly load panoptes-env --- .travis.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 15b79bc28..c271cbbd8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,15 @@ language: python +python: + - "3.6" services: - - docker +- docker before_install: - - ci_env=`bash <(curl -s https://codecov.io/env)` - - docker pull gcr.io/panoptes-survey/pocs -install: true # skips install +- ci_env=`bash <(curl -s https://codecov.io/env)` +- docker pull gcr.io/panoptes-survey/pocs +install: true script: - - docker run -it -v $TRAVIS_BUILD_DIR:/var/panoptes/POCS $ci_env gcr.io/panoptes-survey/pocs scripts/testing/run_tests.sh \ No newline at end of file +- docker run -it + $ci_env + -v $TRAVIS_BUILD_DIR:/var/panoptes/POCS + gcr.io/panoptes-survey/pocs + /bin/bash -c "source activate panoptes-env && scripts/testing/run_tests.sh" From 12a3c3b8f439d8fe33d71fcda2cebad2b5cd6d94 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Mon, 15 Apr 2019 09:29:55 +1000 Subject: [PATCH 046/105] Specific pycodestyle version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7995fa48b..7bca6e923 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ matplotlib mocket numpy psycopg2-binary -pycodestyle +pycodestyle==2.3.1 pyserial pytest python_dateutil From 5819d0365646f499ed38a95196e3da255f694c86 Mon Sep 17 00:00:00 2001 From: Wilfred Gee Date: Tue, 16 Apr 2019 07:33:17 +1000 Subject: [PATCH 047/105] Simplifying scripts. --- scripts/run_messaging_hub.py | 2 +- scripts/run_social_messaging.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/run_messaging_hub.py b/scripts/run_messaging_hub.py index e97b553bb..bea0b7177 100755 --- a/scripts/run_messaging_hub.py +++ b/scripts/run_messaging_hub.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python import argparse import sys diff --git a/scripts/run_social_messaging.py b/scripts/run_social_messaging.py index e79626798..fc4cb24c2 100755 --- a/scripts/run_social_messaging.py +++ b/scripts/run_social_messaging.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python import argparse import sys From 6b937a7b25a54594d5800e8ce1fb9d5bb9f04790 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Tue, 16 Apr 2019 08:37:36 +1000 Subject: [PATCH 048/105] Install docker-compose along with Docker --- scripts/install/install-pocs.sh | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/scripts/install/install-pocs.sh b/scripts/install/install-pocs.sh index 04d4842ff..aaf074332 100755 --- a/scripts/install/install-pocs.sh +++ b/scripts/install/install-pocs.sh @@ -23,7 +23,7 @@ done echo "USER: ${PANUSER}" echo "DIR: ${PANDIR}" -echo "Creating directories" +echo "Creating directories." # Make directories sudo mkdir -p ${PANDIR} sudo chown -R ${PANUSER}:${PANUSER} ${PANDIR} @@ -32,15 +32,15 @@ mkdir -p ${PANDIR}/logs mkdir -p ${PANDIR}/conf_files mkdir -p ${PANDIR}/images -echo "Log files will be stored in ${PANDIR}/logs/install-pocs.log" +echo "Log files will be stored in ${PANDIR}/logs/install-pocs.log." # apt: git, wget echo "Installing system dependencies" sudo apt update &>> ${PANDIR}/logs/install-pocs.log sudo apt --yes install wget git &>> ${PANDIR}/logs/install-pocs.log -echo "Cloning PANOPTES source code" -echo "Github user for PANOPTES repos (POCS, PAWS, panoptes-utils)" +echo "Cloning PANOPTES source code." +echo "Github user for PANOPTES repos (POCS, PAWS, panoptes-utils)." read -p "Github User: [panoptes] " github_user cd ${PANDIR} @@ -52,7 +52,7 @@ for repo in "${repos[@]}"; do echo "Cloning ${repo}" git clone https://github.com/${github_user}/${repo}.git &>> ${PANDIR}/logs/install-pocs.log else - echo "Repo ${repo} already exists on system" + echo "Repo ${repo} already exists on system." fi done @@ -61,11 +61,14 @@ if ! hash docker; then echo "Installing Docker" sh -c "$(wget https://get.docker.com -O -)" sudo usermod -aG docker ${PANUSER} + # Docker compose as container - https://docs.docker.com/compose/install/#install-compose + sudo curl -L --fail https://github.com/docker/compose/releases/download/1.24.0/run.sh -o /usr/local/bin/docker-compose + sudo chmod +x /usr/local/bin/docker-compose + sudo chmod a+x /usr/local/bin/docker-compose fi -echo "Pulling POCS docker images" +echo "Pulling POCS docker images." sudo docker pull gcr.io/panoptes-survey/pocs sudo docker pull gcr.io/panoptes-survey/paws - -echo "Please logout and log back in to being using Docker as ${PANUSER}" +echo "Please logout and log back in to being using Docker as ${PANUSER}." From b64f6c0235060b4273cb298864b22675b78754aa Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Sun, 21 Apr 2019 19:15:12 +1000 Subject: [PATCH 049/105] Being a bit smarter about not creating the directories if already exist. Note that this script still assumes it is only run once. --- scripts/install/install-pocs.sh | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/scripts/install/install-pocs.sh b/scripts/install/install-pocs.sh index 04d4842ff..794db0083 100755 --- a/scripts/install/install-pocs.sh +++ b/scripts/install/install-pocs.sh @@ -1,7 +1,13 @@ #!/bin/bash -e -PANUSER='panoptes' -PANDIR='/var/panoptes' +if [ -z ${PANUSER} ]; then + export PANUSER=$USER + echo "export PANUSER=${PANUSER}" >> ${HOME}/.bashrc +fi +if [ -z ${PANDIR} ]; then + export PANDIR='/var/panoptes' + echo "export PANDIR=${PANDIR}" >> ${HOME}/.bashrc +fi while [[ $# -gt 0 ]] do @@ -23,14 +29,16 @@ done echo "USER: ${PANUSER}" echo "DIR: ${PANDIR}" -echo "Creating directories" -# Make directories -sudo mkdir -p ${PANDIR} -sudo chown -R ${PANUSER}:${PANUSER} ${PANDIR} +if [[ ! -d ${PANDIR} ]] || [[ $(stat -c "%U" ${PANDIR}) -ne $USER ]]; then + echo "Creating directories" + # Make directories + sudo mkdir -p ${PANDIR} + sudo chown -R ${PANUSER}:${PANUSER} ${PANDIR} -mkdir -p ${PANDIR}/logs -mkdir -p ${PANDIR}/conf_files -mkdir -p ${PANDIR}/images + mkdir -p ${PANDIR}/logs + mkdir -p ${PANDIR}/conf_files + mkdir -p ${PANDIR}/images +fi echo "Log files will be stored in ${PANDIR}/logs/install-pocs.log" @@ -49,8 +57,9 @@ declare -a repos=("POCS" "PAWS" "panoptes-utils") for repo in "${repos[@]}"; do if [ ! -d "${PANDIR}/${repo}" ]; then - echo "Cloning ${repo}" - git clone https://github.com/${github_user}/${repo}.git &>> ${PANDIR}/logs/install-pocs.log + echo "Cloning ${repo}" + # Just redirect the errors because otherwise looks like it hangs. + git clone https://github.com/${github_user}/${repo}.git else echo "Repo ${repo} already exists on system" fi From c2c9fc1cdf76899c6bf8410ff269e4c761443efe Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Sun, 21 Apr 2019 20:53:33 +1000 Subject: [PATCH 050/105] * Pull the google-sdk docker image. * Change the final message to user. --- scripts/install/install-pocs.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/install/install-pocs.sh b/scripts/install/install-pocs.sh index 794db0083..98b4c3236 100755 --- a/scripts/install/install-pocs.sh +++ b/scripts/install/install-pocs.sh @@ -73,8 +73,8 @@ if ! hash docker; then fi echo "Pulling POCS docker images" +sudo docker pull google/cloud-sdk:latest sudo docker pull gcr.io/panoptes-survey/pocs sudo docker pull gcr.io/panoptes-survey/paws - -echo "Please logout and log back in to being using Docker as ${PANUSER}" +echo "You must logout and log back in to before using POCS." From c5f4383c880f8b0bb50b6db0d7ef8de6dcc3ee39 Mon Sep 17 00:00:00 2001 From: Wilfred Gee Date: Mon, 22 Apr 2019 17:31:25 +1000 Subject: [PATCH 051/105] Removing docker-compose --- Dockerfile | 2 +- requirements.txt | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6ff7e3895..286a1dd86 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,4 +32,4 @@ RUN apt-get update \ WORKDIR ${POCS} -CMD ["/bin/zsh"] \ No newline at end of file +CMD ["/bin/zsh"] diff --git a/requirements.txt b/requirements.txt index 7bca6e923..641f4804d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,6 @@ astroplan astropy dateparser -docker-compose Flask jupyter-console jupyterlab From 664ed312f273f33ed851e487799315525c466839 Mon Sep 17 00:00:00 2001 From: Wilfred Gee Date: Mon, 22 Apr 2019 20:15:27 +1000 Subject: [PATCH 052/105] * Make PANDIR configurable * Actually install POCS module * ENTRYPOINT should activate environment (panoptes-env). --- Dockerfile | 15 ++++++++++----- resources/docker_files/docker-compose.yaml | 5 ++++- resources/docker_files/docker-entrypoint.sh | 9 +++++++++ 3 files changed, 23 insertions(+), 6 deletions(-) create mode 100755 resources/docker_files/docker-entrypoint.sh diff --git a/Dockerfile b/Dockerfile index 286a1dd86..52a2c9000 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,10 +3,10 @@ ARG arch=amd64 FROM gcr.io/panoptes-survey/panoptes-utils:$arch MAINTAINER Developers for PANOPTES project -ARG pocs_dir=POCS +ARG pandir=/var/panoptes -ENV PANDIR /var/panoptes -ENV POCS ${PANDIR}/${pocs_dir} +ENV PANDIR $pan_dir +ENV POCS ${PANDIR}/POCS COPY . ${POCS} @@ -27,9 +27,14 @@ RUN apt-get update \ && rm gphoto2-updater.sh \ # POCS && cd ${POCS} \ - && ln -s ${POCS}/conf_files/ ${PANDIR}/ \ - && /opt/conda/envs/panoptes-env/bin/pip install --no-cache-dir -r requirements.txt + && /opt/conda/envs/panoptes-env/bin/pip install --no-cache-dir -r requirements.txt \ + && /opt/conda/envs/panoptes-env/bin/pip install -e . \ + # Link conf_files to $PANDIR + && ln -s ${POCS}/conf_files/ ${PANDIR}/ WORKDIR ${POCS} +ENTRYPOINT ["${POCS}/resources/docker_files/docker-entrypoint.sh"] + CMD ["/bin/zsh"] + diff --git a/resources/docker_files/docker-compose.yaml b/resources/docker_files/docker-compose.yaml index d0e43605c..6b6698f67 100644 --- a/resources/docker_files/docker-compose.yaml +++ b/resources/docker_files/docker-compose.yaml @@ -4,11 +4,13 @@ services: image: gcr.io/panoptes-survey/pocs container_name: pocs privileged: true + network_mode: host env_file: - ${PANDIR}/env_file depends_on: - "config-server" - "messaging" + - "peas-shell" volumes: - pandir:/var/panoptes # No-op to keep machine running @@ -19,4 +21,5 @@ volumes: driver_opts: type: none device: /var/panoptes - o: bind \ No newline at end of file + o: bind + diff --git a/resources/docker_files/docker-entrypoint.sh b/resources/docker_files/docker-entrypoint.sh new file mode 100755 index 000000000..e7a06a56a --- /dev/null +++ b/resources/docker_files/docker-entrypoint.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -e + +source /opt/conda/etc/profile.d/conda.sh +conda activate panoptes-env + +# Run the remaining comands +exec "$@" + From 6c621a532eb817521999c6dbf5f5863d05d2f0f3 Mon Sep 17 00:00:00 2001 From: Wilfred Gee Date: Mon, 22 Apr 2019 21:09:49 +1000 Subject: [PATCH 053/105] * Fix bad ARG variable name. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 52a2c9000..09dc7dc0f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ MAINTAINER Developers for PANOPTES project ARG pandir=/var/panoptes -ENV PANDIR $pan_dir +ENV PANDIR $pandir ENV POCS ${PANDIR}/POCS COPY . ${POCS} From 51b7be36721d416efc3c35c5623de10e21946dde Mon Sep 17 00:00:00 2001 From: Wilfred Gee Date: Tue, 23 Apr 2019 08:04:22 +1000 Subject: [PATCH 054/105] * Add pocs-shell for docker-compose --- resources/docker_files/docker-compose.yaml | 26 ++++++++++++++-------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/resources/docker_files/docker-compose.yaml b/resources/docker_files/docker-compose.yaml index 6b6698f67..62fadf8cf 100644 --- a/resources/docker_files/docker-compose.yaml +++ b/resources/docker_files/docker-compose.yaml @@ -1,20 +1,28 @@ version: '3.2' services: - pocs: + peas-shell: image: gcr.io/panoptes-survey/pocs - container_name: pocs + container_name: peas-shell privileged: true network_mode: host - env_file: - - ${PANDIR}/env_file + env_file: ${PANDIR}/env_file depends_on: - - "config-server" - - "messaging" - - "peas-shell" + - "messaging-hub" volumes: - pandir:/var/panoptes - # No-op to keep machine running - command: tail -f /dev/null + command: /usr/bin/byobu new-session -s "POCS-Session" "/opt/conda/envs/panoptes-env/bin/python ${POCS}/scripts/peas-shell.py" + pocs-shell: + image: gcr.io/panoptes-survey/pocs + container_name: pocs-shell + privileged: true + network_mode: host + env_file: ${PANDIR}/env_file + depends_on: + - "peas-shell" + volumes: + - pandir:/var/panoptes + # command: /usr/bin/byobu new-session -s "POCS-Session" "/opt/conda/envs/panoptes-env/bin/python ${POCS}/scripts/pocs-shell.py" + command: tail -f /dev/null volumes: pandir: driver: local From 164710bd2afd3af6bb51458d6cae59b3fb1bcfe2 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Tue, 23 Apr 2019 13:14:55 +1000 Subject: [PATCH 055/105] ENTRYPOINT does not know about env vars --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 09dc7dc0f..4352606c4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,7 +34,7 @@ RUN apt-get update \ WORKDIR ${POCS} -ENTRYPOINT ["${POCS}/resources/docker_files/docker-entrypoint.sh"] +ENTRYPOINT ["resources/docker_files/docker-entrypoint.sh"] CMD ["/bin/zsh"] From 66071454e7db3a1364d10748b38361883435c83f Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Mon, 29 Apr 2019 09:46:02 +1000 Subject: [PATCH 056/105] Fixing add --- scripts/testing/run_tests.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/testing/run_tests.sh b/scripts/testing/run_tests.sh index 304db4120..0504abad0 100755 --- a/scripts/testing/run_tests.sh +++ b/scripts/testing/run_tests.sh @@ -3,10 +3,11 @@ export PYTHONPATH="$PYTHONPATH:$PANDIR/POCS/scripts/coverage" export COVERAGE_PROCESS_START=.coveragerc coverage run $(which pytest) -v --test-databases all -coverage combine # Only worry about coverage if on travis. if [[ $TRAVIS ]]; then coverage combine bash <(curl -s https://codecov.io/bash) -fi \ No newline at end of file +fi + +exit 0 From cf510d5dd33569cd141121fc5fc263b1132fbef4 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Tue, 7 May 2019 06:55:10 +1000 Subject: [PATCH 057/105] Make the startup script more generic; commands at end are simply passed to `docker-compose`. This script basically is just convenience for joining the seperate compose files. --- scripts/start_pocs_docker.sh | 45 ++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/scripts/start_pocs_docker.sh b/scripts/start_pocs_docker.sh index c0b501aea..3a52aa0ae 100755 --- a/scripts/start_pocs_docker.sh +++ b/scripts/start_pocs_docker.sh @@ -6,29 +6,34 @@ usage() { # ################################################## - $ $(basename $0) [START] + $ $(basename $0) [COMMAND] Options: - START Program to start. Currently only option is 'jupyterlab'. If no - option is given then start the state machine in the background. + COMMAND These options are passed at the end of the docker-compose command. + To start all service simply pass 'up'. + + Examples: + + # Start all services. + $POCS/scripts/start-pocs-docker.sh up + + # Start config-server and messaging-hub serivces in the background. + $POCS/scripts/start-pocs-docker.sh up --no-deps -d config-server messaging-hub + + # Read the logs from the config-server + $POCS/scripts/start-pocs-docker.sh logs config-server " } -if [ $# -eq 0 ]; then - docker-compose \ - -f $POCS/resources/docker_files/docker-compose.yml \ - -p panoptes up -else - START=${1} - - if [ ${START} = 'help' ] || [ ${START} = '-h' ] || [ ${START} = '--help' ]; then - usage - exit 1 - else - docker-compose \ - -f $POCS/resources/docker_files/docker-compose.yml \ - -f $POCS/resources/docker_files/docker-compose.${START}.yml \ - -p panoptes up - fi - +START=${1:-help} +if [ ${START} = 'help' ] || [ ${START} = '-h' ] || [ ${START} = '--help' ]; then + usage + exit 1 fi + +docker-compose \ + -f $PANDIR/panoptes-utils/docker/docker-compose.yaml \ + -f $PANDIR/PAWS/docker/docker-compose.yaml \ + -f $PANDIR/POCS/resources/docker_files/docker-compose.yaml \ + -p panoptes "$@" + From f3f7b59495c9cd709ffb440f938fe9186d81124b Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Tue, 7 May 2019 07:03:52 +1000 Subject: [PATCH 058/105] docker-compose file will start a container for `peas-shell` and `pocs-shell` respectively but not actually do anything with them. The idea is to keep them persistent although this could be revisited later. The bin/ scripts then actually execute the respective shells on the containers when called by a user. --- bin/peas | 4 ++-- bin/pocs | 4 ++-- resources/docker_files/docker-compose.yaml | 5 +++-- scripts/{start_pocs_docker.sh => start-pocs-docker.sh} | 0 4 files changed, 7 insertions(+), 6 deletions(-) rename scripts/{start_pocs_docker.sh => start-pocs-docker.sh} (100%) diff --git a/bin/peas b/bin/peas index 5d619a0a2..a0a81443c 100755 --- a/bin/peas +++ b/bin/peas @@ -1,3 +1,3 @@ -#!/bin/bash +#!/bin/bash -e -docker exec -it panoptes_pocs_1 $POCS/scripts/peas_shell \ No newline at end of file +docker exec -it peas-shell /opt/conda/envs/panoptes-env/bin/python $POCS/scripts/peas-shell.py \ No newline at end of file diff --git a/bin/pocs b/bin/pocs index d5f1dbd7f..9bccb450a 100755 --- a/bin/pocs +++ b/bin/pocs @@ -1,3 +1,3 @@ -#!/bin/bash +#!/bin/bash -e -docker exec -it panoptes_pocs_1 $POCS/scripts/pocs_shell \ No newline at end of file +docker exec -it peas-shell /opt/conda/envs/panoptes-env/bin/python $POCS/scripts/pocs-shell.py \ No newline at end of file diff --git a/resources/docker_files/docker-compose.yaml b/resources/docker_files/docker-compose.yaml index 62fadf8cf..3c8a40c04 100644 --- a/resources/docker_files/docker-compose.yaml +++ b/resources/docker_files/docker-compose.yaml @@ -10,7 +10,8 @@ services: - "messaging-hub" volumes: - pandir:/var/panoptes - command: /usr/bin/byobu new-session -s "POCS-Session" "/opt/conda/envs/panoptes-env/bin/python ${POCS}/scripts/peas-shell.py" + # No-op to keep machine running, use $POCS/bin/peas to access + command: tail -f /dev/null pocs-shell: image: gcr.io/panoptes-survey/pocs container_name: pocs-shell @@ -21,7 +22,7 @@ services: - "peas-shell" volumes: - pandir:/var/panoptes - # command: /usr/bin/byobu new-session -s "POCS-Session" "/opt/conda/envs/panoptes-env/bin/python ${POCS}/scripts/pocs-shell.py" + # No-op to keep machine running, use $POCS/bin/pocs to access command: tail -f /dev/null volumes: pandir: diff --git a/scripts/start_pocs_docker.sh b/scripts/start-pocs-docker.sh similarity index 100% rename from scripts/start_pocs_docker.sh rename to scripts/start-pocs-docker.sh From 9622f325b84ebae3d60dc76686992d6b8e69f464 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Tue, 7 May 2019 10:18:18 +1000 Subject: [PATCH 059/105] Slight cleanup --- cloudbuild.yaml | 2 ++ Dockerfile => resources/docker_files/Dockerfile | 0 scripts/start-pocs-docker.sh | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) rename Dockerfile => resources/docker_files/Dockerfile (100%) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 121b6b807..2ba7baf19 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -15,6 +15,7 @@ steps: id: 'amd64' args: - 'build' + - '-f=resources/docker_files/Dockerfile' - '--build-arg=arch=amd64' - '--tag=gcr.io/${PROJECT_ID}/pocs:amd64' - '.' @@ -24,6 +25,7 @@ steps: id: 'arm32v7' args: - 'build' + - '-f=resources/docker_files/Dockerfile' - '--build-arg=arch=arm32v7' - '--tag=gcr.io/${PROJECT_ID}/pocs:arm32v7' - '.' diff --git a/Dockerfile b/resources/docker_files/Dockerfile similarity index 100% rename from Dockerfile rename to resources/docker_files/Dockerfile diff --git a/scripts/start-pocs-docker.sh b/scripts/start-pocs-docker.sh index 3a52aa0ae..6d610268d 100755 --- a/scripts/start-pocs-docker.sh +++ b/scripts/start-pocs-docker.sh @@ -14,7 +14,7 @@ usage() { Examples: - # Start all services. + # Start all services in the foreground. $POCS/scripts/start-pocs-docker.sh up # Start config-server and messaging-hub serivces in the background. From a60016cbaf5d85d41a276721b1fdd94e3104c76a Mon Sep 17 00:00:00 2001 From: Wilfred Gee Date: Tue, 7 May 2019 15:51:22 +1000 Subject: [PATCH 060/105] Change cr2 script to use new split dir --- pocs/utils/images/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pocs/utils/images/__init__.py b/pocs/utils/images/__init__.py index 534542ce4..2e55c7b26 100644 --- a/pocs/utils/images/__init__.py +++ b/pocs/utils/images/__init__.py @@ -215,7 +215,7 @@ def _make_pretty_from_fits(fname=None, def _make_pretty_from_cr2(fname, title=None, timeout=15, **kwargs): verbose = kwargs.get('verbose', False) - script_name = os.path.join(os.getenv('POCS'), 'scripts', 'cr2_to_jpg.sh') + script_name = os.path.join(os.getenv('PANDIR'), 'panoptes-utils', 'scripts', 'cr2_to_jpg.sh') cmd = [script_name, fname] if title: From 1f0aab3de5cde1dda9b619dda6ab04e94886ea76 Mon Sep 17 00:00:00 2001 From: Wilfred Gee Date: Sat, 11 May 2019 17:58:17 +1000 Subject: [PATCH 061/105] Use correct pip --- resources/docker_files/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/resources/docker_files/Dockerfile b/resources/docker_files/Dockerfile index 4352606c4..1d2f7838b 100644 --- a/resources/docker_files/Dockerfile +++ b/resources/docker_files/Dockerfile @@ -4,6 +4,7 @@ FROM gcr.io/panoptes-survey/panoptes-utils:$arch MAINTAINER Developers for PANOPTES project ARG pandir=/var/panoptes +ARG which_pip=pip ENV PANDIR $pandir ENV POCS ${PANDIR}/POCS @@ -27,8 +28,8 @@ RUN apt-get update \ && rm gphoto2-updater.sh \ # POCS && cd ${POCS} \ - && /opt/conda/envs/panoptes-env/bin/pip install --no-cache-dir -r requirements.txt \ - && /opt/conda/envs/panoptes-env/bin/pip install -e . \ + && $which_pip install --no-cache-dir -r requirements.txt \ + && $which_pip install -e . \ # Link conf_files to $PANDIR && ln -s ${POCS}/conf_files/ ${PANDIR}/ From 33430415ff3606662666e383b6b0ca53ef56b0e1 Mon Sep 17 00:00:00 2001 From: Wilfred Gee Date: Sat, 11 May 2019 17:58:50 +1000 Subject: [PATCH 062/105] Use correct pip --- cloudbuild.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 2ba7baf19..91dde0469 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -7,10 +7,10 @@ steps: - '--privileged' - 'multiarch/qemu-user-static:register' - '--reset' - waitFor: ['-'] + waitFor: ['-'] # Build -# AMD Build +# AMD Build - name: 'gcr.io/cloud-builders/docker' id: 'amd64' args: @@ -20,13 +20,14 @@ steps: - '--tag=gcr.io/${PROJECT_ID}/pocs:amd64' - '.' waitFor: ['register-qemu'] -# ARM Build (e.g. Raspberry Pi) +# ARM Build (e.g. Raspberry Pi) - name: 'gcr.io/cloud-builders/docker' id: 'arm32v7' args: - 'build' - '-f=resources/docker_files/Dockerfile' - '--build-arg=arch=arm32v7' + - '--build-arg=which_pip=/opt/conda/envs/panoptes-env/bin/pip' - '--tag=gcr.io/${PROJECT_ID}/pocs:arm32v7' - '.' waitFor: ['register-qemu'] @@ -55,7 +56,7 @@ steps: - 'create' - 'gcr.io/${PROJECT_ID}/pocs:latest' - 'gcr.io/${PROJECT_ID}/pocs:arm32v7' - - 'gcr.io/${PROJECT_ID}/pocs:amd64' + - 'gcr.io/${PROJECT_ID}/pocs:amd64' waitFor: ['push-amd64', 'push-arm'] - name: 'gcr.io/cloud-builders/docker' From acc7c2d1b7c8b3bf8ea297bc925e30fc0d20787a Mon Sep 17 00:00:00 2001 From: Wilfred Gee Date: Sat, 11 May 2019 18:55:55 +1000 Subject: [PATCH 063/105] Rename file --- scripts/{start-pocs-docker.sh => pocs-docker.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/{start-pocs-docker.sh => pocs-docker.sh} (100%) diff --git a/scripts/start-pocs-docker.sh b/scripts/pocs-docker.sh similarity index 100% rename from scripts/start-pocs-docker.sh rename to scripts/pocs-docker.sh From 8b4f4e1ef61568b83127bc383bd1fbf936014816 Mon Sep 17 00:00:00 2001 From: Wilfred Gee Date: Sat, 11 May 2019 19:08:57 +1000 Subject: [PATCH 064/105] Fixing travis run --- .travis.yml | 6 +++--- resources/docker_files/Dockerfile | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index c271cbbd8..e52ca7e97 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,8 +8,8 @@ before_install: - docker pull gcr.io/panoptes-survey/pocs install: true script: -- docker run -it - $ci_env +- docker run -it + $ci_env -v $TRAVIS_BUILD_DIR:/var/panoptes/POCS gcr.io/panoptes-survey/pocs - /bin/bash -c "source activate panoptes-env && scripts/testing/run_tests.sh" + /bin/bash -c "scripts/testing/run_tests.sh" diff --git a/resources/docker_files/Dockerfile b/resources/docker_files/Dockerfile index 1d2f7838b..a865bee04 100644 --- a/resources/docker_files/Dockerfile +++ b/resources/docker_files/Dockerfile @@ -29,7 +29,7 @@ RUN apt-get update \ # POCS && cd ${POCS} \ && $which_pip install --no-cache-dir -r requirements.txt \ - && $which_pip install -e . \ + && $which_pip install --no-cache-dir -e . \ # Link conf_files to $PANDIR && ln -s ${POCS}/conf_files/ ${PANDIR}/ From 402768880e9b81cac95766323d2af79997362b01 Mon Sep 17 00:00:00 2001 From: Wilfred Gee Date: Sat, 11 May 2019 19:13:39 +1000 Subject: [PATCH 065/105] Fixing conda paths (or lack thereof) --- bin/peas | 2 +- bin/pocs | 2 +- resources/docker_files/Dockerfile | 2 -- resources/docker_files/docker-entrypoint.sh | 4 ++-- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/bin/peas b/bin/peas index a0a81443c..14e0a8d4e 100755 --- a/bin/peas +++ b/bin/peas @@ -1,3 +1,3 @@ #!/bin/bash -e -docker exec -it peas-shell /opt/conda/envs/panoptes-env/bin/python $POCS/scripts/peas-shell.py \ No newline at end of file +docker exec -it peas-shell python $POCS/scripts/peas-shell.py diff --git a/bin/pocs b/bin/pocs index 9bccb450a..89dadceae 100755 --- a/bin/pocs +++ b/bin/pocs @@ -1,3 +1,3 @@ #!/bin/bash -e -docker exec -it peas-shell /opt/conda/envs/panoptes-env/bin/python $POCS/scripts/pocs-shell.py \ No newline at end of file +docker exec -it peas-shell python $POCS/scripts/pocs-shell.py diff --git a/resources/docker_files/Dockerfile b/resources/docker_files/Dockerfile index a865bee04..fcfbe6c00 100644 --- a/resources/docker_files/Dockerfile +++ b/resources/docker_files/Dockerfile @@ -35,7 +35,5 @@ RUN apt-get update \ WORKDIR ${POCS} -ENTRYPOINT ["resources/docker_files/docker-entrypoint.sh"] - CMD ["/bin/zsh"] diff --git a/resources/docker_files/docker-entrypoint.sh b/resources/docker_files/docker-entrypoint.sh index e7a06a56a..b9a929361 100755 --- a/resources/docker_files/docker-entrypoint.sh +++ b/resources/docker_files/docker-entrypoint.sh @@ -1,8 +1,8 @@ #!/bin/bash set -e -source /opt/conda/etc/profile.d/conda.sh -conda activate panoptes-env +# source /opt/conda/etc/profile.d/conda.sh +# conda activate panoptes-env # Run the remaining comands exec "$@" From 6b01c0105c11edbb27e1ffdb6d6757365a6f5509 Mon Sep 17 00:00:00 2001 From: Wilfred Gee Date: Sat, 11 May 2019 19:23:14 +1000 Subject: [PATCH 066/105] Move docker related files to consistent location with other repos --- {resources/docker_files => docker}/Dockerfile | 0 cloudbuild.yaml => docker/cloudbuild.yaml | 0 {resources/docker_files => docker}/docker-compose.yaml | 0 {resources/docker_files => docker}/docker-entrypoint.sh | 0 {resources/docker_files => docker}/env_file | 0 resources/docker_files/docker-compose.jupyterlab.yml | 9 --------- 6 files changed, 9 deletions(-) rename {resources/docker_files => docker}/Dockerfile (100%) rename cloudbuild.yaml => docker/cloudbuild.yaml (100%) rename {resources/docker_files => docker}/docker-compose.yaml (100%) rename {resources/docker_files => docker}/docker-entrypoint.sh (100%) rename {resources/docker_files => docker}/env_file (100%) delete mode 100644 resources/docker_files/docker-compose.jupyterlab.yml diff --git a/resources/docker_files/Dockerfile b/docker/Dockerfile similarity index 100% rename from resources/docker_files/Dockerfile rename to docker/Dockerfile diff --git a/cloudbuild.yaml b/docker/cloudbuild.yaml similarity index 100% rename from cloudbuild.yaml rename to docker/cloudbuild.yaml diff --git a/resources/docker_files/docker-compose.yaml b/docker/docker-compose.yaml similarity index 100% rename from resources/docker_files/docker-compose.yaml rename to docker/docker-compose.yaml diff --git a/resources/docker_files/docker-entrypoint.sh b/docker/docker-entrypoint.sh similarity index 100% rename from resources/docker_files/docker-entrypoint.sh rename to docker/docker-entrypoint.sh diff --git a/resources/docker_files/env_file b/docker/env_file similarity index 100% rename from resources/docker_files/env_file rename to docker/env_file diff --git a/resources/docker_files/docker-compose.jupyterlab.yml b/resources/docker_files/docker-compose.jupyterlab.yml deleted file mode 100644 index ca9250ec0..000000000 --- a/resources/docker_files/docker-compose.jupyterlab.yml +++ /dev/null @@ -1,9 +0,0 @@ -version: "3.2" -services: - pocs: - command: bash -c "jupyter-lab --ip 0.0.0.0 --allow-root --no-browser" - ports: - - target: 8888 - published: 9000 - protocol: tcp - mode: host \ No newline at end of file From 95d7fbf8da449cc967262ceb5c6e5b050dd1e0cd Mon Sep 17 00:00:00 2001 From: Wilfred Gee Date: Sat, 11 May 2019 19:23:35 +1000 Subject: [PATCH 067/105] Update buildfile to match --- scripts/build_containers.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_containers.sh b/scripts/build_containers.sh index 7380878ab..2ce26ddd1 100755 --- a/scripts/build_containers.sh +++ b/scripts/build_containers.sh @@ -1,5 +1,5 @@ #!/bin/bash -e SOURCE_DIR=${POCS} -gcloud builds submit --timeout="5h" --config ${SOURCE_DIR}/cloudbuild.yaml --async ${SOURCE_DIR} +gcloud builds submit --timeout="5h" --config ${SOURCE_DIR}/docker/cloudbuild.yaml --async ${SOURCE_DIR} From f8527c98f2f1af74dab68e061a14d7fe054469e9 Mon Sep 17 00:00:00 2001 From: Wilfred Gee Date: Sat, 11 May 2019 19:26:16 +1000 Subject: [PATCH 068/105] Removing old startup scripts --- docker/docker-entrypoint.sh | 9 -- scripts/startup/README.md | 159 ---------------------- scripts/startup/set_panoptes_env_vars.sh | 12 -- scripts/startup/start_log_viewer.sh | 17 --- scripts/startup/start_messaging_hub.sh | 11 -- scripts/startup/start_panoptes_in_tmux.sh | 128 ----------------- scripts/startup/start_paws.sh | 12 -- scripts/startup/start_peas.sh | 26 ---- scripts/startup/start_pocs.sh | 18 --- scripts/startup/start_social_messaging.sh | 11 -- scripts/startup/su_panoptes.sh | 27 ---- scripts/startup/tmux_launch.sh | 81 ----------- 12 files changed, 511 deletions(-) delete mode 100755 docker/docker-entrypoint.sh delete mode 100644 scripts/startup/README.md delete mode 100644 scripts/startup/set_panoptes_env_vars.sh delete mode 100755 scripts/startup/start_log_viewer.sh delete mode 100755 scripts/startup/start_messaging_hub.sh delete mode 100755 scripts/startup/start_panoptes_in_tmux.sh delete mode 100755 scripts/startup/start_paws.sh delete mode 100755 scripts/startup/start_peas.sh delete mode 100755 scripts/startup/start_pocs.sh delete mode 100755 scripts/startup/start_social_messaging.sh delete mode 100644 scripts/startup/su_panoptes.sh delete mode 100755 scripts/startup/tmux_launch.sh diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh deleted file mode 100755 index b9a929361..000000000 --- a/docker/docker-entrypoint.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -set -e - -# source /opt/conda/etc/profile.d/conda.sh -# conda activate panoptes-env - -# Run the remaining comands -exec "$@" - diff --git a/scripts/startup/README.md b/scripts/startup/README.md deleted file mode 100644 index 30cc9e103..000000000 --- a/scripts/startup/README.md +++ /dev/null @@ -1,159 +0,0 @@ -These scripts support starting the PANOPTES software automatically -when the Linux computer that runs a PANOPTES starts or restarts. -We assume here that you are logged in as user $PANUSER, and that user -$PANUSER has permissions to execute `sudo`. - -> Note: We use `tmux`, a [terminal -> multiplexer](https://en.wikipedia.org/wiki/Terminal_multiplexer), -> to run all of the software. This means that when the computer boots, -> it can create several virtual terminals, each running a separate -> program. Later, when you want to see what those programs are doing, -> you can login and attach to those terminals. With `tmux`, the command -> to do that is: -> ```bash -> tmux attach-session -t panoptes -> ``` - -# PANOPTES Environment Variables - -First we need to have the PANOPTES environement variables set in all -the shells that are involved in running PANOPTES. The simplest -solution is to set them globally on the system. For example, -on Ubuntu: - -```bash -sudo cp $POCS/scripts/startup/set_panoptes_env_vars.sh \ - /etc/profile.d/ -sudo chmod +x /etc/profile.d/set_panoptes_env_vars.sh -``` - -This will then be executed when any user logs in. The exact means -of doing this varies on different versions of Unix/Linux. - -If you've changed the location of the PANOPTES directories (e.g. if -log files are not stored in $PANDIR/logs), you'll need to edit that -script accordingly. - -# Start PANOPTES Software on System Start - -We intend for a PANOPTES telescope to run automatically, including after -reboots. We document these ways to achieve this: - -* `@reboot` rule in crontab of user $PANUSER -* Execution by root during boot - -When you are ready to have the telescope run automatically (i.e. have -done all the manual setup and alignment that you need to), follow one -of these steps to arrange for your telescope to run automatically after -reboot. - -> Note: the EDITOR environment variable allows you to specify the -> editor used by commands such as `crontab`. `nano` is a simple editor -> that runs in almost any terminal window. On some systems, the -> default editor may be `vi` or `vim`, which are rather more difficult -> to use than `nano`, unless you're already familiar with them. - -## Crontab - -[Crontab](https://linux.die.net/man/5/crontab) provides a means to run -a command line at various times. Some versions of crontab have support -for [`@reboot`](https://www.google.com/search?q=crontab+%40reboot) as -a *time* at which to run a command. If your system has this feature, -it is the easiest way to launch the PANOPTES software. - -You can test whether `@reboot` works by adding a rule that -leaves an indication of whether it works. For example: - -```bash -@reboot touch /tmp/at-reboot-worked -``` - -Add the rule to your ($PANUSER) crontab by executing the command -`crontab -e`, which will start an editor on your crontab file. After -you add that rule at the end of the file, save the file and exit the -editor. You can confirm that it was added correctly by viewing the -file with the command `crontab -l`. Next, reboot the computer (e.g. -with the command `sudo shutdown -r now`), log back in, remove that -test rule, and see if `/tmp/at-reboot-worked` was created. If it was, -then `@reboot` works. Otherwise, skip the `@reboot` rule in the -steps that follow, and follow the instructions in -__Execution by `root`__ after editing your crontab. - -Add these lines to your ($PANUSER) crontab by executing the command -`crontab -e`: - -```bash -POCS=/var/panoptes/POCS -PANLOG=/var/panoptes/logs - -# m h dom mon dow command -@reboot /bin/bash --login $POCS/scripts/startup/tmux_launch.sh >> $PANLOG/tmux_launch.cron-reboot.log 2>&1 -*/5 * * * * /bin/bash --login $POCS/scripts/plot_weather.sh >> $PANLOG/plot_weather.cron.log 2>&1 -11 12 * * * /bin/bash --login $POCS/scripts/download_support_files.sh >> $PANLOG/download_support_files.cron.log 2>&1 -``` - -If your values for POCS and PANLOG don't match those shown above, please -edit them appropriately. Use the fully evaluated values (e.g. the -value produced by `echo $POCS` and by `echo $PANLOG`), as `cron` -does not expand variables in those lines. - -Save the edited file and exit the editor. - -Notice that each of these commands starts with `/bin/bash --login`. We -do this because, by default, `cron` runs commands in a very minimal -environment with /bin/sh as the shell. This means that the PANOPTES -environment variables are not set, nor is the appropriate conda -environment activated. The `/bin/bash --login` selects the correct -shell and tells that shell to initialize its environment. - -The other two rules are for generating a plot (image) of the -weather sensor data every 5 minutes and for updating astronomical -coordinate system data every day at 11 minutes after noon. - -## Execution by `root` - -If your crontab does not support the `@reboot` directive, you can modify -the Linux boot scripts to start the PANOPTES software. For that, we -provide the script `su_panoptes.sh`. - -First, copy that file to /etc: - -```bash -sudo cp $POCS/scripts/startup/su_panoptes.sh \ - /etc/ -sudo chmod +x /etc/su_panoptes.sh -``` - -Next, edit `/etc/rc.local`, adding a command to run `su_panoptes.sh`. -For example, adding it to the default Ubuntu `/etc/rc.local`: - -```bash -#!/bin/sh -e -# -# rc.local -# -# This script is executed at the end of each multiuser runlevel. -# Make sure that the script will "exit 0" on success or any other -# value on error. -# -# In order to enable or disable this script just change the execution -# bits. -# -# By default this script does nothing. - -/etc/su_panoptes.sh >> /tmp/su_panoptes.log 2>&1 - -exit 0 -``` - -To edit the file, run this command: - -```bash -sudo nano /etc/rc.local -``` - -As the default text says, you may need to enable that script to execute: - -```bash -sudo chmod +x /etc/rc.local -``` diff --git a/scripts/startup/set_panoptes_env_vars.sh b/scripts/startup/set_panoptes_env_vars.sh deleted file mode 100644 index 61a882bbe..000000000 --- a/scripts/startup/set_panoptes_env_vars.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/sh - -# This file is to be copied into /etc/profile.d/ and SOURCED by -# /etc/rc.local. We have the shebang line at the top to help the -# shellcheck tool (a lint-like tool for shell scripts) recognize -# the type of this script. - -export PANUSER=panoptes # User that runs PANOPTES software. -export PANDIR=/var/panoptes # Main directory. -export PANLOG="${PANDIR}/logs" # Log file storage. -export POCS="${PANDIR}/POCS" # PANOPTES Observatory Control Software -export PAWS="${PANDIR}/PAWS" # PANOPTES Web Interface diff --git a/scripts/startup/start_log_viewer.sh b/scripts/startup/start_log_viewer.sh deleted file mode 100755 index 0d52b8c24..000000000 --- a/scripts/startup/start_log_viewer.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -ex - -WINDOW="${1}" -LOGFILE="${2}" -echo "Running $(basename "${0}") at $(date), WINDOW=${WINDOW}, LOGFILE=${LOGFILE}" - -tmux send-keys -t "${WINDOW}" "date" C-m -sleep 0.5s -tmux send-keys -t "${WINDOW}" "cd \"${PANLOG}\"" C-m -sleep 0.5s -tmux send-keys -t "${WINDOW}" "while [[ ! -f \"${LOGFILE}\" ]] ; do echo \"Waiting for ${LOGFILE} to exist\" ; sleep 1s ; done" C-m -sleep 2s -tmux send-keys -t "${WINDOW}" "less --follow-name \"${LOGFILE}\"" C-m -sleep 2s -tmux send-keys -t "${WINDOW}" "F" - -echo "Done at $(date)" diff --git a/scripts/startup/start_messaging_hub.sh b/scripts/startup/start_messaging_hub.sh deleted file mode 100755 index dc4be5203..000000000 --- a/scripts/startup/start_messaging_hub.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -ex - -WINDOW="${1}" -echo "Running $(basename "${0}") at $(date), WINDOW=${WINDOW}" - -tmux send-keys -t "${WINDOW}" "date" C-m -sleep 0.5s -tmux send-keys -t "${WINDOW}" \ - "python $POCS/scripts/run_messaging_hub.py --from_config" C-m - -echo "Done at $(date)" diff --git a/scripts/startup/start_panoptes_in_tmux.sh b/scripts/startup/start_panoptes_in_tmux.sh deleted file mode 100755 index dd44f674c..000000000 --- a/scripts/startup/start_panoptes_in_tmux.sh +++ /dev/null @@ -1,128 +0,0 @@ -#!/bin/bash -ex - -if [[ -z "$BASH_VERSION" ]] ; then - echo "This script must be run by bash." - exit 1 -fi - -# This script is designed to be run inside tmux, launched by -# tmux_launch.sh. Make sure that is so. -echo "Running ${BASH_SOURCE[0]} at $(date)" -if [[ -z "${STARTUP_LOG_DIR_SLASH}" ]] ; then - echo "This script must be run by tmux_launch.sh" - exit 1 -fi - -# And we need the shell to have the the PANOPTES environment -# setup. -if [[ -z "$PANDIR" || -z "$POCS" || -z "$PANLOG" || -z "$PAWS" ]] ; then - echo "The PANOPTES environment variables must be set." - echo "This script should be run from a login shell." - exit 1 -fi - -LOG_NAME="$(basename "${BASH_SOURCE[0]}" .sh).log" -LOG_FILE="${STARTUP_LOG_DIR_SLASH}${LOG_NAME}" - -echo "Will log to ${LOG_FILE}" - -exec 2> "${LOG_FILE}" # send stderr to a log file -exec 1>&2 # send stdout to the same log file - -set +x -# set +x turns off the verbose logging of each line. This makes -# the following block of echoes easier to read in the log file. - -echo "Running ${BASH_SOURCE[0]} at $(date)" - -# Record important debugging info into the log file. These -# environment variables do NOT all have to be set. Note that -# this script is executed inside of a tmux session, which -# SHOULD be treated as interactive (i.e. have the full .bashrc -# executed). Let's check (PATH and conda info are the giveaway). - -echo "Current dir: $(pwd)" -echo "Current user: $(whoami)" -echo "USER: ${USER}" -echo "LOGNAME: ${LOGNAME}" -echo "PATH: ${PATH}" -echo "PANUSER: ${PANUSER}" -echo "PANDIR: ${PANDIR}" -echo "PANLOG: ${PANLOG}" -echo "POCS: ${POCS}" -echo "PAWS: ${PAWS}" -echo "PIAA: ${PIAA}" -# $- expands to the current option flags as specified upon invocation, by the -# set built-in command, or those set by the shell itself (such as the -i). -echo '$-:' "$-" -# Just in case conda isn't setup as expected, don't die here. -(set +e ; conda info) - -function create_and_init_window() { - local -r WINDOW="${1}" - shift - local -r SCRIPT="${1}" - shift - local -r LOG_NAME="$(basename "${SCRIPT}" .sh)-in-${WINDOW}.log" - local -r LOCAL_LOG_FILE="${STARTUP_LOG_DIR_SLASH}${LOG_NAME}" - - echo "Will run ${SCRIPT} in window ${WINDOW}, logging to ${LOCAL_LOG_FILE}" - - # Create the window. - tmux new-window -n "${WINDOW}" - - # Give the window a little while to start up (i.e. to start bash, and - # for bash to run the initialization script). 8 seconds was chosen - # based on how long it takes on PAN006 for a new tmux window to - # print the bash prompt. - sleep 8s - - # Run the script in a separate process, and prevent it from being - # killed when this script ends. - nohup "./${SCRIPT}" ":${WINDOW}" "${@}" > "${LOCAL_LOG_FILE}" 2>&1 & - - # And give that script a couple of seconds to get started. This - # may not be strictly necessary. - sleep 2s -} - -# Resume verbose logging in order to support debugging these scripts. -set -x - -# We get noisy complaints from astroplan about the IERS Bulletin A -# being too old, and this can cause some concern for those running -# this software. Avoid these by downloading fresh data,; ignore -# errors, such as due to the lack of an internet connection. -(set +e ; python "${POCS}/pocs/utils/data.py") - -# Create a window running the zeromq message forwarders. -# These provide connections between message publishers and subscribers. -create_and_init_window msg_hub start_messaging_hub.sh - -# Create a window running the social messaging program. It subscribes to -# zeromq and posts some messages to Twitter or Slack, if -# conf_files/pocs_local.yaml has the appropriate config values; if not, -# the program just exits. -create_and_init_window social start_social_messaging.sh - -# Start PAWS, the PANOPTES Administrative Web Server. -create_and_init_window paws start_paws.sh - -# Monitor the PEAS log file. -create_and_init_window log_peas start_log_viewer.sh \ - "${PANLOG}/peas_shell-all.log" - -# Monitor the POCS log file. -create_and_init_window log_pocs start_log_viewer.sh \ - "${PANLOG}/pocs_shell-all.log" - -# Start PEAS, the PANOPTES Environmental Analysis System -# (primarily takes care of reading from the sensors and loading -# the data into a Mongo Db). -create_and_init_window peas start_peas.sh - -# Start POCS, the PANOPTES Observatory Control System, -# the main software we're interested in having running. -create_and_init_window pocs start_pocs.sh - -exit diff --git a/scripts/startup/start_paws.sh b/scripts/startup/start_paws.sh deleted file mode 100755 index 428ea7086..000000000 --- a/scripts/startup/start_paws.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -e - -WINDOW="${1}" -echo "Running $(basename "${0}") at $(date), WINDOW=${WINDOW}" - -tmux send-keys -t "${WINDOW}" "date" C-m -sleep 0.5s -tmux send-keys -t "${WINDOW}" "cd \"${PAWS}\"" C-m -sleep 0.5s -tmux send-keys -t "${WINDOW}" "python app.py" C-m - -echo "Done at $(date)" diff --git a/scripts/startup/start_peas.sh b/scripts/startup/start_peas.sh deleted file mode 100755 index 9423df10d..000000000 --- a/scripts/startup/start_peas.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -ex - -WINDOW="${1}" -echo "Running $(basename "${0}") at $(date), WINDOW=${WINDOW}" - -tmux send-keys -t "${WINDOW}" "date" C-m -sleep 0.5s -tmux send-keys -t "${WINDOW}" "cd ${POCS}" C-m -sleep 0.5s -tmux send-keys -t "${WINDOW}" "bin/peas_shell" C-m -sleep 10s -tmux send-keys -t "${WINDOW}" "display_config" C-m -sleep 1s -tmux send-keys -t "${WINDOW}" "load_environment" C-m -sleep 5s -tmux send-keys -t "${WINDOW}" "load_weather" C-m -sleep 5s -tmux send-keys -t "${WINDOW}" "start" C-m -sleep 20s -tmux send-keys -t "${WINDOW}" "status" C-m -sleep 10s -tmux send-keys -t "${WINDOW}" "last_reading environment" C-m -sleep 10s -tmux send-keys -t "${WINDOW}" "last_reading weather" C-m - -echo "Done at $(date)" diff --git a/scripts/startup/start_pocs.sh b/scripts/startup/start_pocs.sh deleted file mode 100755 index af7daed3f..000000000 --- a/scripts/startup/start_pocs.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -ex - -WINDOW="${1}" -echo "Running $(basename "${0}") at $(date), WINDOW=${WINDOW}" - -tmux send-keys -t "${WINDOW}" "date" C-m -sleep 0.5s -tmux send-keys -t "${WINDOW}" "cd ${POCS}" C-m -sleep 0.5s -tmux send-keys -t "${WINDOW}" "bin/pocs_shell" C-m -sleep 10s -tmux send-keys -t "${WINDOW}" "setup_pocs" C-m -sleep 20s -tmux send-keys -t "${WINDOW}" "display_config" C-m -sleep 1s -tmux send-keys -t "${WINDOW}" "run_pocs" C-m - -echo "Done at $(date)" diff --git a/scripts/startup/start_social_messaging.sh b/scripts/startup/start_social_messaging.sh deleted file mode 100755 index 1b9125c78..000000000 --- a/scripts/startup/start_social_messaging.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -ex - -WINDOW="${1}" -echo "Running $(basename "${0}") at $(date), WINDOW=${WINDOW}" - -tmux send-keys -t "${WINDOW}" "date" C-m -sleep 0.5s -tmux send-keys -t "${WINDOW}" \ - "python $POCS/scripts/run_social_messaging.py --from_config" C-m - -echo "Done at $(date)" diff --git a/scripts/startup/su_panoptes.sh b/scripts/startup/su_panoptes.sh deleted file mode 100644 index 1fa456374..000000000 --- a/scripts/startup/su_panoptes.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -ex - -# Start running the script tmux_launch as user $PANUSER in a login -# shell, i.e. one that will run that user's shell initialization -# scripts, such as .profile and/or .bashrc. - -# This script is designed to be run from /etc/rc.local (or similar) -# by the user root, after the PANOPTES environment variables have -# been set. - -# Put the date & time into the log (e.g. /tmp/su_panoptes.log). -echo -echo "Running ${0} at $(/bin/date)" -echo - -# Make sure that the PANOPTES environment variables have been setup -# (e.g. by /etc/profile). -if [[ -z "${POCS}" || -z "${PANUSER}" ]] ; then - echo "The POCS and PANUSER environment variables must be set!" - exit 1 -fi - -# Execute tmux_launch.sh in a login shell for user $PANUSER. -/bin/su --login --command \ - "${POCS}/scripts/startup/tmux_launch.sh" "${PANUSER}" - -echo "Done at $(/bin/date)" diff --git a/scripts/startup/tmux_launch.sh b/scripts/startup/tmux_launch.sh deleted file mode 100755 index 76dd48a51..000000000 --- a/scripts/startup/tmux_launch.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/bin/bash -ex - -# Start the PANOPTES software in a detached tmux (terminal multiplexer) -# session; this enables the PANOPTES unit's owner/administrator to -# attach to that session later, see the output and take control of all -# the shells. You can attach to the session with this command: - -# tmux attach-session -t panoptes - -# This script is intended to be executed in a fully initialized bash -# shell (i.e. one in which .profile, .bashrc, etc. have been executed) -# because those setup the environment variables needed. -# In particular, this script can be run by su_panoptes.sh or by -# a @reboot rule in the crontab of user $PANUSER. - -if [[ -z "$BASH_VERSION" ]] ; then - echo "This script must be run by bash." - exit 1 -fi - -# And we need the shell to have the the PANOPTES environment -# setup. -if [[ -z "$PANDIR" || -z "$POCS" || -z "$PANLOG" || -z "$PAWS" ]] ; then - echo "The PANOPTES environment variables must be set." - echo "This script should be run from a login shell." - exit 1 -fi - -echo "Running ${BASH_SOURCE[0]} at $(date)" - -# cd to the directory of this script, as that is where the -# other start scripts are located. -cd "$(dirname "${BASH_SOURCE[0]}")" - -echo "Setting up logging..." - -# Setup a directory for the log file from this script and from those -# it invokes. By creating a unique directory per startup, we make it -# easier to view the group of files from a single reboot. -export STARTUP_LOG_DIR_SLASH="${PANLOG}/per-run/startup/$(date +%Y%m%d-%H%M%S-%Z)/" -mkdir -p "${STARTUP_LOG_DIR_SLASH}" - -LOG_NAME="$(basename "${BASH_SOURCE[0]}" .sh).log" -LOG_FILE="${STARTUP_LOG_DIR_SLASH}${LOG_NAME}" - -echo "Will log to ${LOG_FILE}" - -exec 2> "${LOG_FILE}" # send stderr to a log file -exec 1>&2 # send stdout to the same log file -set +x - -# Record a bunch of environment variables into the log file. This -# helps us later if we need to debug the execution of this script -# and those it invokes. -echo "Running ${BASH_SOURCE[0]} at $(date)" -echo "Current dir: $(pwd)" -echo "Current user: $(whoami)" -echo "USER: ${USER}" -echo "LOGNAME: ${LOGNAME}" -echo "PATH: ${PATH}" -echo "PANUSER: ${PANUSER}" -echo "PANDIR: ${PANDIR}" -echo "PANLOG: ${PANLOG}" -echo "POCS: ${POCS}" -echo "PAWS: ${PAWS}" -echo "PIAA: ${PIAA}" - -# $- expands to the current option flags as specified upon invocation, by the -# set built-in command, or those set by the shell itself (such as the -i). -echo 'Shell options ($-):' "$-" - -set -x - -# Finally, the point of this script: create a detached (-d) tmux session -# called panoptes (-s), with a scrollback buffer of 5000 lines, where -# the shell used by new-window is bash. -tmux set-option -g history-limit 5000 \; \ - set-option -g default-shell /bin/bash \; \ - new-session -d -s panoptes ./start_panoptes_in_tmux.sh - -exit From 84c754221d8d5f2d55f9bd38b985edfcb2579388 Mon Sep 17 00:00:00 2001 From: Wilfred Gee Date: Sat, 11 May 2019 19:49:27 +1000 Subject: [PATCH 069/105] Camera conversion uses new panoptes-utils --- pocs/camera/canon_gphoto2.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pocs/camera/canon_gphoto2.py b/pocs/camera/canon_gphoto2.py index 53879921c..965fcbccb 100644 --- a/pocs/camera/canon_gphoto2.py +++ b/pocs/camera/canon_gphoto2.py @@ -1,13 +1,12 @@ import os import subprocess -from astropy import units as u from threading import Event from threading import Timer -from pocs.utils import current_time -from pocs.utils import error -from pocs.utils.images import cr2 as cr2_utils +from panoptes_utils import current_time +from panoptes_utils import error +from panoptes_utils.images import cr2 as cr2_utils from pocs.camera import AbstractGPhotoCamera From d73a0214fd22de32a6b793d6b01becc20cd0871f Mon Sep 17 00:00:00 2001 From: Wilfred Gee Date: Sat, 11 May 2019 19:50:03 +1000 Subject: [PATCH 070/105] Require utils --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 641f4804d..c4ac865e0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,6 +7,7 @@ jupyterlab matplotlib mocket numpy +panoptes-utils psycopg2-binary pycodestyle==2.3.1 pyserial From cfaaa80d231a85a0d7c6acb09195a9f0c82171f0 Mon Sep 17 00:00:00 2001 From: Wilfred Gee Date: Sat, 11 May 2019 20:00:26 +1000 Subject: [PATCH 071/105] Switching the base camera to utils --- pocs/camera/camera.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pocs/camera/camera.py b/pocs/camera/camera.py index f87f0ebc8..e6125c223 100644 --- a/pocs/camera/camera.py +++ b/pocs/camera/camera.py @@ -14,14 +14,14 @@ import astropy.units as u from pocs.base import PanBase -from pocs.utils import current_time -from pocs.utils import error -from pocs.utils import listify -from pocs.utils import load_module -from pocs.utils import images as img_utils -from pocs.utils import get_quantity_value -from pocs.utils import CountdownTimer -from pocs.utils.images import fits as fits_utils +from panoptes_utils import current_time +from panoptes_utils import error +from panoptes_utils import listify +from panoptes_utils import get_quantity_value +from panoptes_utils import CountdownTimer +from panoptes_utils import images as img_utils +from panoptes_utils.images import fits as fits_utils +from panoptes_utils.library import load_module from pocs.focuser import AbstractFocuser from pocs.filterwheel import AbstractFilterWheel From c1dd76be1dd3d474a355caf01b4c8ae9a452bc48 Mon Sep 17 00:00:00 2001 From: Wilfred Gee Date: Sat, 11 May 2019 20:16:20 +1000 Subject: [PATCH 072/105] Update name to match --- pocs/camera/camera.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pocs/camera/camera.py b/pocs/camera/camera.py index e6125c223..7f3aa9d61 100644 --- a/pocs/camera/camera.py +++ b/pocs/camera/camera.py @@ -631,7 +631,7 @@ def _process_fits(self, file_path, info): Add FITS headers from info the same as images.cr2_to_fits() """ self.logger.debug("Updating FITS headers: {}".format(file_path)) - fits_utils.update_observation_headers(file_path, info) + fits_utils.update_headers(file_path, info) return file_path def _create_subcomponent(self, subcomponent, sub_name, class_name, base_class): From 461ac460b64d650227ff8334591bac6d37eab90f Mon Sep 17 00:00:00 2001 From: Wilfred Gee Date: Sat, 11 May 2019 20:35:54 +1000 Subject: [PATCH 073/105] Use correct image type for pretty images --- pocs/camera/camera.py | 3 ++- pocs/camera/canon_gphoto2.py | 11 +++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/pocs/camera/camera.py b/pocs/camera/camera.py index 7f3aa9d61..a27dabae9 100644 --- a/pocs/camera/camera.py +++ b/pocs/camera/camera.py @@ -345,7 +345,8 @@ def process_exposure(self, info, observation_event, exposure_event=None): self.logger.debug("Processing {}".format(image_title)) img_utils.make_pretty_image(file_path, title=image_title, - link_latest=info['is_primary']) + link_latest=info['is_primary'], + img_type=info.get('img_type', None)) except Exception as e: # pragma: no cover self.logger.warning('Problem with extracting pretty image: {}'.format(e)) diff --git a/pocs/camera/canon_gphoto2.py b/pocs/camera/canon_gphoto2.py index 965fcbccb..2e4f7fd72 100644 --- a/pocs/camera/canon_gphoto2.py +++ b/pocs/camera/canon_gphoto2.py @@ -90,7 +90,7 @@ def take_observation(self, observation, headers=None, filename=None, *args, **kw threading.Event: An event to be set when the image is done processing """ # To be used for marking when exposure is complete (see `process_exposure`) - camera_event = Event() + observation_event = Event() exptime, file_path, image_id, metadata = self._setup_observation(observation, headers, @@ -98,7 +98,7 @@ def take_observation(self, observation, headers=None, filename=None, *args, **kw *args, **kwargs) - proc = self.take_exposure(seconds=exptime, filename=file_path) + exposure_event = self.take_exposure(seconds=exptime, filename=file_path) # Add most recent exposure to list if self.is_primary: @@ -109,11 +109,14 @@ def take_observation(self, observation, headers=None, filename=None, *args, **kw # Process the image after a set amount of time wait_time = exptime + self.readout_time - t = Timer(wait_time, self.process_exposure, (metadata, camera_event, proc)) + + metadata['img_type'] = '.cr2' + + t = Timer(wait_time, self.process_exposure, (metadata, observation_event, exposure_event)) t.name = '{}Thread'.format(self.name) t.start() - return camera_event + return observation_event def _start_exposure(self, seconds, filename, dark, header, *args, **kwargs): """Take an exposure for given number of seconds and saves to provided filename From 864e5d0d9ccc00c752c89bbec8dcc8844c97451a Mon Sep 17 00:00:00 2001 From: Wilfred Gee Date: Sat, 11 May 2019 20:41:10 +1000 Subject: [PATCH 074/105] Small wrapper to process fits correctly --- pocs/camera/camera.py | 3 +-- pocs/camera/canon_gphoto2.py | 9 +++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/pocs/camera/camera.py b/pocs/camera/camera.py index a27dabae9..7f3aa9d61 100644 --- a/pocs/camera/camera.py +++ b/pocs/camera/camera.py @@ -345,8 +345,7 @@ def process_exposure(self, info, observation_event, exposure_event=None): self.logger.debug("Processing {}".format(image_title)) img_utils.make_pretty_image(file_path, title=image_title, - link_latest=info['is_primary'], - img_type=info.get('img_type', None)) + link_latest=info['is_primary']) except Exception as e: # pragma: no cover self.logger.warning('Problem with extracting pretty image: {}'.format(e)) diff --git a/pocs/camera/canon_gphoto2.py b/pocs/camera/canon_gphoto2.py index 2e4f7fd72..b853df850 100644 --- a/pocs/camera/canon_gphoto2.py +++ b/pocs/camera/canon_gphoto2.py @@ -110,8 +110,6 @@ def take_observation(self, observation, headers=None, filename=None, *args, **kw # Process the image after a set amount of time wait_time = exptime + self.readout_time - metadata['img_type'] = '.cr2' - t = Timer(wait_time, self.process_exposure, (metadata, observation_event, exposure_event)) t.name = '{}Thread'.format(self.name) t.start() @@ -158,3 +156,10 @@ def _readout(self, cr2_path, info): self.logger.debug("Converting CR2 -> FITS: {}".format(cr2_path)) fits_path = cr2_utils.cr2_to_fits(cr2_path, headers=info, remove_cr2=False) return fits_path + + def _process_fits(self, file_path, info): + """ + Add FITS headers from info the same as images.cr2_to_fits() + """ + file_path = file_path.replace('.cr2', '.fits') + return super()._process_fits(file_path, info) From d493bf11e2c6a96abef9e0a2c873e6aadea1d8c2 Mon Sep 17 00:00:00 2001 From: Wilfred Gee Date: Sat, 11 May 2019 21:02:53 +1000 Subject: [PATCH 075/105] Adding custom poll exposure so we can properly set the _is_exposing. (I swear I've done this?) --- pocs/camera/canon_gphoto2.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/pocs/camera/canon_gphoto2.py b/pocs/camera/canon_gphoto2.py index b853df850..89baa01e7 100644 --- a/pocs/camera/canon_gphoto2.py +++ b/pocs/camera/canon_gphoto2.py @@ -5,6 +5,7 @@ from threading import Timer from panoptes_utils import current_time +from panoptes_utils import CountdownTimer from panoptes_utils import error from panoptes_utils.images import cr2 as cr2_utils from pocs.camera import AbstractGPhotoCamera @@ -135,6 +136,7 @@ def _start_exposure(self, seconds, filename, dark, header, *args, **kwargs): # Take Picture try: + self._is_exposing = True proc = subprocess.Popen(run_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, @@ -163,3 +165,18 @@ def _process_fits(self, file_path, info): """ file_path = file_path.replace('.cr2', '.fits') return super()._process_fits(file_path, info) + + def _poll_exposure(self, readout_args): + timer = CountdownTimer(duration=self._timeout) + try: + timer.sleep() + except (RuntimeError, error.PanError) as err: + # Error returned by driver at some point while polling + self.logger.error('Error while waiting for exposure on {}: {}'.format(self, err)) + raise err + else: + # Camera type specific readout function + self._readout(*readout_args) + finally: + self._is_exposing = False + self._exposure_event.set() # Make sure this gets set regardless of readout errors From c163066ecb8cf618f893ea769d4fe7e559d29605 Mon Sep 17 00:00:00 2001 From: Wilfred Gee Date: Sun, 12 May 2019 05:23:37 +1000 Subject: [PATCH 076/105] Setting a default google credentials env var. Need to test what happens when file doesn't exist but I assume it just skips that auth method. Hopefully silenly. Other minor cleanup --- docker/Dockerfile | 8 +++++--- docker/cloudbuild.yaml | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index fcfbe6c00..e74e03fc7 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -8,12 +8,14 @@ ARG which_pip=pip ENV PANDIR $pandir ENV POCS ${PANDIR}/POCS +# ENV GOOGLE_APPLICATION_CREDENTIALS="${PANDIR}/.key/panoptes-survey.json" COPY . ${POCS} RUN apt-get update \ - && apt-get --yes install \ + && apt-get install --no-install-recommends --yes \ byobu \ + ffmpeg \ gcc \ pkg-config \ libncurses5-dev \ @@ -28,8 +30,8 @@ RUN apt-get update \ && rm gphoto2-updater.sh \ # POCS && cd ${POCS} \ - && $which_pip install --no-cache-dir -r requirements.txt \ - && $which_pip install --no-cache-dir -e . \ + && $which_pip install --no-cache-dir --no-deps -r requirements.txt \ + && $which_pip install --no-cache-dir --no-deps -e . \ # Link conf_files to $PANDIR && ln -s ${POCS}/conf_files/ ${PANDIR}/ diff --git a/docker/cloudbuild.yaml b/docker/cloudbuild.yaml index 91dde0469..606a0bd21 100644 --- a/docker/cloudbuild.yaml +++ b/docker/cloudbuild.yaml @@ -15,7 +15,7 @@ steps: id: 'amd64' args: - 'build' - - '-f=resources/docker_files/Dockerfile' + - '-f=docker/Dockerfile' - '--build-arg=arch=amd64' - '--tag=gcr.io/${PROJECT_ID}/pocs:amd64' - '.' @@ -25,7 +25,7 @@ steps: id: 'arm32v7' args: - 'build' - - '-f=resources/docker_files/Dockerfile' + - '-f=docker/Dockerfile' - '--build-arg=arch=arm32v7' - '--build-arg=which_pip=/opt/conda/envs/panoptes-env/bin/pip' - '--tag=gcr.io/${PROJECT_ID}/pocs:arm32v7' From b5b93b0a2108003b309b3abc9614d045e68f40c3 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Sun, 12 May 2019 11:39:50 +1000 Subject: [PATCH 077/105] Small change to skip errors on solve --- pocs/utils/images/fits.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pocs/utils/images/fits.py b/pocs/utils/images/fits.py index 153913f62..87763955b 100644 --- a/pocs/utils/images/fits.py +++ b/pocs/utils/images/fits.py @@ -177,7 +177,7 @@ def get_solve_field(fname, replace=True, remove_extras=True, **kwargs): except Exception as e: warn('Cannot remove extra files: {}'.format(e)) - if errs is not None: + if errs is not None and errs > '': warn("Error in solving: {}".format(errs)) else: From 130ac1fc6b3e3094be4b76fb4efeb50fdddaeff7 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Sun, 12 May 2019 11:46:37 +1000 Subject: [PATCH 078/105] Update path for docker compose --- docker/Dockerfile | 1 - scripts/pocs-docker.sh | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index e74e03fc7..7b036f7d0 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -8,7 +8,6 @@ ARG which_pip=pip ENV PANDIR $pandir ENV POCS ${PANDIR}/POCS -# ENV GOOGLE_APPLICATION_CREDENTIALS="${PANDIR}/.key/panoptes-survey.json" COPY . ${POCS} diff --git a/scripts/pocs-docker.sh b/scripts/pocs-docker.sh index 6d610268d..29c5b2dce 100755 --- a/scripts/pocs-docker.sh +++ b/scripts/pocs-docker.sh @@ -34,6 +34,6 @@ fi docker-compose \ -f $PANDIR/panoptes-utils/docker/docker-compose.yaml \ -f $PANDIR/PAWS/docker/docker-compose.yaml \ - -f $PANDIR/POCS/resources/docker_files/docker-compose.yaml \ + -f $PANDIR/POCS/docker/docker-compose.yaml \ -p panoptes "$@" From 9fa9134b80017992dd162b7d69aefc2a1175d8ef Mon Sep 17 00:00:00 2001 From: Wilfred Gee Date: Sun, 12 May 2019 14:43:45 +1000 Subject: [PATCH 079/105] Install docker-compose --- scripts/install/install-pocs.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/install/install-pocs.sh b/scripts/install/install-pocs.sh index 140862fa4..e961abc55 100755 --- a/scripts/install/install-pocs.sh +++ b/scripts/install/install-pocs.sh @@ -77,9 +77,11 @@ if ! hash docker; then fi echo "Pulling POCS docker images" -sudo docker pull google/cloud-sdk:latest sudo docker pull gcr.io/panoptes-survey/pocs sudo docker pull gcr.io/panoptes-survey/paws +echo "Installing docker-compose" +pip install --no-deps docker-compose + echo "You must logout and log back in to before using POCS." From e0f258f490980d59e6e386d3108ebfeef323097d Mon Sep 17 00:00:00 2001 From: Wilfred Gee Date: Sun, 12 May 2019 14:46:30 +1000 Subject: [PATCH 080/105] docker-compose from docker --- scripts/install/install-pocs.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/install/install-pocs.sh b/scripts/install/install-pocs.sh index e961abc55..25c03c262 100755 --- a/scripts/install/install-pocs.sh +++ b/scripts/install/install-pocs.sh @@ -81,7 +81,8 @@ sudo docker pull gcr.io/panoptes-survey/pocs sudo docker pull gcr.io/panoptes-survey/paws echo "Installing docker-compose" -pip install --no-deps docker-compose +sudo curl -L --fail https://github.com/docker/compose/releases/download/1.24.0/run.sh -o /usr/local/bin/docker-compose +sudo chmod +x /usr/local/bin/docker-compose echo "You must logout and log back in to before using POCS." From 4a152b379e93e994e48c775197cb41f114fe5c50 Mon Sep 17 00:00:00 2001 From: Wilfred Gee Date: Sun, 12 May 2019 14:46:54 +1000 Subject: [PATCH 081/105] Docker-compose is already there --- scripts/install/install-pocs.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scripts/install/install-pocs.sh b/scripts/install/install-pocs.sh index 25c03c262..24a929853 100755 --- a/scripts/install/install-pocs.sh +++ b/scripts/install/install-pocs.sh @@ -80,9 +80,5 @@ echo "Pulling POCS docker images" sudo docker pull gcr.io/panoptes-survey/pocs sudo docker pull gcr.io/panoptes-survey/paws -echo "Installing docker-compose" -sudo curl -L --fail https://github.com/docker/compose/releases/download/1.24.0/run.sh -o /usr/local/bin/docker-compose -sudo chmod +x /usr/local/bin/docker-compose - echo "You must logout and log back in to before using POCS." From c2b4ed3131f64919e823d8729337cf68353f2060 Mon Sep 17 00:00:00 2001 From: Wilfred Gee Date: Sun, 12 May 2019 14:47:57 +1000 Subject: [PATCH 082/105] docker-compose separate from docker --- scripts/install/install-pocs.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/install/install-pocs.sh b/scripts/install/install-pocs.sh index 24a929853..b20c77e56 100755 --- a/scripts/install/install-pocs.sh +++ b/scripts/install/install-pocs.sh @@ -70,9 +70,11 @@ if ! hash docker; then echo "Installing Docker" sh -c "$(wget https://get.docker.com -O -)" sudo usermod -aG docker ${PANUSER} +fi + +if ! hash docker-compose; then # Docker compose as container - https://docs.docker.com/compose/install/#install-compose sudo curl -L --fail https://github.com/docker/compose/releases/download/1.24.0/run.sh -o /usr/local/bin/docker-compose - sudo chmod +x /usr/local/bin/docker-compose sudo chmod a+x /usr/local/bin/docker-compose fi From 384f285afb0621a278a93e88f24643c4645a26df Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Mon, 13 May 2019 07:18:46 +1000 Subject: [PATCH 083/105] Adding entrypoing script to pocs-shell image. If the user has added the GOOGLE_APPLICATION_CREDENTIALS to `$PANDIR/env_file` then this will attempt to authenticate. --- docker/Dockerfile | 2 ++ docker/entrypoint.sh | 9 +++++++++ 2 files changed, 11 insertions(+) create mode 100644 docker/entrypoint.sh diff --git a/docker/Dockerfile b/docker/Dockerfile index 7b036f7d0..9f67e02f6 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -36,5 +36,7 @@ RUN apt-get update \ WORKDIR ${POCS} +ENTRYPOINT ["${POCS}/docker/entrypoint.sh"] + CMD ["/bin/zsh"] diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100644 index 000000000..8694d05b5 --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,9 @@ +#!/bin/bash -e + +# Authenticate if key has been set +if [ ! -z "$GOOGLE_APPLICATION_CREDENTIALS" ]; then + gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS +fi + +# Pass arguments +exec "$@" \ No newline at end of file From 8538abe00604d9d4b38e49c8ddd85814d2720115 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Wed, 15 May 2019 07:12:55 +1000 Subject: [PATCH 084/105] Run entrypoint script properly --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 9f67e02f6..db448c6fb 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -36,7 +36,7 @@ RUN apt-get update \ WORKDIR ${POCS} -ENTRYPOINT ["${POCS}/docker/entrypoint.sh"] +ENTRYPOINT ["${pandir}/POCS/docker/entrypoint.sh"] CMD ["/bin/zsh"] From c97193f5b2101831552eb43d0537cb0c4667598d Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Wed, 15 May 2019 07:24:24 +1000 Subject: [PATCH 085/105] Adding newline --- docker/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 8694d05b5..9d5c278ed 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -6,4 +6,4 @@ if [ ! -z "$GOOGLE_APPLICATION_CREDENTIALS" ]; then fi # Pass arguments -exec "$@" \ No newline at end of file +exec "$@" From 66580cd504202dfb38c5efd0249eacc7b023d545 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Wed, 15 May 2019 13:19:43 +1000 Subject: [PATCH 086/105] Fixing entrypoint --- docker/Dockerfile | 2 +- docker/entrypoint.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index db448c6fb..deee6d693 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -36,7 +36,7 @@ RUN apt-get update \ WORKDIR ${POCS} -ENTRYPOINT ["${pandir}/POCS/docker/entrypoint.sh"] +ENTRYPOINT ["/bin/bash", "-c", "${POCS}/docker/entrypoint.sh"] CMD ["/bin/zsh"] diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 9d5c278ed..65b885015 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -2,6 +2,7 @@ # Authenticate if key has been set if [ ! -z "$GOOGLE_APPLICATION_CREDENTIALS" ]; then + echo "Found Google credentials, activating service account" gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS fi From 673d335cc3637b4dff95932b72ac40d3bcc88d37 Mon Sep 17 00:00:00 2001 From: Wilfred Gee Date: Wed, 15 May 2019 15:08:10 +1000 Subject: [PATCH 087/105] Make script relative and executable --- docker/Dockerfile | 2 +- docker/entrypoint.sh | 0 2 files changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 docker/entrypoint.sh diff --git a/docker/Dockerfile b/docker/Dockerfile index deee6d693..803dbdf0a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -36,7 +36,7 @@ RUN apt-get update \ WORKDIR ${POCS} -ENTRYPOINT ["/bin/bash", "-c", "${POCS}/docker/entrypoint.sh"] +ENTRYPOINT ["/bin/bash", "-c", "docker/entrypoint.sh"] CMD ["/bin/zsh"] diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh old mode 100644 new mode 100755 From 7e50c5b607aeb87c1519454263dafb41d3ecdb30 Mon Sep 17 00:00:00 2001 From: Wilfred Gee Date: Wed, 15 May 2019 15:30:37 +1000 Subject: [PATCH 088/105] Remove entrypoint script, added to panoptes-utils --- docker/Dockerfile | 2 -- docker/entrypoint.sh | 10 ---------- 2 files changed, 12 deletions(-) delete mode 100755 docker/entrypoint.sh diff --git a/docker/Dockerfile b/docker/Dockerfile index 803dbdf0a..7b036f7d0 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -36,7 +36,5 @@ RUN apt-get update \ WORKDIR ${POCS} -ENTRYPOINT ["/bin/bash", "-c", "docker/entrypoint.sh"] - CMD ["/bin/zsh"] diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh deleted file mode 100755 index 65b885015..000000000 --- a/docker/entrypoint.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -e - -# Authenticate if key has been set -if [ ! -z "$GOOGLE_APPLICATION_CREDENTIALS" ]; then - echo "Found Google credentials, activating service account" - gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS -fi - -# Pass arguments -exec "$@" From f8113a0ecc788671b1167988dc83976a428416c2 Mon Sep 17 00:00:00 2001 From: Wilfred Gee Date: Wed, 15 May 2019 16:02:32 +1000 Subject: [PATCH 089/105] Use entrypoint from base image in all images. --- docker/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index 7b036f7d0..18ba23358 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -36,5 +36,8 @@ RUN apt-get update \ WORKDIR ${POCS} +# Comes from base image +ENTRYPOINT ["/var/panoptes/panoptes-utils/docker/entrypoint.sh"] + CMD ["/bin/zsh"] From 48f3876436733567b41b7b8cd202698615ab3317 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Mon, 20 May 2019 14:27:07 +1000 Subject: [PATCH 090/105] Remove entrypoint --- docker/Dockerfile | 2 -- docker/entrypoint.sh | 10 ---------- 2 files changed, 12 deletions(-) delete mode 100644 docker/entrypoint.sh diff --git a/docker/Dockerfile b/docker/Dockerfile index deee6d693..7b036f7d0 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -36,7 +36,5 @@ RUN apt-get update \ WORKDIR ${POCS} -ENTRYPOINT ["/bin/bash", "-c", "${POCS}/docker/entrypoint.sh"] - CMD ["/bin/zsh"] diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh deleted file mode 100644 index 65b885015..000000000 --- a/docker/entrypoint.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -e - -# Authenticate if key has been set -if [ ! -z "$GOOGLE_APPLICATION_CREDENTIALS" ]; then - echo "Found Google credentials, activating service account" - gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS -fi - -# Pass arguments -exec "$@" From def093810459fb402a26ad5a22494d606f983c22 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Wed, 22 May 2019 07:32:43 +1000 Subject: [PATCH 091/105] Docker updates * Install arduino-cli. * Split into separate images. --- docker/Dockerfile | 23 ++++++++++++++++++----- docker/cloudbuild.yaml | 1 + scripts/build_containers.sh | 8 ++++++-- 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 7b036f7d0..73b5445d1 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,10 +1,11 @@ ARG arch=amd64 -FROM gcr.io/panoptes-survey/panoptes-utils:$arch +FROM gcr.io/panoptes-survey/panoptes-utils:$arch AS pocs-base MAINTAINER Developers for PANOPTES project ARG pandir=/var/panoptes -ARG which_pip=pip + +ARG arduino_url=https://downloads.arduino.cc/arduino-cli/arduino-cli-latest-linux64.tar.bz2 ENV PANDIR $pandir ENV POCS ${PANDIR}/POCS @@ -23,12 +24,24 @@ RUN apt-get update \ && wget https://raw.githubusercontent.com/gonzalo/gphoto2-updater/master/gphoto2-updater.sh \ && chmod +x gphoto2-updater.sh \ && /bin/bash gphoto2-updater.sh --stable \ + # arduino-cli + && wget -q $arduino_url -O arduino-cli.tar.bz2 \ + && bunzip2 arduino-cli.tar.bz2 \ + && tar xof arduino-cli.tar \ + && chmod +x arduino-cli \ + && mv arduino-cli /usr/local/bin/ \ # Cleanup && apt-get clean \ && rm -rf /var/lib/apt/lists/* \ - && rm gphoto2-updater.sh \ - # POCS - && cd ${POCS} \ + && rm gphoto2-updater.sh + + +FROM pocs-base + +ARG which_pip=pip + +# POCS +RUN cd ${POCS} \ && $which_pip install --no-cache-dir --no-deps -r requirements.txt \ && $which_pip install --no-cache-dir --no-deps -e . \ # Link conf_files to $PANDIR diff --git a/docker/cloudbuild.yaml b/docker/cloudbuild.yaml index 606a0bd21..32e82917e 100644 --- a/docker/cloudbuild.yaml +++ b/docker/cloudbuild.yaml @@ -28,6 +28,7 @@ steps: - '-f=docker/Dockerfile' - '--build-arg=arch=arm32v7' - '--build-arg=which_pip=/opt/conda/envs/panoptes-env/bin/pip' + - '--build-arg=arduino_url=https://downloads.arduino.cc/arduino-cli/arduino-cli-latest-linuxaarch64.tar.bz2' - '--tag=gcr.io/${PROJECT_ID}/pocs:arm32v7' - '.' waitFor: ['register-qemu'] diff --git a/scripts/build_containers.sh b/scripts/build_containers.sh index 2ce26ddd1..4a93bbc70 100755 --- a/scripts/build_containers.sh +++ b/scripts/build_containers.sh @@ -1,5 +1,9 @@ #!/bin/bash -e -SOURCE_DIR=${POCS} +SOURCE_DIR="${POCS}" -gcloud builds submit --timeout="5h" --config ${SOURCE_DIR}/docker/cloudbuild.yaml --async ${SOURCE_DIR} +gcloud builds submit \ + --timeout="5h" \ + --config "${SOURCE_DIR}/docker/cloudbuild.yaml" + --async \ + "${SOURCE_DIR}" From d3f172dcd1b4eb9226fdeb237f18666655677e47 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Wed, 22 May 2019 15:08:19 +1000 Subject: [PATCH 092/105] * arduino-cli comments for install and cleanup * udev added for gphoto2 detection --- docker/Dockerfile | 8 +++++++- scripts/build_containers.sh | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 73b5445d1..5cc8157cb 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -5,7 +5,7 @@ MAINTAINER Developers for PANOPTES project ARG pandir=/var/panoptes -ARG arduino_url=https://downloads.arduino.cc/arduino-cli/arduino-cli-latest-linux64.tar.bz2 +ARG arduino_url="https://downloads.arduino.cc/arduino-cli/arduino-cli-latest-linux64.tar.bz2" ENV PANDIR $pandir ENV POCS ${PANDIR}/POCS @@ -19,6 +19,7 @@ RUN apt-get update \ gcc \ pkg-config \ libncurses5-dev \ + udev \ vim-nox \ # GPhoto2 && wget https://raw.githubusercontent.com/gonzalo/gphoto2-updater/master/gphoto2-updater.sh \ @@ -28,6 +29,11 @@ RUN apt-get update \ && wget -q $arduino_url -O arduino-cli.tar.bz2 \ && bunzip2 arduino-cli.tar.bz2 \ && tar xof arduino-cli.tar \ + && rm arduino-cli.tar \ + # Don't know exact name of untarred file. + && ARDUINO_FILE="$(ls arduino-cli*)" \ + # Rename so we do know. + && mv ${ARDUINO_FILE} arduino-cli \ && chmod +x arduino-cli \ && mv arduino-cli /usr/local/bin/ \ # Cleanup diff --git a/scripts/build_containers.sh b/scripts/build_containers.sh index 4a93bbc70..3269715cd 100755 --- a/scripts/build_containers.sh +++ b/scripts/build_containers.sh @@ -3,7 +3,7 @@ SOURCE_DIR="${POCS}" gcloud builds submit \ --timeout="5h" \ - --config "${SOURCE_DIR}/docker/cloudbuild.yaml" + --config "${SOURCE_DIR}/docker/cloudbuild.yaml" \ --async \ "${SOURCE_DIR}" From 6a1ff4fd727d976a1b8666343ea1266d4b3cfd97 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Fri, 7 Jun 2019 16:33:33 +1000 Subject: [PATCH 093/105] Improved docker install script --- scripts/install/install-pocs.sh | 142 +++++++++++++++++++++----------- 1 file changed, 96 insertions(+), 46 deletions(-) diff --git a/scripts/install/install-pocs.sh b/scripts/install/install-pocs.sh index b20c77e56..a8e2069ce 100755 --- a/scripts/install/install-pocs.sh +++ b/scripts/install/install-pocs.sh @@ -1,4 +1,48 @@ -#!/bin/bash -e +#!/usr/bin/env bash + +usage() { + echo -n "################################################## +# Install POCS and friends. +# +# This script is designed to install the PANOPTES Observatory +# Control System (POCS) on a cleanly install Ubuntu system. +# +# The script will insure that Docker is installed, download the +# latest Docker images (see list below) and clone a copy of the +# relevant PANOPTES repositories. +# +# Docker Images: +# +# gcr.io/panoptes-survey/pocs +# gcr.io/panoptes-survey/paws +# +# Github Repositories: +# +# The script will ask for a github user name in order to install +# forked versions of the repos if you are actively developing the +# software. otherwise the default user (panotpes) is okay for +# running the unit. +# +# github.com/panoptes/POCS +# github.com/panoptes/PAWS +# github.com/panoptes/panoptes-utils +# +# The script has been tested with a fresh install of Ubuntu 19.04 +# but may work on other linux systems. +############################################################# + $ $(basename $0) [--user panoptes] [--pandir /var/panoptes] + + Options: + USER The default user. This is saved as the PANUSER environment variable. + PANDIR Default install directory, defaults to /var/panoptes. Saved as PANDIR + environment variable. +" +} + +if [ $# -eq 0 ]; then + usage + exit 1 +fi if [ -z ${PANUSER} ]; then export PANUSER=$USER @@ -26,61 +70,67 @@ case $key in esac done -echo "USER: ${PANUSER}" -echo "DIR: ${PANDIR}" +do_install() { + echo "Installing PANOPTES software." + echo "USER: ${PANUSER}" + echo "DIR: ${PANDIR}" -if [[ ! -d ${PANDIR} ]] || [[ $(stat -c "%U" ${PANDIR}) -ne $USER ]]; then - echo "Creating directories" - # Make directories - sudo mkdir -p ${PANDIR} - sudo chown -R ${PANUSER}:${PANUSER} ${PANDIR} + if [[ ! -d ${PANDIR} ]] || [[ $(stat -c "%U" ${PANDIR}) -ne $USER ]]; then + echo "Creating directories" + # Make directories + sudo mkdir -p ${PANDIR} + sudo chown -R ${PANUSER}:${PANUSER} ${PANDIR} - mkdir -p ${PANDIR}/logs - mkdir -p ${PANDIR}/conf_files - mkdir -p ${PANDIR}/images -fi + mkdir -p ${PANDIR}/logs + mkdir -p ${PANDIR}/conf_files + mkdir -p ${PANDIR}/images + fi -echo "Log files will be stored in ${PANDIR}/logs/install-pocs.log." + echo "Log files will be stored in ${PANDIR}/logs/install-pocs.log." -# apt: git, wget -echo "Installing system dependencies" -sudo apt update &>> ${PANDIR}/logs/install-pocs.log -sudo apt --yes install wget git &>> ${PANDIR}/logs/install-pocs.log + # apt: git, wget + echo "Installing system dependencies" + sudo apt update &>> ${PANDIR}/logs/install-pocs.log + sudo apt --yes install wget git &>> ${PANDIR}/logs/install-pocs.log -echo "Cloning PANOPTES source code." -echo "Github user for PANOPTES repos (POCS, PAWS, panoptes-utils)." -read -p "Github User: [panoptes] " github_user + echo "Cloning PANOPTES source code." + echo "Github user for PANOPTES repos (POCS, PAWS, panoptes-utils)." + read -p "Github User: [panoptes] " github_user -cd ${PANDIR} + cd ${PANDIR} -declare -a repos=("POCS" "PAWS" "panoptes-utils") + declare -a repos=("POCS" "PAWS" "panoptes-utils") -for repo in "${repos[@]}"; do - if [ ! -d "${PANDIR}/${repo}" ]; then - echo "Cloning ${repo}" - # Just redirect the errors because otherwise looks like it hangs. - git clone https://github.com/${github_user}/${repo}.git - else - echo "Repo ${repo} already exists on system." - fi -done + for repo in "${repos[@]}"; do + if [ ! -d "${PANDIR}/${repo}" ]; then + echo "Cloning ${repo}" + # Just redirect the errors because otherwise looks like it hangs. + git clone https://github.com/${github_user}/${repo}.git + else + echo "Repo ${repo} already exists on system." + fi + done -# Get Docker -if ! hash docker; then - echo "Installing Docker" - sh -c "$(wget https://get.docker.com -O -)" - sudo usermod -aG docker ${PANUSER} -fi + # Get Docker + if ! hash docker; then + echo "Installing Docker" + sh -c "$(wget https://get.docker.com -O -)" + sudo usermod -aG docker ${PANUSER} + fi -if ! hash docker-compose; then - # Docker compose as container - https://docs.docker.com/compose/install/#install-compose - sudo curl -L --fail https://github.com/docker/compose/releases/download/1.24.0/run.sh -o /usr/local/bin/docker-compose - sudo chmod a+x /usr/local/bin/docker-compose -fi + if ! hash docker-compose; then + # Docker compose as container - https://docs.docker.com/compose/install/#install-compose + sudo curl -L --fail https://github.com/docker/compose/releases/download/1.24.0/run.sh -o /usr/local/bin/docker-compose + sudo chmod a+x /usr/local/bin/docker-compose + fi -echo "Pulling POCS docker images" -sudo docker pull gcr.io/panoptes-survey/pocs -sudo docker pull gcr.io/panoptes-survey/paws + echo "Pulling POCS docker images" + sudo docker pull gcr.io/panoptes-survey/pocs + sudo docker pull gcr.io/panoptes-survey/paws -echo "You must logout and log back in to before using POCS." + echo "You must logout and log back in to before using POCS." +} +# wrapped up in a function so that we have some protection against only getting +# half the file during "curl | sh" - copied from get.docker.com +do_install From 8983ae5f760fd15735e5864f99b2a27a6d9c9aa9 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Fri, 7 Jun 2019 16:39:46 +1000 Subject: [PATCH 094/105] Fixing import statements --- pocs/camera/camera.py | 16 ++++++++-------- pocs/camera/canon_gphoto2.py | 8 ++++---- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/pocs/camera/camera.py b/pocs/camera/camera.py index 7f3aa9d61..3e8bbe969 100644 --- a/pocs/camera/camera.py +++ b/pocs/camera/camera.py @@ -14,14 +14,14 @@ import astropy.units as u from pocs.base import PanBase -from panoptes_utils import current_time -from panoptes_utils import error -from panoptes_utils import listify -from panoptes_utils import get_quantity_value -from panoptes_utils import CountdownTimer -from panoptes_utils import images as img_utils -from panoptes_utils.images import fits as fits_utils -from panoptes_utils.library import load_module +from panoptes.utils import current_time +from panoptes.utils import error +from panoptes.utils import listify +from panoptes.utils import get_quantity_value +from panoptes.utils import CountdownTimer +from panoptes.utils import images as img_utils +from panoptes.utils.images import fits as fits_utils +from panoptes.utils.library import load_module from pocs.focuser import AbstractFocuser from pocs.filterwheel import AbstractFilterWheel diff --git a/pocs/camera/canon_gphoto2.py b/pocs/camera/canon_gphoto2.py index 89baa01e7..01b5d40e3 100644 --- a/pocs/camera/canon_gphoto2.py +++ b/pocs/camera/canon_gphoto2.py @@ -4,10 +4,10 @@ from threading import Event from threading import Timer -from panoptes_utils import current_time -from panoptes_utils import CountdownTimer -from panoptes_utils import error -from panoptes_utils.images import cr2 as cr2_utils +from panoptes.utils import current_time +from panoptes.utils import CountdownTimer +from panoptes.utils import error +from panoptes.utils.images import cr2 as cr2_utils from pocs.camera import AbstractGPhotoCamera From 26c73419eac22909d3f4a2f1d5bdf0d7ffa7dd3e Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Sat, 15 Jun 2019 20:00:15 +1000 Subject: [PATCH 095/105] Adding openssh-server to list of default apt tools --- scripts/install/install-pocs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install/install-pocs.sh b/scripts/install/install-pocs.sh index a8e2069ce..2b03548a9 100755 --- a/scripts/install/install-pocs.sh +++ b/scripts/install/install-pocs.sh @@ -91,7 +91,7 @@ do_install() { # apt: git, wget echo "Installing system dependencies" sudo apt update &>> ${PANDIR}/logs/install-pocs.log - sudo apt --yes install wget git &>> ${PANDIR}/logs/install-pocs.log + sudo apt --yes install wget git openssh-server &>> ${PANDIR}/logs/install-pocs.log echo "Cloning PANOPTES source code." echo "Github user for PANOPTES repos (POCS, PAWS, panoptes-utils)." From 1eaf33a5c87058d58718039d11394b5cc7a61903 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Sun, 16 Jun 2019 08:28:51 +1000 Subject: [PATCH 096/105] Tabs to spaces --- scripts/install/install-pocs.sh | 36 +++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/scripts/install/install-pocs.sh b/scripts/install/install-pocs.sh index 2b03548a9..3dd7cc2cc 100755 --- a/scripts/install/install-pocs.sh +++ b/scripts/install/install-pocs.sh @@ -45,12 +45,12 @@ if [ $# -eq 0 ]; then fi if [ -z ${PANUSER} ]; then - export PANUSER=$USER - echo "export PANUSER=${PANUSER}" >> ${HOME}/.bashrc + export PANUSER=$USER + echo "export PANUSER=${PANUSER}" >> ${HOME}/.bashrc fi if [ -z ${PANDIR} ]; then - export PANDIR='/var/panoptes' - echo "export PANDIR=${PANDIR}" >> ${HOME}/.bashrc + export PANDIR='/var/panoptes' + echo "export PANDIR=${PANDIR}" >> ${HOME}/.bashrc fi while [[ $# -gt 0 ]] @@ -76,14 +76,14 @@ do_install() { echo "DIR: ${PANDIR}" if [[ ! -d ${PANDIR} ]] || [[ $(stat -c "%U" ${PANDIR}) -ne $USER ]]; then - echo "Creating directories" - # Make directories - sudo mkdir -p ${PANDIR} - sudo chown -R ${PANUSER}:${PANUSER} ${PANDIR} - - mkdir -p ${PANDIR}/logs - mkdir -p ${PANDIR}/conf_files - mkdir -p ${PANDIR}/images + echo "Creating directories" + # Make directories + sudo mkdir -p ${PANDIR} + sudo chown -R ${PANUSER}:${PANUSER} ${PANDIR} + + mkdir -p ${PANDIR}/logs + mkdir -p ${PANDIR}/conf_files + mkdir -p ${PANDIR}/images fi echo "Log files will be stored in ${PANDIR}/logs/install-pocs.log." @@ -91,7 +91,7 @@ do_install() { # apt: git, wget echo "Installing system dependencies" sudo apt update &>> ${PANDIR}/logs/install-pocs.log - sudo apt --yes install wget git openssh-server &>> ${PANDIR}/logs/install-pocs.log + sudo apt --yes install wget git openssh-server byobu &>> ${PANDIR}/logs/install-pocs.log echo "Cloning PANOPTES source code." echo "Github user for PANOPTES repos (POCS, PAWS, panoptes-utils)." @@ -103,9 +103,12 @@ do_install() { for repo in "${repos[@]}"; do if [ ! -d "${PANDIR}/${repo}" ]; then - echo "Cloning ${repo}" - # Just redirect the errors because otherwise looks like it hangs. + echo "Cloning ${repo}" + # Just redirect the errors because otherwise looks like it hangs. git clone https://github.com/${github_user}/${repo}.git + + # TODO + # echo "export PANDIR=${PANDIR}" >> ${HOME}/.bashrc else echo "Repo ${repo} already exists on system." fi @@ -128,6 +131,9 @@ do_install() { sudo docker pull gcr.io/panoptes-survey/pocs sudo docker pull gcr.io/panoptes-survey/paws + # Start docker - TODO: always start? + sudo service docker start + echo "You must logout and log back in to before using POCS." } From 2593941a23f6d35d30375b9b3e63e9b622b8cfb0 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Sun, 16 Jun 2019 08:32:28 +1000 Subject: [PATCH 097/105] Add help option --- scripts/install/install-pocs.sh | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/scripts/install/install-pocs.sh b/scripts/install/install-pocs.sh index 3dd7cc2cc..a707b2ef1 100755 --- a/scripts/install/install-pocs.sh +++ b/scripts/install/install-pocs.sh @@ -39,11 +39,6 @@ usage() { " } -if [ $# -eq 0 ]; then - usage - exit 1 -fi - if [ -z ${PANUSER} ]; then export PANUSER=$USER echo "export PANUSER=${PANUSER}" >> ${HOME}/.bashrc @@ -62,11 +57,16 @@ case $key in shift # past argument shift # past value ;; - -d|--dir) + -d|--pandir) PANDIR="$2" shift # past argument shift # past value ;; + -h|--help) + PANDIR="$2" + usage + exit 1 + ;; esac done @@ -131,10 +131,7 @@ do_install() { sudo docker pull gcr.io/panoptes-survey/pocs sudo docker pull gcr.io/panoptes-survey/paws - # Start docker - TODO: always start? - sudo service docker start - - echo "You must logout and log back in to before using POCS." + echo "Please reboot your machine before using POCS." } # wrapped up in a function so that we have some protection against only getting From cea82f2e55c83181e8c08ef70a124f263304c673 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Sun, 16 Jun 2019 08:38:48 +1000 Subject: [PATCH 098/105] Fix default github user --- scripts/install/install-pocs.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/install/install-pocs.sh b/scripts/install/install-pocs.sh index a707b2ef1..30d972f11 100755 --- a/scripts/install/install-pocs.sh +++ b/scripts/install/install-pocs.sh @@ -95,7 +95,11 @@ do_install() { echo "Cloning PANOPTES source code." echo "Github user for PANOPTES repos (POCS, PAWS, panoptes-utils)." - read -p "Github User: [panoptes] " github_user + + # Default user + read -p "Github User [panoptes]: " github_user + github_user=${github_user:-panoptes} + echo "Using repositories from user '${github_user}'." cd ${PANDIR} From 510d3461ea14d650750cc2440e0939cc163a92f6 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Tue, 18 Jun 2019 10:24:57 +1000 Subject: [PATCH 099/105] Trying to make multi-container compose file work in a sane way --- scripts/pocs-docker.sh | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/scripts/pocs-docker.sh b/scripts/pocs-docker.sh index 29c5b2dce..6ce91cd1f 100755 --- a/scripts/pocs-docker.sh +++ b/scripts/pocs-docker.sh @@ -3,11 +3,11 @@ usage() { echo -n "################################################## # Start POCS via Docker. -# +# ################################################## $ $(basename $0) [COMMAND] - + Options: COMMAND These options are passed at the end of the docker-compose command. To start all service simply pass 'up'. @@ -15,13 +15,16 @@ usage() { Examples: # Start all services in the foreground. - $POCS/scripts/start-pocs-docker.sh up + $POCS/scripts/pocs-docker.sh up # Start config-server and messaging-hub serivces in the background. - $POCS/scripts/start-pocs-docker.sh up --no-deps -d config-server messaging-hub + $POCS/scripts/pocs-docker.sh up --no-deps -d config-server messaging-hub # Read the logs from the config-server - $POCS/scripts/start-pocs-docker.sh logs config-server + $POCS/scripts/pocs-docker.sh logs config-server + + # Run the software tests (no hardware) + $POCS/scripts/pocs-docker.sh up " } @@ -31,9 +34,10 @@ if [ ${START} = 'help' ] || [ ${START} = '-h' ] || [ ${START} = '--help' ]; then exit 1 fi +cd $PANDIR docker-compose \ - -f $PANDIR/panoptes-utils/docker/docker-compose.yaml \ - -f $PANDIR/PAWS/docker/docker-compose.yaml \ - -f $PANDIR/POCS/docker/docker-compose.yaml \ + -f panoptes-utils/docker/docker-compose.yaml \ + -f PAWS/docker/docker-compose.yaml \ + -f POCS/docker/docker-compose.yaml \ -p panoptes "$@" From fcc34101430070d62cbd7e5c7cd3542e91d240ea Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Tue, 18 Jun 2019 10:25:42 +1000 Subject: [PATCH 100/105] docker-compose 3.7 --- docker/docker-compose.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 3c8a40c04..c99b7e06c 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -1,4 +1,4 @@ -version: '3.2' +version: '3.7' services: peas-shell: image: gcr.io/panoptes-survey/pocs @@ -7,7 +7,7 @@ services: network_mode: host env_file: ${PANDIR}/env_file depends_on: - - "messaging-hub" + - "messaging-hub" volumes: - pandir:/var/panoptes # No-op to keep machine running, use $POCS/bin/peas to access @@ -19,7 +19,7 @@ services: network_mode: host env_file: ${PANDIR}/env_file depends_on: - - "peas-shell" + - "peas-shell" volumes: - pandir:/var/panoptes # No-op to keep machine running, use $POCS/bin/pocs to access From aa82c92972da9dc36c41a0275ae3169bd450ccb4 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Tue, 18 Jun 2019 12:22:07 +1000 Subject: [PATCH 101/105] Default for env vars --- docker/docker-compose.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index c99b7e06c..c213f0176 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -5,7 +5,7 @@ services: container_name: peas-shell privileged: true network_mode: host - env_file: ${PANDIR}/env_file + env_file: "${PANDIR:-/var/panoptes}/env_file" depends_on: - "messaging-hub" volumes: @@ -17,7 +17,7 @@ services: container_name: pocs-shell privileged: true network_mode: host - env_file: ${PANDIR}/env_file + env_file: "${PANDIR:-/var/panoptes}/env_file" depends_on: - "peas-shell" volumes: From 799c09e408a9a3cad6e2ac7b2d827d3bf519801c Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Tue, 18 Jun 2019 12:27:37 +1000 Subject: [PATCH 102/105] Adding project directory and sanitzing script --- scripts/pocs-docker.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/pocs-docker.sh b/scripts/pocs-docker.sh index 6ce91cd1f..609cefd4d 100755 --- a/scripts/pocs-docker.sh +++ b/scripts/pocs-docker.sh @@ -29,13 +29,14 @@ usage() { } START=${1:-help} -if [ ${START} = 'help' ] || [ ${START} = '-h' ] || [ ${START} = '--help' ]; then +if [ "${START}" = 'help' ] || [ "${START}" = '-h' ] || [ "${START}" = '--help' ]; then usage exit 1 fi -cd $PANDIR +cd "$PANDIR" docker-compose \ + --project-directory "${PANDIR}" -f panoptes-utils/docker/docker-compose.yaml \ -f PAWS/docker/docker-compose.yaml \ -f POCS/docker/docker-compose.yaml \ From 79cdef1795788ca8a94c2b03a7781f5c06db954c Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Fri, 21 Jun 2019 11:39:20 +1000 Subject: [PATCH 103/105] Update pocs-docker.sh Missing line continuation --- scripts/pocs-docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/pocs-docker.sh b/scripts/pocs-docker.sh index 609cefd4d..d49713afe 100755 --- a/scripts/pocs-docker.sh +++ b/scripts/pocs-docker.sh @@ -36,7 +36,7 @@ fi cd "$PANDIR" docker-compose \ - --project-directory "${PANDIR}" + --project-directory "${PANDIR}" \ -f panoptes-utils/docker/docker-compose.yaml \ -f PAWS/docker/docker-compose.yaml \ -f POCS/docker/docker-compose.yaml \ From 9b91611dfb601d07a035efe2d3f0c8e55d34a0f2 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Fri, 21 Jun 2019 12:13:33 +1000 Subject: [PATCH 104/105] Update peas uPDATE FILE NAME --- bin/peas | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/peas b/bin/peas index 14e0a8d4e..aaa4bc36a 100755 --- a/bin/peas +++ b/bin/peas @@ -1,3 +1,3 @@ #!/bin/bash -e -docker exec -it peas-shell python $POCS/scripts/peas-shell.py +docker exec -it peas-shell python $POCS/scripts/peas_shell From 85118ed13ee8e5af1e28703ecc9a372e43531636 Mon Sep 17 00:00:00 2001 From: Wilfred Tyler Gee Date: Fri, 21 Jun 2019 12:13:51 +1000 Subject: [PATCH 105/105] Update pocs Upate file name --- bin/pocs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/pocs b/bin/pocs index 89dadceae..c16c9062a 100755 --- a/bin/pocs +++ b/bin/pocs @@ -1,3 +1,3 @@ #!/bin/bash -e -docker exec -it peas-shell python $POCS/scripts/pocs-shell.py +docker exec -it peas-shell python $POCS/scripts/pocs_shell