From ea44b144a06fca48016ee4bfe81fe63ceb4057b2 Mon Sep 17 00:00:00 2001 From: Wilfred Gee Date: Mon, 1 Oct 2018 10:30:28 +1000 Subject: [PATCH] Remove cloud-sql information * Some of this will probably move over to the panoptes/panoptes-network repo. --- conf_files/pocs.yaml | 9 - pocs/utils/db/__init__.py | 0 pocs/utils/db/postgres.py | 196 ---------------------- scripts/connect_clouddb_proxy.py | 74 -------- scripts/startup/start_cloudsql_proxy.sh | 13 -- scripts/startup/start_panoptes_in_tmux.sh | 4 - 6 files changed, 296 deletions(-) delete mode 100644 pocs/utils/db/__init__.py delete mode 100644 pocs/utils/db/postgres.py delete mode 100755 scripts/connect_clouddb_proxy.py delete mode 100755 scripts/startup/start_cloudsql_proxy.sh diff --git a/conf_files/pocs.yaml b/conf_files/pocs.yaml index 6f08fd2d9..b450125ed 100644 --- a/conf_files/pocs.yaml +++ b/conf_files/pocs.yaml @@ -78,15 +78,6 @@ panoptes_network: project_id: panoptes-survey buckets: images: panoptes-survey - cloudsql_instances: - meta: - location: us-central1 - database: panoptes-meta - local_port: 5432 - tess: - location: us-central1 - database: tess-catalog - local_port: 5433 #Enable to output POCS messages to social accounts # social_accounts: diff --git a/pocs/utils/db/__init__.py b/pocs/utils/db/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/pocs/utils/db/postgres.py b/pocs/utils/db/postgres.py deleted file mode 100644 index 775fdf083..000000000 --- a/pocs/utils/db/postgres.py +++ /dev/null @@ -1,196 +0,0 @@ -import os - -from warnings import warn - -import psycopg2 -from psycopg2.extras import DictCursor -from astropy.wcs import WCS - - -def get_db_proxy_conn( - host='127.0.0.1', - port=5432, - db_name='panoptes', - db_user='panoptes', - db_pass=None, - **kwargs): - """Return postgres connection to local proxy. - - Note: - The proxy must be started and authenticated to the appropriate instance - before this function will work. See `$POCS/scripts/connect_clouddb_proxy.py`. - - Args: - host (str, optional): Hostname, default localhost. - port (str, optional): Port, default 5432. - db_user (str, optional): Name of db user, default 'panoptes'. - db_name (str, optional): Name of db, default 'postgres'. - db_pass (str, optional): Password for given db and user, defaults to - $PG_PASSWORD or None if not set. - - Returns: - `psycopg2.Connection`: DB connection object. - """ - if db_pass is None: - db_pass = os.getenv('PGPASSWORD') - - conn_params = { - 'host': host, - 'port': port, - 'user': db_user, - 'dbname': db_name, - 'password': db_pass, - } - - conn = psycopg2.connect(**conn_params) - return conn - - -def get_cursor(**kwargs): - """Get a Cursor object. - - Args: - **kwargs: Passed to `get_db_prox_conn` - - Returns: - `psycopg2.Cursor`: Cursor object. - """ - conn = get_db_proxy_conn(**kwargs) - cur = conn.cursor(cursor_factory=DictCursor) - - return cur - - -def meta_insert(table, conn=None, logger=None, **kwargs): - """Inserts arbitrary key/value pairs into a table. - - Args: - table (str): Table in which to insert. - conn (None, optional): DB connection, if None then `get_db_proxy_conn` - is used. - logger (None, optional): A logger. - **kwargs: List of key/value pairs corresponding to columns in the - table. - - Returns: - tuple|None: Returns the inserted row or None. - """ - if conn is None: - conn = get_db_proxy_conn() - - cur = conn.cursor() - - col_names = ','.join(kwargs.keys()) - col_val_holders = ','.join(['%s' for _ in range(len(kwargs))]) - - insert_sql = 'INSERT INTO {} ({}) VALUES ({}) ON CONFLICT DO NOTHING RETURNING *'.format( - table, col_names, col_val_holders) - - try: - cur.execute(insert_sql, list(kwargs.values())) - conn.commit() - return cur.fetchone() - except Exception as e: - conn.rollback() - warn("Error on fetch: {}".format(e)) - if logger: - logger.log_text("Can't insert row: {}".format(e)) - return None - - -def add_header_to_db(header, conn=None, logger=None): - """Add FITS image info to metadb. - - Note: - This function doesn't check header for proper entries and - assumes a large list of keywords. See source for details. - - Args: - header (dict): FITS Header data from an observation. - conn (None, optional): DB connection, if None then `get_db_proxy_conn` - is used. - logger (None, optional): A logger. - - Returns: - str: The image_id. - """ - unit_id = int(header['OBSERVER'].strip().replace('PAN', '')) - seq_id = header['SEQID'].strip() - img_id = header['IMAGEID'].strip() - camera_id = header['INSTRUME'].strip() - - unit_data = { - 'id': unit_id, - 'name': header['OBSERVER'].strip(), - 'lat': float(header['LAT-OBS']), - 'lon': float(header['LONG-OBS']), - 'elevation': float(header['ELEV-OBS']), - } - meta_insert('units', conn=conn, logger=logger, **unit_data) - - camera_data = { - 'unit_id': unit_id, - 'id': camera_id, - } - meta_insert('cameras', conn=conn, logger=logger, **camera_data) - - seq_data = { - 'id': seq_id, - 'unit_id': unit_id, - 'start_date': header['SEQID'].split('_')[-1], - 'exp_time': header['EXPTIME'], - 'ra_rate': header['RA-RATE'], - 'pocs_version': header['CREATOR'], - 'piaa_state': header['PSTATE'], - } - logger.log_text("Inserting sequence: {}".format(seq_data)) - try: - bl, tl, tr, br = WCS(header).calc_footprint() # Corners - seq_data['coord_bounds'] = '(({}, {}), ({}, {}))'.format( - bl[0], bl[1], - tr[0], tr[1] - ) - meta_insert('sequences', conn=conn, logger=logger, **seq_data) - logger.log_text("Sequence inserted: {}".format(seq_id)) - except Exception as e: - logger.log_text("Can't get bounds: {}".format(e)) - if 'coord_bounds' in seq_data: - del seq_data['coord_bounds'] - try: - meta_insert('sequences', conn=conn, logger=logger, **seq_data) - except Exception as e: - logger.log_text("Can't insert sequence: {}".format(seq_id)) - raise e - - image_data = { - 'id': img_id, - 'sequence_id': seq_id, - 'date_obs': header['DATE-OBS'], - 'moon_fraction': header['MOONFRAC'], - 'moon_separation': header['MOONSEP'], - 'ra_mnt': header['RA-MNT'], - 'ha_mnt': header['HA-MNT'], - 'dec_mnt': header['DEC-MNT'], - 'airmass': header['AIRMASS'], - 'exp_time': header['EXPTIME'], - 'iso': header['ISO'], - 'camera_id': camera_id, - 'cam_temp': header['CAMTEMP'].split(' ')[0], - 'cam_colortmp': header['COLORTMP'], - 'cam_circconf': header['CIRCCONF'].split(' ')[0], - 'cam_measrggb': header['MEASRGGB'], - 'cam_red_balance': header['REDBAL'], - 'cam_blue_balance': header['BLUEBAL'], - 'file_path': header['FILEPATH'] - } - - # Add plate-solved info. - try: - image_data['center_ra'] = header['CRVAL1'] - image_data['center_dec'] = header['CRVAL2'] - except KeyError: - pass - - img_row = meta_insert('images', conn=conn, logger=logger, **image_data) - - return img_row diff --git a/scripts/connect_clouddb_proxy.py b/scripts/connect_clouddb_proxy.py deleted file mode 100755 index 823547653..000000000 --- a/scripts/connect_clouddb_proxy.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env python3 - -######################################################################## -# connect_clouddb_proxy.py -# -# This is a simple wrapper that looks up the connection information in the -# config and establishes a local proxy. -######################################################################## - - -import os -import sys -import subprocess -from pprint import pprint - -from pocs.utils.config import load_config - - -def main(instance_id, local_port, verbose=False): - proxy_cmd = os.path.join(os.environ['POCS'], 'bin', 'cloud_sql_proxy') - - connection_str = '{}=tcp:{}'.format(instance_id, local_port) - instances_arg = '-instances={}'.format(connection_str) - - assert os.path.isfile(proxy_cmd) - - run_proxy_cmd = [proxy_cmd, instances_arg] - - stdout_handler = subprocess.PIPE - if verbose: - stdout_handler = None - - try: - subprocess.run(run_proxy_cmd, stdout=stdout_handler, stderr=stdout_handler) - except KeyboardInterrupt: - print("Shutting down") - - -if __name__ == '__main__': - import argparse - - # Get the command line option - parser = argparse.ArgumentParser(description="Connect to a google CloudSQL via a local proxy") - parser.add_argument('-d', '--database', default=None, required=True, - help="Database, currently 'meta' or 'tess' from the config.") - parser.add_argument('-v', '--verbose', action='store_true', default=False, - help="Print results to stdout") - args = parser.parse_args() - - config = load_config() - try: - network_config = config['panoptes_network'] - if args.verbose: - print("Found config:") - pprint(network_config) - instance_info = network_config['cloudsql_instances'][args.database] - - # Get connection details - project_id = network_config['project_id'] - server_location = instance_info['location'] - db_name = instance_info['database'] - local_port = instance_info['local_port'] - - instance_id = '{}:{}:{}'.format(project_id, server_location, db_name) - - except KeyError as e: - print("Invalid configuration. Check panoptes_network config.") - print(e) - sys.exit(1) - - if args.verbose: - print("Connecting to {} on local port {}".format(instance_id, local_port)) - - main(instance_id, local_port, verbose=args.verbose) diff --git a/scripts/startup/start_cloudsql_proxy.sh b/scripts/startup/start_cloudsql_proxy.sh deleted file mode 100755 index bfe867c79..000000000 --- a/scripts/startup/start_cloudsql_proxy.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -ex - -WINDOW="${1}" -DB="${2:-meta}" -echo "Running $(basename "${0}") at $(date), WINDOW=${WINDOW}" -echo "Creating proxy connection to ${DB} database" - -tmux send-keys -t "${WINDOW}" "date" C-m -sleep 0.5s -tmux send-keys -t "${WINDOW}" \ - "python $POCS/scripts/connect_clouddb_proxy --database ${DB} --verbose" C-m - -echo "Done at $(date)" diff --git a/scripts/startup/start_panoptes_in_tmux.sh b/scripts/startup/start_panoptes_in_tmux.sh index 3319292f7..dd44f674c 100755 --- a/scripts/startup/start_panoptes_in_tmux.sh +++ b/scripts/startup/start_panoptes_in_tmux.sh @@ -99,10 +99,6 @@ set -x # These provide connections between message publishers and subscribers. create_and_init_window msg_hub start_messaging_hub.sh -# Create a window running the Google CloudSQL proxy script, connecting -# to the meta database. -create_and_init_window meta_db start_cloudsql_proxy.sh meta - # Create a window running the social messaging program. It subscribes to # zeromq and posts some messages to Twitter or Slack, if # conf_files/pocs_local.yaml has the appropriate config values; if not,