From 66f420cc30c4a78c50d0c197c7fedcd8e923b3a6 Mon Sep 17 00:00:00 2001 From: Luca Date: Thu, 7 Jun 2018 08:59:09 +1000 Subject: [PATCH 1/6] First implementation social sinks for POCS messages --- conf_files/pocs.yaml | 11 +++++ pocs/core.py | 84 +++++++++++++++++++++++++++++++++++- pocs/utils/social_slack.py | 25 +++++++++++ pocs/utils/social_twitter.py | 32 ++++++++++++++ requirements.txt | 2 + 5 files changed, 153 insertions(+), 1 deletion(-) create mode 100644 pocs/utils/social_slack.py create mode 100644 pocs/utils/social_twitter.py diff --git a/conf_files/pocs.yaml b/conf_files/pocs.yaml index fa2843a8b..7c5fc8605 100644 --- a/conf_files/pocs.yaml +++ b/conf_files/pocs.yaml @@ -52,4 +52,15 @@ messaging: # Must match ports in peas.yaml. cmd_port: 6500 msg_port: 6510 +#Enable to output POCS messages to social accounts +# social_accounts: +# twitter: +# consumer_key: [your_consumer_key] +# consumer_secret: [your_consumer_secret] +# access_token: [your_access_token] +# access_token_secret: [your_access_token_secret] +# slack: +# webhook_url: [your_webhook_url] +# output_timestamp: True + state_machine: simple_state_table diff --git a/pocs/core.py b/pocs/core.py index 1e47b1985..52b9ae171 100644 --- a/pocs/core.py +++ b/pocs/core.py @@ -14,7 +14,8 @@ from pocs.utils import current_time from pocs.utils import get_free_space from pocs.utils.messaging import PanMessaging - +from pocs.utils.social_twitter import SocialTwitter +from pocs.utils.social_slack import SocialSlack class POCS(PanStateMachine, PanBase): @@ -571,10 +572,91 @@ def check_message_loop(cmd_queue): target=check_message_loop, args=(self._cmd_queue,)) check_messages_process.name = 'MessageCheckLoop' check_messages_process.start() + self.logger.debug('Command message subscriber set up on port {}'.format(cmd_port)) + + def check_social_messages_loop(): + cmd_social_subscriber = PanMessaging.create_subscriber(msg_port + 1, 'PANCHAT') + + poller = zmq.Poller() + poller.register(cmd_social_subscriber.socket, zmq.POLLIN) + + try: + while self._do_social_msg_check: + # Poll for messages + sockets = dict(poller.poll(500)) # 500 ms timeout + + if cmd_social_subscriber.socket in sockets and \ + sockets[cmd_social_subscriber.socket] == zmq.POLLIN: + + msg_type, msg_obj = cmd_social_subscriber.receive_message(flags=zmq.NOBLOCK) + + #Check the various social sinks + if self.social_twitter is not None: + self.social_twitter.send_message(msg_obj['message'], msg_obj['timestamp']) + + if self.social_slack is not None: + self.social_slack.send_message(msg_obj['message'], msg_obj['timestamp']) + + time.sleep(1) + except KeyboardInterrupt: + pass + + social_messages_process = multiprocessing.Process( + target=check_social_messages_loop, args=()) + social_messages_process.name = 'SocialMessageCheckLoop' + + #Initialise social sinks to None + self.social_twitter = None + self.social_slack = None + if 'social_accounts' in self.config: + self._do_social_msg_check = True + socialConfig = self.config['social_accounts'] + + #See which social sink we can create based on config + + #Twitter sink + if socialConfig is not None and 'twitter' in socialConfig: + twitterConfig = socialConfig['twitter'] + if twitterConfig is not None and \ + 'consumer_key' in twitterConfig and \ + 'consumer_secret' in twitterConfig and \ + 'access_token' in twitterConfig and \ + 'access_token_secret' in twitterConfig: + + #Output timestamp should always be True in Twitter + #otherwise Twitter will reject duplicate statuses + if 'output_timestamp' in twitterConfig: + output_timestamp = twitterConfig['output_timestamp'] + else: + output_timestamp = True + + self.social_twitter = SocialTwitter(twitterConfig['consumer_key'], + twitterConfig['consumer_secret'], + twitterConfig['access_token'], + twitterConfig['access_token_secret'], + output_timestamp) + + #Slack sink + if socialConfig is not None and 'slack' in socialConfig: + slackConfig = socialConfig['slack'] + if slackConfig is not None and \ + 'webhook_url' in slackConfig: + + if 'output_timestamp' in slackConfig: + output_timestamp = slackConfig['output_timestamp'] + else: + output_timestamp = True + + self.social_slack = SocialSlack(slackConfig['webhook_url'], + output_timestamp) + + self.logger.debug('Starting social subscriber message loop') + social_messages_process.start() self._processes = { 'check_messages': check_messages_process, 'cmd_forwarder': cmd_forwarder_process, 'msg_forwarder': msg_forwarder_process, + 'social_messages': social_messages_process, } diff --git a/pocs/utils/social_slack.py b/pocs/utils/social_slack.py new file mode 100644 index 000000000..7611efcfc --- /dev/null +++ b/pocs/utils/social_slack.py @@ -0,0 +1,25 @@ +import requests + +from pocs.utils import current_time +from pocs.utils.logger import get_root_logger + +class SocialSlack(object): + + """Messaging class to output to Slack + """ + logger = get_root_logger() + + def __init__(self, web_hook, output_timestamp): + self.output_timestamp = output_timestamp + self.web_hook = web_hook + + def send_message(self, msg, timestamp): + try: + if self.output_timestamp: + post_msg = '{} - {}'.format(timestamp, msg) + else: + post_msg = msg + + response = requests.post(self.web_hook, json={"text": post_msg}) + except Exception as e: + self.logger.debug("Error posting to slack: {}".format(e)) \ No newline at end of file diff --git a/pocs/utils/social_twitter.py b/pocs/utils/social_twitter.py new file mode 100644 index 000000000..401a36bea --- /dev/null +++ b/pocs/utils/social_twitter.py @@ -0,0 +1,32 @@ +import tweepy + +from pocs.utils import current_time +from pocs.utils.logger import get_root_logger + +class SocialTwitter(object): + + """Messaging class to output to Twitter + """ + logger = get_root_logger() + + def __init__(self, consumer_key, consumer_secret, access_token, access_token_secret, output_timestamp): + # Create a new twitter api object + try: + auth = tweepy.OAuthHandler(consumer_key, consumer_secret) + auth.set_access_token(access_token, access_token_secret) + + self.api = tweepy.API(auth) + self.output_timestamp = output_timestamp + + except tweepy.TweepError as e: + self.logger.error('Error connecting to Twitter. Err: {} - Message: {}'.format(e.args[0][0]['code'], e.args[0][0]['message'])) + + + def send_message(self, msg, timestamp): + try: + if self.output_timestamp: + retStatus = self.api.update_status('{} - {}'.format(timestamp, msg)) + else: + retStatus = self.api.update_status(msg) + except tweepy.TweepError as e: + self.logger.debug('Error tweeting message. Err: {} - Message: {}'.format(e.args[0][0]['code'], e.args[0][0]['message'])) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 8aef75afc..e40aa5a80 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,3 +22,5 @@ scikit_image >= 0.12.3 scipy >= 0.17.1 transitions >= 0.4.0 wcsaxes +tweepy + From 19f6935ddc4bb622d4af7f4acad5b8bea5d0fadf Mon Sep 17 00:00:00 2001 From: Luca Date: Thu, 7 Jun 2018 09:12:01 +1000 Subject: [PATCH 2/6] Fixed code conventions to pass test --- pocs/core.py | 42 ++++++++++++++++++------------------ pocs/utils/social_slack.py | 3 ++- pocs/utils/social_twitter.py | 4 ++-- 3 files changed, 25 insertions(+), 24 deletions(-) diff --git a/pocs/core.py b/pocs/core.py index 52b9ae171..1cfea4bac 100644 --- a/pocs/core.py +++ b/pocs/core.py @@ -17,6 +17,7 @@ from pocs.utils.social_twitter import SocialTwitter from pocs.utils.social_slack import SocialSlack + class POCS(PanStateMachine, PanBase): """The main class representing the Panoptes Observatory Control Software (POCS). @@ -574,7 +575,7 @@ def check_message_loop(cmd_queue): check_messages_process.start() self.logger.debug('Command message subscriber set up on port {}'.format(cmd_port)) - + def check_social_messages_loop(): cmd_social_subscriber = PanMessaging.create_subscriber(msg_port + 1, 'PANCHAT') @@ -591,7 +592,7 @@ def check_social_messages_loop(): msg_type, msg_obj = cmd_social_subscriber.receive_message(flags=zmq.NOBLOCK) - #Check the various social sinks + # Check the various social sinks if self.social_twitter is not None: self.social_twitter.send_message(msg_obj['message'], msg_obj['timestamp']) @@ -606,42 +607,41 @@ def check_social_messages_loop(): target=check_social_messages_loop, args=()) social_messages_process.name = 'SocialMessageCheckLoop' - #Initialise social sinks to None + # Initialise social sinks to None self.social_twitter = None self.social_slack = None if 'social_accounts' in self.config: self._do_social_msg_check = True socialConfig = self.config['social_accounts'] - #See which social sink we can create based on config + # See which social sink we can create based on config - #Twitter sink - if socialConfig is not None and 'twitter' in socialConfig: + # Twitter sink + if socialConfig is not None and 'twitter' in socialConfig: twitterConfig = socialConfig['twitter'] if twitterConfig is not None and \ - 'consumer_key' in twitterConfig and \ - 'consumer_secret' in twitterConfig and \ - 'access_token' in twitterConfig and \ - 'access_token_secret' in twitterConfig: - - #Output timestamp should always be True in Twitter - #otherwise Twitter will reject duplicate statuses + 'consumer_key' in twitterConfig and \ + 'consumer_secret' in twitterConfig and \ + 'access_token' in twitterConfig and \ + 'access_token_secret' in twitterConfig: + # Output timestamp should always be True in Twitter + # otherwise Twitter will reject duplicate statuses if 'output_timestamp' in twitterConfig: output_timestamp = twitterConfig['output_timestamp'] else: output_timestamp = True self.social_twitter = SocialTwitter(twitterConfig['consumer_key'], - twitterConfig['consumer_secret'], - twitterConfig['access_token'], - twitterConfig['access_token_secret'], - output_timestamp) + twitterConfig['consumer_secret'], + twitterConfig['access_token'], + twitterConfig['access_token_secret'], + output_timestamp) - #Slack sink - if socialConfig is not None and 'slack' in socialConfig: + # Slack sink + if socialConfig is not None and 'slack' in socialConfig: slackConfig = socialConfig['slack'] if slackConfig is not None and \ - 'webhook_url' in slackConfig: + 'webhook_url' in slackConfig: if 'output_timestamp' in slackConfig: output_timestamp = slackConfig['output_timestamp'] @@ -649,7 +649,7 @@ def check_social_messages_loop(): output_timestamp = True self.social_slack = SocialSlack(slackConfig['webhook_url'], - output_timestamp) + output_timestamp) self.logger.debug('Starting social subscriber message loop') social_messages_process.start() diff --git a/pocs/utils/social_slack.py b/pocs/utils/social_slack.py index 7611efcfc..1dc0daef3 100644 --- a/pocs/utils/social_slack.py +++ b/pocs/utils/social_slack.py @@ -3,6 +3,7 @@ from pocs.utils import current_time from pocs.utils.logger import get_root_logger + class SocialSlack(object): """Messaging class to output to Slack @@ -22,4 +23,4 @@ def send_message(self, msg, timestamp): response = requests.post(self.web_hook, json={"text": post_msg}) except Exception as e: - self.logger.debug("Error posting to slack: {}".format(e)) \ No newline at end of file + self.logger.debug("Error posting to slack: {}".format(e)) diff --git a/pocs/utils/social_twitter.py b/pocs/utils/social_twitter.py index 401a36bea..e53512656 100644 --- a/pocs/utils/social_twitter.py +++ b/pocs/utils/social_twitter.py @@ -3,6 +3,7 @@ from pocs.utils import current_time from pocs.utils.logger import get_root_logger + class SocialTwitter(object): """Messaging class to output to Twitter @@ -21,7 +22,6 @@ def __init__(self, consumer_key, consumer_secret, access_token, access_token_sec except tweepy.TweepError as e: self.logger.error('Error connecting to Twitter. Err: {} - Message: {}'.format(e.args[0][0]['code'], e.args[0][0]['message'])) - def send_message(self, msg, timestamp): try: if self.output_timestamp: @@ -29,4 +29,4 @@ def send_message(self, msg, timestamp): else: retStatus = self.api.update_status(msg) except tweepy.TweepError as e: - self.logger.debug('Error tweeting message. Err: {} - Message: {}'.format(e.args[0][0]['code'], e.args[0][0]['message'])) \ No newline at end of file + self.logger.debug('Error tweeting message. Err: {} - Message: {}'.format(e.args[0][0]['code'], e.args[0][0]['message'])) From faa4ac9f02b9f4a9dc6ab9c4c4611759b18ee25c Mon Sep 17 00:00:00 2001 From: Luca Date: Sat, 16 Jun 2018 16:41:32 +1000 Subject: [PATCH 3/6] Moved code to separate script. Amended based on feedback. --- conf_files/pocs.yaml | 2 +- pocs/core.py | 82 ----------- pocs/utils/social_slack.py | 1 - pocs/utils/social_twitter.py | 6 +- requirements.txt | 3 +- scripts/run_social_messaging.py | 167 ++++++++++++++++++++++ scripts/startup/start_social_messaging.sh | 11 ++ 7 files changed, 182 insertions(+), 90 deletions(-) create mode 100755 scripts/run_social_messaging.py create mode 100755 scripts/startup/start_social_messaging.sh diff --git a/conf_files/pocs.yaml b/conf_files/pocs.yaml index 7c5fc8605..1a6e3048d 100644 --- a/conf_files/pocs.yaml +++ b/conf_files/pocs.yaml @@ -61,6 +61,6 @@ messaging: # access_token_secret: [your_access_token_secret] # slack: # webhook_url: [your_webhook_url] -# output_timestamp: True +# output_timestamp: False state_machine: simple_state_table diff --git a/pocs/core.py b/pocs/core.py index 1cfea4bac..1e47b1985 100644 --- a/pocs/core.py +++ b/pocs/core.py @@ -14,8 +14,6 @@ from pocs.utils import current_time from pocs.utils import get_free_space from pocs.utils.messaging import PanMessaging -from pocs.utils.social_twitter import SocialTwitter -from pocs.utils.social_slack import SocialSlack class POCS(PanStateMachine, PanBase): @@ -573,90 +571,10 @@ def check_message_loop(cmd_queue): target=check_message_loop, args=(self._cmd_queue,)) check_messages_process.name = 'MessageCheckLoop' check_messages_process.start() - self.logger.debug('Command message subscriber set up on port {}'.format(cmd_port)) - def check_social_messages_loop(): - cmd_social_subscriber = PanMessaging.create_subscriber(msg_port + 1, 'PANCHAT') - - poller = zmq.Poller() - poller.register(cmd_social_subscriber.socket, zmq.POLLIN) - - try: - while self._do_social_msg_check: - # Poll for messages - sockets = dict(poller.poll(500)) # 500 ms timeout - - if cmd_social_subscriber.socket in sockets and \ - sockets[cmd_social_subscriber.socket] == zmq.POLLIN: - - msg_type, msg_obj = cmd_social_subscriber.receive_message(flags=zmq.NOBLOCK) - - # Check the various social sinks - if self.social_twitter is not None: - self.social_twitter.send_message(msg_obj['message'], msg_obj['timestamp']) - - if self.social_slack is not None: - self.social_slack.send_message(msg_obj['message'], msg_obj['timestamp']) - - time.sleep(1) - except KeyboardInterrupt: - pass - - social_messages_process = multiprocessing.Process( - target=check_social_messages_loop, args=()) - social_messages_process.name = 'SocialMessageCheckLoop' - - # Initialise social sinks to None - self.social_twitter = None - self.social_slack = None - if 'social_accounts' in self.config: - self._do_social_msg_check = True - socialConfig = self.config['social_accounts'] - - # See which social sink we can create based on config - - # Twitter sink - if socialConfig is not None and 'twitter' in socialConfig: - twitterConfig = socialConfig['twitter'] - if twitterConfig is not None and \ - 'consumer_key' in twitterConfig and \ - 'consumer_secret' in twitterConfig and \ - 'access_token' in twitterConfig and \ - 'access_token_secret' in twitterConfig: - # Output timestamp should always be True in Twitter - # otherwise Twitter will reject duplicate statuses - if 'output_timestamp' in twitterConfig: - output_timestamp = twitterConfig['output_timestamp'] - else: - output_timestamp = True - - self.social_twitter = SocialTwitter(twitterConfig['consumer_key'], - twitterConfig['consumer_secret'], - twitterConfig['access_token'], - twitterConfig['access_token_secret'], - output_timestamp) - - # Slack sink - if socialConfig is not None and 'slack' in socialConfig: - slackConfig = socialConfig['slack'] - if slackConfig is not None and \ - 'webhook_url' in slackConfig: - - if 'output_timestamp' in slackConfig: - output_timestamp = slackConfig['output_timestamp'] - else: - output_timestamp = True - - self.social_slack = SocialSlack(slackConfig['webhook_url'], - output_timestamp) - - self.logger.debug('Starting social subscriber message loop') - social_messages_process.start() - self._processes = { 'check_messages': check_messages_process, 'cmd_forwarder': cmd_forwarder_process, 'msg_forwarder': msg_forwarder_process, - 'social_messages': social_messages_process, } diff --git a/pocs/utils/social_slack.py b/pocs/utils/social_slack.py index 1dc0daef3..122db23e8 100644 --- a/pocs/utils/social_slack.py +++ b/pocs/utils/social_slack.py @@ -1,6 +1,5 @@ import requests -from pocs.utils import current_time from pocs.utils.logger import get_root_logger diff --git a/pocs/utils/social_twitter.py b/pocs/utils/social_twitter.py index e53512656..c804916a0 100644 --- a/pocs/utils/social_twitter.py +++ b/pocs/utils/social_twitter.py @@ -1,6 +1,5 @@ import tweepy -from pocs.utils import current_time from pocs.utils.logger import get_root_logger @@ -18,9 +17,8 @@ def __init__(self, consumer_key, consumer_secret, access_token, access_token_sec self.api = tweepy.API(auth) self.output_timestamp = output_timestamp - except tweepy.TweepError as e: - self.logger.error('Error connecting to Twitter. Err: {} - Message: {}'.format(e.args[0][0]['code'], e.args[0][0]['message'])) + self.logger.warning('Error connecting to Twitter. Err: {} - Message: {}'.format(e.args[0][0]['code'], e.args[0][0]['message'])) def send_message(self, msg, timestamp): try: @@ -29,4 +27,4 @@ def send_message(self, msg, timestamp): else: retStatus = self.api.update_status(msg) except tweepy.TweepError as e: - self.logger.debug('Error tweeting message. Err: {} - Message: {}'.format(e.args[0][0]['code'], e.args[0][0]['message'])) + self.logger.debug('Error tweeting message. Please check your Twitter configuration.') diff --git a/requirements.txt b/requirements.txt index e40aa5a80..97f9fc54d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,6 +21,5 @@ readline scikit_image >= 0.12.3 scipy >= 0.17.1 transitions >= 0.4.0 -wcsaxes tweepy - +wcsaxes diff --git a/scripts/run_social_messaging.py b/scripts/run_social_messaging.py new file mode 100755 index 000000000..c76743f92 --- /dev/null +++ b/scripts/run_social_messaging.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python + +import argparse +import sys +import threading +import time +import zmq + +from pocs.utils.config import load_config +from pocs.utils.logger import get_root_logger +from pocs.utils.messaging import PanMessaging +from pocs.utils.social_twitter import SocialTwitter +from pocs.utils.social_slack import SocialSlack + +the_root_logger = None + + +def say(fmt, *args, error=False): + if args: + msg = fmt.format(*args) + else: + msg = fmt + if error: + print(msg, file=sys.stderr) + the_root_logger.error(msg) + else: + print(msg) + the_root_logger.info(msg) + + +def check_social_messages_loop(msg_port, social_twitter, social_slack): + cmd_social_subscriber = PanMessaging.create_subscriber(msg_port, 'PANCHAT') + + poller = zmq.Poller() + poller.register(cmd_social_subscriber.socket, zmq.POLLIN) + + try: + while True: + # Poll for messages + sockets = dict(poller.poll(500)) # 500 ms timeout + + if cmd_social_subscriber.socket in sockets and \ + sockets[cmd_social_subscriber.socket] == zmq.POLLIN: + + msg_type, msg_obj = cmd_social_subscriber.receive_message(flags=zmq.NOBLOCK) + + # Check the various social sinks + if social_twitter is not None: + social_twitter.send_message(msg_obj['message'], msg_obj['timestamp']) + + if social_slack is not None: + social_slack.send_message(msg_obj['message'], msg_obj['timestamp']) + + time.sleep(1) + except KeyboardInterrupt: + pass + + +def run_social_sinks(msg_port, social_twitter, social_slack): + the_root_logger.info('Creating sockets') + + threads = [] + + name='social_messaging' + + t = threading.Thread( + target=check_social_messages_loop, name=name, args=(msg_port, social_twitter, social_slack), daemon=True) + the_root_logger.info('Starting thread {}', name) + t.start() + threads.append(t) + + time.sleep(0.05) + if not any([t.is_alive() for t in threads]): + say('Failed to start social sinks thread!', error=True) + sys.exit(1) + else: + the_root_logger.info('Started social messaging') + print() + print('Hit Ctrl-c to stop') + try: + # Keep running until they've all died. + while threads: + for t in threads: + t.join(timeout=100) + if t.is_alive(): + continue + say('Thread {} has stopped', t.name, error=True) + threads.remove(t) + break + # If we get here, then the forwarders died for some reason. + sys.exit(1) + except KeyboardInterrupt: + sys.exit(0) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Run social messaging to forward platform messages to social channels.') + parser.add_argument( + '--from_config', + action='store_true', + help='Read social channels config from the pocs.yaml and pocs_local.yaml config files.') + args = parser.parse_args() + + def arg_error(msg): + print(msg, file=sys.stderr) + parser.print_help() + sys.exit(1) + + # Initialise social sinks to None + social_twitter = None + social_slack = None + + if args.from_config: + config = load_config(config_files=['pocs']) + + if 'social_accounts' in config: + social_config = config['social_accounts'] + + # See which social sink we can create based on config + + # Twitter sink + if social_config is not None and 'twitter' in social_config: + twitter_config = social_config['twitter'] + if twitter_config is not None and \ + 'consumer_key' in twitter_config and \ + 'consumer_secret' in twitter_config and \ + 'access_token' in twitter_config and \ + 'access_token_secret' in twitter_config: + # Output timestamp should always be True in Twitter + # otherwise Twitter will reject duplicate statuses + if 'output_timestamp' in twitter_config: + output_timestamp = twitter_config['output_timestamp'] + else: + output_timestamp = True + + social_twitter = SocialTwitter(twitter_config['consumer_key'], + twitter_config['consumer_secret'], + twitter_config['access_token'], + twitter_config['access_token_secret'], + output_timestamp) + + # Slack sink + if social_config is not None and 'slack' in social_config: + slack_config = social_config['slack'] + if slack_config is not None and \ + 'webhook_url' in slack_config: + + if 'output_timestamp' in slack_config: + output_timestamp = slack_config['output_timestamp'] + else: + output_timestamp = False + + social_slack = SocialSlack(slack_config['webhook_url'], + output_timestamp) + else: + arg_error('social_accounts setting not defined in config.') + + if not social_twitter and not social_slack: + arg_error('No social messaging sinks could be initialised. Please check your config settings.') + + # Messaging port to subscribe on + msg_port = config['messaging']['msg_port'] + 1 + + the_root_logger = get_root_logger() + + run_social_sinks(msg_port, social_twitter, social_slack) diff --git a/scripts/startup/start_social_messaging.sh b/scripts/startup/start_social_messaging.sh new file mode 100755 index 000000000..1b9125c78 --- /dev/null +++ b/scripts/startup/start_social_messaging.sh @@ -0,0 +1,11 @@ +#!/bin/bash -ex + +WINDOW="${1}" +echo "Running $(basename "${0}") at $(date), WINDOW=${WINDOW}" + +tmux send-keys -t "${WINDOW}" "date" C-m +sleep 0.5s +tmux send-keys -t "${WINDOW}" \ + "python $POCS/scripts/run_social_messaging.py --from_config" C-m + +echo "Done at $(date)" From 9af9492c8b92d916a0a81e470d31ecef2d428bef Mon Sep 17 00:00:00 2001 From: Luca Date: Tue, 19 Jun 2018 12:01:54 +1000 Subject: [PATCH 4/6] Addressed review points and refactored sink classes --- pocs/utils/social_slack.py | 13 +++++--- pocs/utils/social_twitter.py | 27 +++++++++++++--- scripts/run_social_messaging.py | 56 +++++++++++---------------------- 3 files changed, 49 insertions(+), 47 deletions(-) diff --git a/pocs/utils/social_slack.py b/pocs/utils/social_slack.py index 122db23e8..0cf6227d3 100644 --- a/pocs/utils/social_slack.py +++ b/pocs/utils/social_slack.py @@ -5,13 +5,16 @@ class SocialSlack(object): - """Messaging class to output to Slack - """ + """Social Messaging sink to output to Slack.""" + logger = get_root_logger() - def __init__(self, web_hook, output_timestamp): - self.output_timestamp = output_timestamp - self.web_hook = web_hook + def __init__(self, **kwargs): + self.web_hook = kwargs.get("webhook_url") + if self.web_hook: + self.output_timestamp = kwargs.get("output_timestamp", False) + else: + raise ValueError('webhook_url parameter is not defined.') def send_message(self, msg, timestamp): try: diff --git a/pocs/utils/social_twitter.py b/pocs/utils/social_twitter.py index c804916a0..a027472af 100644 --- a/pocs/utils/social_twitter.py +++ b/pocs/utils/social_twitter.py @@ -5,20 +5,37 @@ class SocialTwitter(object): - """Messaging class to output to Twitter - """ + """Social Messaging sink to output to Twitter.""" + logger = get_root_logger() - def __init__(self, consumer_key, consumer_secret, access_token, access_token_secret, output_timestamp): + def __init__(self, **kwargs): + consumer_key = kwargs.get("consumer_key") + if consumer_key is None: + raise ValueError('consumer_key parameter is not defined.') + consumer_secret = kwargs.get("consumer_secret") + if consumer_secret is None: + raise ValueError('consumer_secret parameter is not defined.') + access_token = kwargs.get("access_token") + if access_token is None: + raise ValueError('access_token parameter is not defined.') + access_token_secret = kwargs.get("access_token_secret") + if access_token_secret is None: + raise ValueError('access_token_secret parameter is not defined.') + + # Output timestamp should always be True by default otherwise Twitter will reject duplicate statuses. + self.output_timestamp = kwargs.get("output_timestamp", True) + # Create a new twitter api object try: auth = tweepy.OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_token, access_token_secret) self.api = tweepy.API(auth) - self.output_timestamp = output_timestamp except tweepy.TweepError as e: - self.logger.warning('Error connecting to Twitter. Err: {} - Message: {}'.format(e.args[0][0]['code'], e.args[0][0]['message'])) + msg = 'Error authenicating with Twitter. Please check your Twitter configuration.' + self.logger.warning(msg) + raise ValueError(msg) def send_message(self, msg, timestamp): try: diff --git a/scripts/run_social_messaging.py b/scripts/run_social_messaging.py index c76743f92..c669fe44b 100755 --- a/scripts/run_social_messaging.py +++ b/scripts/run_social_messaging.py @@ -113,51 +113,33 @@ def arg_error(msg): if args.from_config: config = load_config(config_files=['pocs']) - - if 'social_accounts' in config: - social_config = config['social_accounts'] - # See which social sink we can create based on config + social_config = config.get('social_accounts') + if social_config: + # Check which social sinks we can create based on config # Twitter sink - if social_config is not None and 'twitter' in social_config: - twitter_config = social_config['twitter'] - if twitter_config is not None and \ - 'consumer_key' in twitter_config and \ - 'consumer_secret' in twitter_config and \ - 'access_token' in twitter_config and \ - 'access_token_secret' in twitter_config: - # Output timestamp should always be True in Twitter - # otherwise Twitter will reject duplicate statuses - if 'output_timestamp' in twitter_config: - output_timestamp = twitter_config['output_timestamp'] - else: - output_timestamp = True - - social_twitter = SocialTwitter(twitter_config['consumer_key'], - twitter_config['consumer_secret'], - twitter_config['access_token'], - twitter_config['access_token_secret'], - output_timestamp) + twitter_config = social_config.get('twitter') + if twitter_config: + try: + social_twitter = SocialTwitter(**twitter_config) + except ValueError as e: + print('Twitter sink could not be initialised. Please check your config. Error: {}'.format(str(e))) # Slack sink - if social_config is not None and 'slack' in social_config: - slack_config = social_config['slack'] - if slack_config is not None and \ - 'webhook_url' in slack_config: - - if 'output_timestamp' in slack_config: - output_timestamp = slack_config['output_timestamp'] - else: - output_timestamp = False - - social_slack = SocialSlack(slack_config['webhook_url'], - output_timestamp) + slack_config = social_config.get('slack') + if slack_config: + try: + social_slack = SocialSlack(**slack_config) + except ValueError as e: + print('Slack sink could not be initialised. Please check your config. Error: {}'.format(str(e))) else: - arg_error('social_accounts setting not defined in config.') + print('No social accounts defined in config, exiting.') + sys.exit(0) if not social_twitter and not social_slack: - arg_error('No social messaging sinks could be initialised. Please check your config settings.') + print('No social messaging sinks defined, exiting.') + sys.exit(0) # Messaging port to subscribe on msg_port = config['messaging']['msg_port'] + 1 From 76ae63c1978292220021192ce2b0a78ecf9a37d1 Mon Sep 17 00:00:00 2001 From: Luca Date: Sat, 23 Jun 2018 10:26:54 +1000 Subject: [PATCH 5/6] Added unit tests for Social Messaging, improved parameter defaults in sink classes --- pocs/tests/test_social_messaging.py | 103 ++++++++++++++++++++++++++++ pocs/utils/social_slack.py | 12 ++-- pocs/utils/social_twitter.py | 16 ++--- 3 files changed, 117 insertions(+), 14 deletions(-) create mode 100644 pocs/tests/test_social_messaging.py diff --git a/pocs/tests/test_social_messaging.py b/pocs/tests/test_social_messaging.py new file mode 100644 index 000000000..0f1f35740 --- /dev/null +++ b/pocs/tests/test_social_messaging.py @@ -0,0 +1,103 @@ +import pytest +import tweepy +import requests +import unittest.mock + +from pocs.utils.social_twitter import SocialTwitter +from pocs.utils.social_slack import SocialSlack + + +@pytest.fixture(scope='module') +def twitter_config(): + twitter_config = {'consumer_key': 'mock_consumer_key', 'consumer_secret': 'mock_consumer_secret', 'access_token': 'mock_access_token', 'access_token_secret': 'access_token_secret'} + return twitter_config + + +@pytest.fixture(scope='module') +def slack_config(): + slack_config = {'webhook_url': 'mock_webhook_url', 'output_timestamp': False} + return slack_config + + +# Twitter sink tests +def test_no_consumer_key(twitter_config): + with unittest.mock.patch.dict(twitter_config): + del twitter_config['consumer_key'] + with pytest.raises(ValueError) as ve: + social_twitter = SocialTwitter(**twitter_config) + assert 'consumer_key parameter is not defined.' == str(ve.value) + + +def test_no_consumer_secret(twitter_config): + with unittest.mock.patch.dict(twitter_config): + del twitter_config['consumer_secret'] + with pytest.raises(ValueError) as ve: + social_twitter = SocialTwitter(**twitter_config) + assert 'consumer_secret parameter is not defined.' == str(ve.value) + + +def test_no_access_token(twitter_config): + with unittest.mock.patch.dict(twitter_config): + del twitter_config['access_token'] + with pytest.raises(ValueError) as ve: + social_twitter = SocialTwitter(**twitter_config) + assert 'access_token parameter is not defined.' == str(ve.value) + + +def test_no_access_token_secret(twitter_config): + with unittest.mock.patch.dict(twitter_config): + del twitter_config['access_token_secret'] + with pytest.raises(ValueError) as ve: + social_twitter = SocialTwitter(**twitter_config) + assert 'access_token_secret parameter is not defined.' == str(ve.value) + + +def test_send_message_twitter(twitter_config): + with unittest.mock.patch.object(tweepy.API, 'update_status') as mock_update_status: + social_twitter = SocialTwitter(**twitter_config) + mock_message = "mock_message" + mock_timestamp = "mock_timestamp" + social_twitter.send_message(mock_message, mock_timestamp) + + mock_update_status.assert_called_once_with('{} - {}'.format(mock_timestamp, mock_message)) + + +def test_send_message_twitter_no_timestamp(twitter_config): + with unittest.mock.patch.dict(twitter_config, {'output_timestamp': False}): + with unittest.mock.patch.object(tweepy.API, 'update_status') as mock_update_status: + social_twitter = SocialTwitter(**twitter_config) + mock_message = "mock_message" + mock_timestamp = "mock_timestamp" + social_twitter.send_message(mock_message, mock_timestamp) + + mock_update_status.assert_called_once_with(mock_message) + + +# Slack sink tests +def test_no_webhook_url(slack_config): + with unittest.mock.patch.dict(slack_config): + del slack_config['webhook_url'] + with pytest.raises(ValueError) as ve: + slack_config = SocialSlack(**slack_config) + assert 'webhook_url parameter is not defined.' == str(ve.value) + + +def test_send_message_slack(slack_config): + with unittest.mock.patch.object(requests, 'post') as mock_post: + social_slack = SocialSlack(**slack_config) + mock_message = "mock_message" + mock_timestamp = "mock_timestamp" + social_slack.send_message(mock_message, mock_timestamp) + + mock_post.assert_called_once_with(slack_config['webhook_url'], json={'text': mock_message}) + + +def test_send_message_slack_timestamp(slack_config): + with unittest.mock.patch.dict(slack_config, {'output_timestamp': True}): + with unittest.mock.patch.object(requests, 'post') as mock_post: + social_slack = SocialSlack(**slack_config) + mock_message = "mock_message" + mock_timestamp = "mock_timestamp" + social_slack.send_message(mock_message, mock_timestamp) + + mock_post.assert_called_once_with(slack_config['webhook_url'], json={'text': '{} - {}'.format(mock_timestamp, mock_message)}) diff --git a/pocs/utils/social_slack.py b/pocs/utils/social_slack.py index 0cf6227d3..d03fe9ab0 100644 --- a/pocs/utils/social_slack.py +++ b/pocs/utils/social_slack.py @@ -10,11 +10,11 @@ class SocialSlack(object): logger = get_root_logger() def __init__(self, **kwargs): - self.web_hook = kwargs.get("webhook_url") - if self.web_hook: - self.output_timestamp = kwargs.get("output_timestamp", False) - else: + self.web_hook = kwargs.get('webhook_url', '') + if self.web_hook == '': raise ValueError('webhook_url parameter is not defined.') + else: + self.output_timestamp = kwargs.get('output_timestamp', False) def send_message(self, msg, timestamp): try: @@ -23,6 +23,6 @@ def send_message(self, msg, timestamp): else: post_msg = msg - response = requests.post(self.web_hook, json={"text": post_msg}) + response = requests.post(self.web_hook, json={'text': post_msg}) except Exception as e: - self.logger.debug("Error posting to slack: {}".format(e)) + self.logger.debug('Error posting to slack: {}'.format(e)) diff --git a/pocs/utils/social_twitter.py b/pocs/utils/social_twitter.py index a027472af..f77c68684 100644 --- a/pocs/utils/social_twitter.py +++ b/pocs/utils/social_twitter.py @@ -10,17 +10,17 @@ class SocialTwitter(object): logger = get_root_logger() def __init__(self, **kwargs): - consumer_key = kwargs.get("consumer_key") - if consumer_key is None: + consumer_key = kwargs.get('consumer_key', '') + if consumer_key == '': raise ValueError('consumer_key parameter is not defined.') - consumer_secret = kwargs.get("consumer_secret") - if consumer_secret is None: + consumer_secret = kwargs.get('consumer_secret', '') + if consumer_secret == '': raise ValueError('consumer_secret parameter is not defined.') - access_token = kwargs.get("access_token") - if access_token is None: + access_token = kwargs.get('access_token', '') + if access_token == '': raise ValueError('access_token parameter is not defined.') - access_token_secret = kwargs.get("access_token_secret") - if access_token_secret is None: + access_token_secret = kwargs.get('access_token_secret', '') + if access_token_secret == '': raise ValueError('access_token_secret parameter is not defined.') # Output timestamp should always be True by default otherwise Twitter will reject duplicate statuses. From 0499f64656d7888d2ab2616257c83c16239b24b6 Mon Sep 17 00:00:00 2001 From: "jeremylan@hotmail.com" Date: Fri, 29 Jun 2018 14:31:37 +1000 Subject: [PATCH 6/6] Small changes from review --- pocs/tests/test_social_messaging.py | 55 +++++++++++++---------------- pocs/utils/social_slack.py | 2 +- pocs/utils/social_twitter.py | 2 +- requirements.txt | 1 + 4 files changed, 27 insertions(+), 33 deletions(-) diff --git a/pocs/tests/test_social_messaging.py b/pocs/tests/test_social_messaging.py index 0f1f35740..b40cef1fa 100644 --- a/pocs/tests/test_social_messaging.py +++ b/pocs/tests/test_social_messaging.py @@ -21,34 +21,30 @@ def slack_config(): # Twitter sink tests def test_no_consumer_key(twitter_config): - with unittest.mock.patch.dict(twitter_config): + with unittest.mock.patch.dict(twitter_config), pytest.raises(ValueError) as ve: del twitter_config['consumer_key'] - with pytest.raises(ValueError) as ve: - social_twitter = SocialTwitter(**twitter_config) + social_twitter = SocialTwitter(**twitter_config) assert 'consumer_key parameter is not defined.' == str(ve.value) def test_no_consumer_secret(twitter_config): - with unittest.mock.patch.dict(twitter_config): + with unittest.mock.patch.dict(twitter_config), pytest.raises(ValueError) as ve: del twitter_config['consumer_secret'] - with pytest.raises(ValueError) as ve: - social_twitter = SocialTwitter(**twitter_config) + social_twitter = SocialTwitter(**twitter_config) assert 'consumer_secret parameter is not defined.' == str(ve.value) def test_no_access_token(twitter_config): - with unittest.mock.patch.dict(twitter_config): + with unittest.mock.patch.dict(twitter_config), pytest.raises(ValueError) as ve: del twitter_config['access_token'] - with pytest.raises(ValueError) as ve: - social_twitter = SocialTwitter(**twitter_config) + social_twitter = SocialTwitter(**twitter_config) assert 'access_token parameter is not defined.' == str(ve.value) def test_no_access_token_secret(twitter_config): - with unittest.mock.patch.dict(twitter_config): + with unittest.mock.patch.dict(twitter_config), pytest.raises(ValueError) as ve: del twitter_config['access_token_secret'] - with pytest.raises(ValueError) as ve: - social_twitter = SocialTwitter(**twitter_config) + social_twitter = SocialTwitter(**twitter_config) assert 'access_token_secret parameter is not defined.' == str(ve.value) @@ -59,26 +55,24 @@ def test_send_message_twitter(twitter_config): mock_timestamp = "mock_timestamp" social_twitter.send_message(mock_message, mock_timestamp) - mock_update_status.assert_called_once_with('{} - {}'.format(mock_timestamp, mock_message)) + mock_update_status.assert_called_once_with('{} - {}'.format(mock_message, mock_timestamp)) def test_send_message_twitter_no_timestamp(twitter_config): - with unittest.mock.patch.dict(twitter_config, {'output_timestamp': False}): - with unittest.mock.patch.object(tweepy.API, 'update_status') as mock_update_status: - social_twitter = SocialTwitter(**twitter_config) - mock_message = "mock_message" - mock_timestamp = "mock_timestamp" - social_twitter.send_message(mock_message, mock_timestamp) + with unittest.mock.patch.dict(twitter_config, {'output_timestamp': False}), unittest.mock.patch.object(tweepy.API, 'update_status') as mock_update_status: + social_twitter = SocialTwitter(**twitter_config) + mock_message = "mock_message" + mock_timestamp = "mock_timestamp" + social_twitter.send_message(mock_message, mock_timestamp) - mock_update_status.assert_called_once_with(mock_message) + mock_update_status.assert_called_once_with(mock_message) # Slack sink tests def test_no_webhook_url(slack_config): - with unittest.mock.patch.dict(slack_config): + with unittest.mock.patch.dict(slack_config), pytest.raises(ValueError) as ve: del slack_config['webhook_url'] - with pytest.raises(ValueError) as ve: - slack_config = SocialSlack(**slack_config) + slack_config = SocialSlack(**slack_config) assert 'webhook_url parameter is not defined.' == str(ve.value) @@ -93,11 +87,10 @@ def test_send_message_slack(slack_config): def test_send_message_slack_timestamp(slack_config): - with unittest.mock.patch.dict(slack_config, {'output_timestamp': True}): - with unittest.mock.patch.object(requests, 'post') as mock_post: - social_slack = SocialSlack(**slack_config) - mock_message = "mock_message" - mock_timestamp = "mock_timestamp" - social_slack.send_message(mock_message, mock_timestamp) - - mock_post.assert_called_once_with(slack_config['webhook_url'], json={'text': '{} - {}'.format(mock_timestamp, mock_message)}) + with unittest.mock.patch.dict(slack_config, {'output_timestamp': True}), unittest.mock.patch.object(requests, 'post') as mock_post: + social_slack = SocialSlack(**slack_config) + mock_message = "mock_message" + mock_timestamp = "mock_timestamp" + social_slack.send_message(mock_message, mock_timestamp) + + mock_post.assert_called_once_with(slack_config['webhook_url'], json={'text': '{} - {}'.format(mock_message, mock_timestamp)}) diff --git a/pocs/utils/social_slack.py b/pocs/utils/social_slack.py index d03fe9ab0..8b5cc466a 100644 --- a/pocs/utils/social_slack.py +++ b/pocs/utils/social_slack.py @@ -19,7 +19,7 @@ def __init__(self, **kwargs): def send_message(self, msg, timestamp): try: if self.output_timestamp: - post_msg = '{} - {}'.format(timestamp, msg) + post_msg = '{} - {}'.format(msg, timestamp) else: post_msg = msg diff --git a/pocs/utils/social_twitter.py b/pocs/utils/social_twitter.py index f77c68684..4e830564f 100644 --- a/pocs/utils/social_twitter.py +++ b/pocs/utils/social_twitter.py @@ -40,7 +40,7 @@ def __init__(self, **kwargs): def send_message(self, msg, timestamp): try: if self.output_timestamp: - retStatus = self.api.update_status('{} - {}'.format(timestamp, msg)) + retStatus = self.api.update_status('{} - {}'.format(msg, timestamp)) else: retStatus = self.api.update_status(msg) except tweepy.TweepError as e: diff --git a/requirements.txt b/requirements.txt index 97f9fc54d..186ea552d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,6 +18,7 @@ python_dateutil >= 2.5.3 PyYAML >= 3.11 pyzmq >= 15.3.0 readline +requests scikit_image >= 0.12.3 scipy >= 0.17.1 transitions >= 0.4.0