From 5b74ae01feab52d5586c6f0d79e45295c421050c Mon Sep 17 00:00:00 2001 From: Alina Buzachis Date: Fri, 21 Jan 2022 11:29:48 +0100 Subject: [PATCH] Move some shared logic in module utils Signed-off-by: Alina Buzachis --- plugins/module_utils/sns.py | 125 +++++++++++++++++++++++++++++++++++ plugins/modules/sns.py | 19 +----- plugins/modules/sns_topic.py | 114 +++++--------------------------- 3 files changed, 143 insertions(+), 115 deletions(-) create mode 100644 plugins/module_utils/sns.py diff --git a/plugins/module_utils/sns.py b/plugins/module_utils/sns.py new file mode 100644 index 00000000000..9530fd133df --- /dev/null +++ b/plugins/module_utils/sns.py @@ -0,0 +1,125 @@ +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import re +import copy + +try: + import botocore +except ImportError: + pass # handled by AnsibleAWSModule + +from ansible_collections.amazon.aws.plugins.module_utils.core import is_boto3_error_code +from ansible_collections.amazon.aws.plugins.module_utils.ec2 import AWSRetry +from ansible_collections.amazon.aws.plugins.module_utils.ec2 import camel_dict_to_snake_dict + + +@AWSRetry.jittered_backoff() +def _list_topics_with_backoff(client): + paginator = client.get_paginator('list_topics') + return paginator.paginate().build_full_result()['Topics'] + + +@AWSRetry.jittered_backoff(catch_extra_error_codes=['NotFound']) +def _list_topic_subscriptions_with_backoff(client, topic_arn): + paginator = client.get_paginator('list_subscriptions_by_topic') + return paginator.paginate(TopicArn=topic_arn).build_full_result()['Subscriptions'] + + +@AWSRetry.jittered_backoff(catch_extra_error_codes=['NotFound']) +def _list_subscriptions_with_backoff(client): + paginator = client.get_paginator('list_subscriptions') + return paginator.paginate().build_full_result()['Subscriptions'] + + +def list_topic_subscriptions(client, module, topic_arn): + try: + return _list_topic_subscriptions_with_backoff(client, topic_arn) + except is_boto3_error_code('AuthorizationError'): + try: + # potentially AuthorizationError when listing subscriptions for third party topic + return [sub for sub in _list_subscriptions_with_backoff(client) + if sub['TopicArn'] == topic_arn] + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + module.fail_json_aws(e, msg="Couldn't get subscriptions list for topic %s" % topic_arn) + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: # pylint: disable=duplicate-except + module.fail_json_aws(e, msg="Couldn't get subscriptions list for topic %s" % topic_arn) + + +def list_topics(client, module): + try: + topics = _list_topics_with_backoff(client) + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + module.fail_json_aws(e, msg="Couldn't get topic list") + return [t['TopicArn'] for t in topics] + + +def topic_arn_lookup(client, module, name): + # topic names cannot have colons, so this captures the full topic name + all_topics = list_topics(client, module) + lookup_topic = ':%s' % name + for topic in all_topics: + if topic.endswith(lookup_topic): + return topic + + +def compare_delivery_policies(policy_a, policy_b): + _policy_a = copy.deepcopy(policy_a) + _policy_b = copy.deepcopy(policy_b) + # AWS automatically injects disableSubscriptionOverrides if you set an + # http policy + if 'http' in policy_a: + if 'disableSubscriptionOverrides' not in policy_a['http']: + _policy_a['http']['disableSubscriptionOverrides'] = False + if 'http' in policy_b: + if 'disableSubscriptionOverrides' not in policy_b['http']: + _policy_b['http']['disableSubscriptionOverrides'] = False + comparison = (_policy_a != _policy_b) + return comparison + + +def canonicalize_endpoint(protocol, endpoint): + # AWS SNS expects phone numbers in + # and canonicalizes to E.164 format + # See + if protocol == 'sms': + return re.sub('[^0-9+]*', '', endpoint) + return endpoint + + +def get_info(connection, module, topic_arn): + name = module.params.get('name') + topic_type = module.params.get('topic_type') + state = module.params.get('state') + subscriptions = module.params.get('subscriptions') + purge_subscriptions = module.params.get('purge_subscriptions') + subscriptions_existing = module.params.get('subscriptions_existing', []) + subscriptions_deleted = module.params.get('subscriptions_deleted', []) + subscriptions_added = module.params.get('subscriptions_added', []) + subscriptions_added = module.params.get('subscriptions_added', []) + topic_created = module.params.get('topic_created', False) + topic_deleted = module.params.get('topic_deleted', False) + attributes_set = module.params.get('attributes_set', []) + check_mode = module.check_mode + + info = { + 'name': name, + 'topic_type': topic_type, + 'state': state, + 'subscriptions_new': subscriptions, + 'subscriptions_existing': subscriptions_existing, + 'subscriptions_deleted': subscriptions_deleted, + 'subscriptions_added': subscriptions_added, + 'subscriptions_purge': purge_subscriptions, + 'check_mode': check_mode, + 'topic_created': topic_created, + 'topic_deleted': topic_deleted, + 'attributes_set': attributes_set, + } + if state != 'absent': + if topic_arn in list_topics(connection, module): + info.update(camel_dict_to_snake_dict(connection.get_topic_attributes(TopicArn=topic_arn)['Attributes'])) + info['delivery_policy'] = info.pop('effective_delivery_policy') + info['subscriptions'] = [camel_dict_to_snake_dict(sub) for sub in list_topic_subscriptions(connection, module, topic_arn)] + + return info \ No newline at end of file diff --git a/plugins/modules/sns.py b/plugins/modules/sns.py index a18c3279173..42dabd4a04f 100644 --- a/plugins/modules/sns.py +++ b/plugins/modules/sns.py @@ -134,22 +134,7 @@ pass # Handled by AnsibleAWSModule from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule - - -def arn_topic_lookup(module, client, short_topic): - lookup_topic = ':{0}'.format(short_topic) - - try: - paginator = client.get_paginator('list_topics') - topic_iterator = paginator.paginate() - for response in topic_iterator: - for topic in response['Topics']: - if topic['TopicArn'].endswith(lookup_topic): - return topic['TopicArn'] - except (BotoCoreError, ClientError) as e: - module.fail_json_aws(e, msg='Failed to look up topic ARN') - - return None +from ansible_collections.community.aws.plugins.module_utils.sns import topic_arn_lookup def main(): @@ -205,7 +190,7 @@ def main(): # Short names can't contain ':' so we'll assume this is the full ARN sns_kwargs['TopicArn'] = topic else: - sns_kwargs['TopicArn'] = arn_topic_lookup(module, client, topic) + sns_kwargs['TopicArn'] = topic_arn_lookup(module, client, topic) if not sns_kwargs['TopicArn']: module.fail_json(msg='Could not find topic: {0}'.format(topic)) diff --git a/plugins/modules/sns_topic.py b/plugins/modules/sns_topic.py index dd5af417bab..bf694c7e9b4 100644 --- a/plugins/modules/sns_topic.py +++ b/plugins/modules/sns_topic.py @@ -226,7 +226,13 @@ pass # handled by AnsibleAWSModule from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule, is_boto3_error_code -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import compare_policies, AWSRetry, camel_dict_to_snake_dict +from ansible_collections.amazon.aws.plugins.module_utils.ec2 import compare_policies +from ansible_collections.community.aws.plugins.module_utils.sns import list_topics +from ansible_collections.community.aws.plugins.module_utils.sns import topic_arn_lookup +from ansible_collections.community.aws.plugins.module_utils.sns import compare_delivery_policies +from ansible_collections.community.aws.plugins.module_utils.sns import list_topic_subscriptions +from ansible_collections.community.aws.plugins.module_utils.sns import canonicalize_endpoint +from ansible_collections.community.aws.plugins.module_utils.sns import get_info class SnsTopicManager(object): @@ -263,36 +269,6 @@ def __init__(self, self.topic_arn = None self.attributes_set = [] - @AWSRetry.jittered_backoff() - def _list_topics_with_backoff(self): - paginator = self.connection.get_paginator('list_topics') - return paginator.paginate().build_full_result()['Topics'] - - @AWSRetry.jittered_backoff(catch_extra_error_codes=['NotFound']) - def _list_topic_subscriptions_with_backoff(self): - paginator = self.connection.get_paginator('list_subscriptions_by_topic') - return paginator.paginate(TopicArn=self.topic_arn).build_full_result()['Subscriptions'] - - @AWSRetry.jittered_backoff(catch_extra_error_codes=['NotFound']) - def _list_subscriptions_with_backoff(self): - paginator = self.connection.get_paginator('list_subscriptions') - return paginator.paginate().build_full_result()['Subscriptions'] - - def _list_topics(self): - try: - topics = self._list_topics_with_backoff() - except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: - self.module.fail_json_aws(e, msg="Couldn't get topic list") - return [t['TopicArn'] for t in topics] - - def _topic_arn_lookup(self): - # topic names cannot have colons, so this captures the full topic name - all_topics = self._list_topics() - lookup_topic = ':%s' % self.name - for topic in all_topics: - if topic.endswith(lookup_topic): - return topic - def _create_topic(self): attributes = {'FifoTopic': 'false'} tags = [] @@ -310,20 +286,6 @@ def _create_topic(self): self.topic_arn = response['TopicArn'] return True - def _compare_delivery_policies(self, policy_a, policy_b): - _policy_a = copy.deepcopy(policy_a) - _policy_b = copy.deepcopy(policy_b) - # AWS automatically injects disableSubscriptionOverrides if you set an - # http policy - if 'http' in policy_a: - if 'disableSubscriptionOverrides' not in policy_a['http']: - _policy_a['http']['disableSubscriptionOverrides'] = False - if 'http' in policy_b: - if 'disableSubscriptionOverrides' not in policy_b['http']: - _policy_b['http']['disableSubscriptionOverrides'] = False - comparison = (_policy_a != _policy_b) - return comparison - def _set_topic_attrs(self): changed = False try: @@ -352,7 +314,7 @@ def _set_topic_attrs(self): self.module.fail_json_aws(e, msg="Couldn't set topic policy") if self.delivery_policy and ('DeliveryPolicy' not in topic_attributes or - self._compare_delivery_policies(self.delivery_policy, json.loads(topic_attributes['DeliveryPolicy']))): + compare_delivery_policies(self.delivery_policy, json.loads(topic_attributes['DeliveryPolicy']))): changed = True self.attributes_set.append('delivery_policy') if not self.check_mode: @@ -363,22 +325,14 @@ def _set_topic_attrs(self): self.module.fail_json_aws(e, msg="Couldn't set topic delivery policy") return changed - def _canonicalize_endpoint(self, protocol, endpoint): - # AWS SNS expects phone numbers in - # and canonicalizes to E.164 format - # See - if protocol == 'sms': - return re.sub('[^0-9+]*', '', endpoint) - return endpoint - def _set_topic_subs(self): changed = False subscriptions_existing_list = set() desired_subscriptions = [(sub['protocol'], - self._canonicalize_endpoint(sub['protocol'], sub['endpoint'])) for sub in + canonicalize_endpoint(sub['protocol'], sub['endpoint'])) for sub in self.subscriptions] - for sub in self._list_topic_subscriptions(): + for sub in list_topic_subscriptions(self.connection, self.module, self.topic_arn): sub_key = (sub['Protocol'], sub['Endpoint']) subscriptions_existing_list.add(sub_key) if (self.purge_subscriptions and sub_key not in desired_subscriptions and @@ -401,23 +355,10 @@ def _set_topic_subs(self): self.module.fail_json_aws(e, msg="Couldn't subscribe to topic %s" % self.topic_arn) return changed - def _list_topic_subscriptions(self): - try: - return self._list_topic_subscriptions_with_backoff() - except is_boto3_error_code('AuthorizationError'): - try: - # potentially AuthorizationError when listing subscriptions for third party topic - return [sub for sub in self._list_subscriptions_with_backoff() - if sub['TopicArn'] == self.topic_arn] - except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: - self.module.fail_json_aws(e, msg="Couldn't get subscriptions list for topic %s" % self.topic_arn) - except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: # pylint: disable=duplicate-except - self.module.fail_json_aws(e, msg="Couldn't get subscriptions list for topic %s" % self.topic_arn) - def _delete_subscriptions(self): # NOTE: subscriptions in 'PendingConfirmation' timeout in 3 days # https://forums.aws.amazon.com/thread.jspa?threadID=85993 - subscriptions = self._list_topic_subscriptions() + subscriptions = list_topic_subscriptions(self.connection, self.module, self.topic_arn) if not subscriptions: return False for sub in subscriptions: @@ -447,10 +388,10 @@ def ensure_ok(self): if self._name_is_arn(): self.topic_arn = self.name else: - self.topic_arn = self._topic_arn_lookup() + self.topic_arn = topic_arn_lookup(self.connection, self.module, self.name) if not self.topic_arn: changed = self._create_topic() - if self.topic_arn in self._list_topics(): + if self.topic_arn in list_topics(self.connection, self.module): changed |= self._set_topic_attrs() elif self.display_name or self.policy or self.delivery_policy: self.module.fail_json(msg="Cannot set display name, policy or delivery policy for SNS topics not owned by this account") @@ -462,37 +403,14 @@ def ensure_gone(self): if self._name_is_arn(): self.topic_arn = self.name else: - self.topic_arn = self._topic_arn_lookup() + self.topic_arn = topic_arn_lookup(self.connection, self.module, self.name) if self.topic_arn: - if self.topic_arn not in self._list_topics(): + if self.topic_arn not in list_topics(self.connection, self.module): self.module.fail_json(msg="Cannot use state=absent with third party ARN. Use subscribers=[] to unsubscribe") changed = self._delete_subscriptions() changed |= self._delete_topic() return changed - def get_info(self): - info = { - 'name': self.name, - 'topic_type': self.topic_type, - 'state': self.state, - 'subscriptions_new': self.subscriptions, - 'subscriptions_existing': self.subscriptions_existing, - 'subscriptions_deleted': self.subscriptions_deleted, - 'subscriptions_added': self.subscriptions_added, - 'subscriptions_purge': self.purge_subscriptions, - 'check_mode': self.check_mode, - 'topic_created': self.topic_created, - 'topic_deleted': self.topic_deleted, - 'attributes_set': self.attributes_set, - } - if self.state != 'absent': - if self.topic_arn in self._list_topics(): - info.update(camel_dict_to_snake_dict(self.connection.get_topic_attributes(TopicArn=self.topic_arn)['Attributes'])) - info['delivery_policy'] = info.pop('effective_delivery_policy') - info['subscriptions'] = [camel_dict_to_snake_dict(sub) for sub in self._list_topic_subscriptions()] - - return info - def main(): argument_spec = dict( @@ -538,7 +456,7 @@ def main(): sns_facts = dict(changed=changed, sns_arn=sns_topic.topic_arn, - sns_topic=sns_topic.get_info()) + sns_topic=get_info(sns_topic.connection, module, sns_topic.topic_arn)) module.exit_json(**sns_facts)