Skip to content

Commit

Permalink
Merge pull request #158 from /issues/85
Browse files Browse the repository at this point in the history 
Fix #85 and #154
  • Loading branch information
@jantman
jantman committed Mar 12, 2016
2 parents 0d64881 + 19a2526 commit 89db702
Show file tree
Hide file tree
Showing 8 changed files with 312 additions and 501 deletions.
10 changes: 9 additions & 1 deletion CHANGES.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,14 @@ Changelog
Pre-release (develop branch)
----------------------------

* `#150 <https://github.com/jantman/awslimitchecker/issues/150>`_ add CHANGES.rst to Sphinx docs
* `#85 <https://github.com/jantman/awslimitchecker/issues/85>`_ and `#154 <https://github.com/jantman/awslimitchecker/issues/154>`_

* add support for RDS 'DB Clusters' and 'DB Cluster Parameter Groups' limits
* use API to retrieve RDS limits
* switch RDS from calculating usage to using the DescribeAccountAttributes usage information, for all limits other than those which are per-resource and need resource IDs (Max auths per security group, Read replicas per master, Subnets per Subnet Group)
* awslimitchecker now **requires an additional IAM permission**, ``rds:DescribeAccountAttributes``

0.3.2 (2016-03-11)
------------------

Expand Down Expand Up @@ -55,7 +63,7 @@ Pre-release (develop branch)
* Attempt at fixing `#47 <https://github.com/jantman/awslimitchecker/issues/47>`_ where versioncheck acceptance tests fail under TravisCI, when testing master after a tagged release (when there's a tag for the current commit)
* Fix `#73 <https://github.com/jantman/awslimitchecker/issues/73>`_ versioncheck.py reports incorrect information when package is installed in a virtualenv inside a git repository
* Fix `#87 <https://github.com/jantman/awslimitchecker/issues/87>`_ run coverage in all unit test Tox environments, not a dedicated env
* Fix `#75 <https://github.com/jantman/awslimitchecker/issues/75>`_ re-enable py26 Travis builds now that `pytest-dev/pytest#1035 <https://github.com/pytest-dev/pytest/issues/1035`_ is fixed (pytest >= 2.8.3)
* Fix `#75 <https://github.com/jantman/awslimitchecker/issues/75>`_ re-enable py26 Travis builds now that `pytest-dev/pytest#1035 <https://github.com/pytest-dev/pytest/issues/1035>`_ is fixed (pytest >= 2.8.3)
* Fix `#13 <https://github.com/jantman/awslimitchecker/issues/13>`_ re-enable Sphinx documentation linkcheck
* Fix `#40 <https://github.com/jantman/awslimitchecker/issues/40>`_ add support for pagination of API responses (to get all results) and handle pagination for all current services
* Fix `#88 <https://github.com/jantman/awslimitchecker/issues/88>`_ add support for API-derived limits. This is a change to the public API for ``awslimitchecker.limit.AwsLimit`` and the CLI output.
Expand Down
163 changes: 61 additions & 102 deletions awslimitchecker/services/rds.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,25 @@ class _RDSService(_AwsService):
service_name = 'RDS'
api_name = 'rds'

# Mapping of RDS DescribeAccountAttributes action AccountQuotaName string
# to our Limit name
API_NAME_TO_LIMIT = {
'DBInstances': 'DB instances',
'ReservedDBInstances': 'Reserved Instances',
'AllocatedStorage': 'Storage quota (GB)',
'DBSecurityGroups': 'DB security groups',
'AuthorizationsPerDBSecurityGroup': 'Max auths per security group',
'DBParameterGroups': 'DB parameter groups',
'ManualSnapshots': 'DB snapshots per user',
'EventSubscriptions': 'Event Subscriptions',
'DBSubnetGroups': 'Subnet Groups',
'OptionGroups': 'Option Groups',
'SubnetsPerDBSubnetGroup': 'Subnets per Subnet Group',
'ReadReplicasPerMaster': 'Read replicas per master',
'DBClusters': 'DB Clusters',
'DBClusterParameterGroups': 'DB Cluster Parameter Groups',
}

def find_usage(self):
"""
Determine the current usage for each limit of this service,
Expand All @@ -62,149 +81,50 @@ def find_usage(self):
for lim in self.limits.values():
lim._reset_usage()
self._find_usage_instances()
self._find_usage_snapshots()
self._find_usage_param_groups()
self._find_usage_subnet_groups()
self._find_usage_option_groups()
self._find_usage_event_subscriptions()
self._find_usage_security_groups()
self._find_usage_reserved_instances()
# RDS API also provides usage information
self._update_limits_from_api()
self._have_usage = True
logger.debug("Done checking usage.")

def _find_usage_instances(self):
"""find usage for DB Instances and related limits"""
count = 0
allocated_gb = 0

paginator = self.conn.get_paginator('describe_db_instances')
for page in paginator.paginate():
for instance in page['DBInstances']:
count += 1
allocated_gb += instance['AllocatedStorage']
self.limits['Read replicas per master']._add_current_usage(
len(instance['ReadReplicaDBInstanceIdentifiers']),
aws_type='AWS::RDS::DBInstance',
resource_id=instance['DBInstanceIdentifier']
)

self.limits['DB instances']._add_current_usage(
count,
aws_type='AWS::RDS::DBInstance'
)

self.limits['Storage quota (GB)']._add_current_usage(
allocated_gb,
aws_type='AWS::RDS::DBInstance'
)

def _find_usage_reserved_instances(self):
"""find usage for reserved instances"""
count = 0

paginator = self.conn.get_paginator('describe_reserved_db_instances')
for page in paginator.paginate():
for inst in page['ReservedDBInstances']:
count += 1
self.limits['Reserved Instances']._add_current_usage(
count,
aws_type='AWS::RDS::DBInstance'
)

def _find_usage_snapshots(self):
"""find usage for (manual) DB snapshots"""
count = 0

paginator = self.conn.get_paginator('describe_db_snapshots')
for page in paginator.paginate():
for snap in page['DBSnapshots']:
if snap['SnapshotType'] == 'manual':
count += 1
self.limits['DB snapshots per user']._add_current_usage(
count,
aws_type='AWS::RDS::DBSnapshot'
)

def _find_usage_param_groups(self):
"""find usage for parameter groups"""
count = 0

paginator = self.conn.get_paginator('describe_db_parameter_groups')
for page in paginator.paginate():
for group in page['DBParameterGroups']:
count += 1
self.limits['DB parameter groups']._add_current_usage(
count,
aws_type='AWS::RDS::DBParameterGroup'
)

def _find_usage_subnet_groups(self):
"""find usage for subnet groups"""
count = 0

paginator = self.conn.get_paginator('describe_db_subnet_groups')
for page in paginator.paginate():
for group in page['DBSubnetGroups']:
count += 1
self.limits['Subnets per Subnet Group']._add_current_usage(
len(group['Subnets']),
aws_type='AWS::RDS::DBSubnetGroup',
resource_id=group["DBSubnetGroupName"],
)
self.limits['Subnet Groups']._add_current_usage(
count,
aws_type='AWS::RDS::DBSubnetGroup',
)

def _find_usage_option_groups(self):
"""find usage for option groups"""
count = 0

paginator = self.conn.get_paginator('describe_option_groups')
for page in paginator.paginate():
for group in page['OptionGroupsList']:
count += 1
self.limits['Option Groups']._add_current_usage(
count,
aws_type='AWS::RDS::DBOptionGroup',
)

def _find_usage_event_subscriptions(self):
"""find usage for event subscriptions"""
count = 0

paginator = self.conn.get_paginator('describe_event_subscriptions')
for page in paginator.paginate():
for group in page['EventSubscriptionsList']:
count += 1
self.limits['Event Subscriptions']._add_current_usage(
count,
aws_type='AWS::RDS::EventSubscription',
)

def _find_usage_security_groups(self):
"""find usage for security groups"""
vpc_count = 0
classic_count = 0

paginator = self.conn.get_paginator('describe_db_security_groups')
for page in paginator.paginate():
for group in page['DBSecurityGroups']:
if 'VpcId' not in group or group['VpcId'] is None:
classic_count += 1
else:
if 'VpcId' in group and group['VpcId'] is not None:
vpc_count += 1
self.limits['Max auths per security group']._add_current_usage(
len(group["EC2SecurityGroups"]) + len(group["IPRanges"]),
aws_type='AWS::RDS::DBSecurityGroup',
resource_id=group['DBSecurityGroupName']
)

self.limits['DB security groups']._add_current_usage(
classic_count,
aws_type='AWS::RDS::DBSecurityGroup',
)

self.limits['VPC Security Groups']._add_current_usage(
vpc_count,
aws_type='AWS::RDS::DBSecurityGroup',
Expand Down Expand Up @@ -327,9 +247,47 @@ def get_limits(self):
limit_type='AWS::RDS::DBSecurityGroup',
limit_subtype='AWS::RDS::DBSecurityGroupIngress',
)
limits['DB Clusters'] = AwsLimit(
'DB Clusters',
self,
40,
self.warning_threshold,
self.critical_threshold,
limit_type='AWS::RDS::DBCluster',
)
limits['DB Cluster Parameter Groups'] = AwsLimit(
'DB Cluster Parameter Groups',
self,
50,
self.warning_threshold,
self.critical_threshold,
limit_type='AWS::RDS::DBClusterParameterGroup',
)
self.limits = limits
return limits

def _update_limits_from_api(self):
"""
Query RDS's DescribeAccountAttributes API action, and update limits
with the quotas returned. Updates ``self.limits``.
We ignore the usage information from the API,
"""
self.connect()
logger.info("Querying RDS DescribeAccountAttributes for limits")
lims = self.conn.describe_account_attributes()['AccountQuotas']
for lim in lims:
if lim['AccountQuotaName'] not in self.API_NAME_TO_LIMIT:
logger.info('RDS DescribeAccountAttributes returned unknown'
'limit: %s (max: %s; used: %s)',
lim['AccountQuotaName'], lim['Max'], lim['Used'])
continue
lname = self.API_NAME_TO_LIMIT[lim['AccountQuotaName']]
self.limits[lname]._set_api_limit(lim['Max'])
if len(self.limits[lname].get_current_usage()) < 1:
self.limits[lname]._add_current_usage(lim['Used'])
logger.debug('Done setting limits from API.')

def required_iam_permissions(self):
"""
Return a list of IAM Actions required for this Service to function
Expand All @@ -340,6 +298,7 @@ def required_iam_permissions(self):
:rtype: list
"""
return [
"rds:DescribeAccountAttributes",
"rds:DescribeDBInstances",
"rds:DescribeDBParameterGroups",
"rds:DescribeDBSecurityGroups",
Expand Down
Loading

0 comments on commit 89db702

Please sign in to comment.