Release 1.5.4

Release 1.5.4

.travis.yml

@@ -24,7 +24,10 @@ install:
 sudo: false
 
 script:
-  - if [[ $TRAVIS_PYTHON_VERSION != 2.6 ]];
-    then python cli/tests/cfncluster-unittest.py;
+  - if [[ $TRAVIS_PYTHON_VERSION != 2.6 ]]; then
+    pip install 'boto3==1.7.84' --force-reinstall;
+    pip install moto
+    python cli/tests/cfncluster-unittest.py;
+    pip install boto3 --upgrade;
     fi
   - sh tests/test.sh

CHANGELOG.rst

@@ -2,6 +2,13 @@
 CHANGELOG
 =========
 
+1.5.4
+=====
+* Add option to disable ganglia `extra_json = { "cfncluster" : { "ganglia_enabled" : "no" } }`
+* Fix `cfncluster update` bug
+* Set SGE Accounting summary to be true, this reports a single accounting record for a mpi job
+* Upgrade cfncluster-node to Boto3
+
 1.5.3
 =====
 * Add support for GovCloud, us-gov-west-1 region

README.rst

@@ -28,6 +28,11 @@ Please open a GitHub issue for any feedback or issues:
 https://github.com/awslabs/cfncluster.  There is also an active AWS
 HPC forum which may be helpful:https://forums.aws.amazon.com/forum.jspa?forumID=192.
 
+CfnCluster 1.6 IAM Change
+=========================
+Between CfnCluster 1.5.3 and 1.6.0 we made a change to the CfnClusterInstancePolicy that adds “s3:GetObject” permissions on objects in <REGION>-cfncluster bucket.
+If you’re using a custom policy (e.g. you specify "ec2_iam_role" in your config) be sure it includes this new permission. See https://cfncluster.readthedocs.io/en/latest/iam.html
+
 CfnCluster 1.5 IAM Change
 =========================
 Between CfnCluster 1.4.2 and 1.5.0 we made a change to the CfnClusterInstancePolicy that adds “ec2:DescribeVolumes” permissions. If you’re using a custom policy (e.g. you specify "ec2_iam_role" in your config) be sure it includes this new permission. See https://cfncluster.readthedocs.io/en/latest/iam.html

amis.txt

@@ -1,88 +1,88 @@
 # alinux
-ap-northeast-1: ami-05468a90639fdc483
-ap-northeast-2: ami-0b4d5a4a999581db1
-ap-northeast-3: ami-0594d5d373771f10a
-ap-south-1: ami-0ec7be770210af7c0
-ap-southeast-1: ami-02181eb4bbe285f28
-ap-southeast-2: ami-03199596cd474bb05
-ca-central-1: ami-099a5f4b65d56215e
-eu-central-1: ami-00e39ce170dbd6e25
-eu-west-1: ami-059b3db381c18305a
-eu-west-2: ami-0df738bfb4cf98f4e
-eu-west-3: ami-0b1472bf5fed029bd
-sa-east-1: ami-0a390e1998fb6b22f
-us-east-1: ami-0fd6854902340b407
-us-east-2: ami-04fc90a3f60cb3dc7
-us-gov-west-1: ami-af2cbece
-us-west-1: ami-0054acc8361046ccc
-us-west-2: ami-01382f63e2667a7d1
+ap-northeast-1: ami-00047986f877e6760
+ap-northeast-2: ami-0ca5afbc9d0b20f3d
+ap-northeast-3: ami-0b3af1c2fe35b1ae4
+ap-south-1: ami-0250a8315b099785d
+ap-southeast-1: ami-038312aa862774599
+ap-southeast-2: ami-0dce078cd2c343a5c
+ca-central-1: ami-04e27bd54e171fbad
+eu-central-1: ami-075bf1c36cf6b855f
+eu-west-1: ami-08d507a759b9ee0e4
+eu-west-2: ami-03ddfb1462e2c92fd
+eu-west-3: ami-0e72015cf2b21159f
+sa-east-1: ami-02bd755bfe8704cef
+us-east-1: ami-07f61c3e18538c70c
+us-east-2: ami-096b7925b1f9c8cbb
+us-gov-west-1: ami-cfd34dae
+us-west-1: ami-04794da830cb9dc86
+us-west-2: ami-03f86a2920360dfbe
 # centos6
-ap-northeast-1: ami-0603daa665c23cf74
-ap-northeast-2: ami-06ddec14c10c7de14
-ap-northeast-3: ami-0e718e0794e781ead
-ap-south-1: ami-043871a624a63c4b3
-ap-southeast-1: ami-07df16800a8f39379
-ap-southeast-2: ami-09b23d993fbc703d9
-ca-central-1: ami-0af06655a7341c2b5
-eu-central-1: ami-0362d5eb8ec477a30
-eu-west-1: ami-04c68c9303ee0165c
-eu-west-2: ami-0d3a080fd047d6f03
-eu-west-3: ami-0ce8fcafcd4cb66ba
-sa-east-1: ami-010873231de6443db
-us-east-1: ami-0051f4828527649d0
-us-east-2: ami-0a585da16a01304bb
-us-west-1: ami-0fcbc3afb66cfcffd
-us-west-2: ami-0430c74956ce8e41a
+ap-northeast-1: ami-00c9ad360d255b462
+ap-northeast-2: ami-08e84ffe9aead7873
+ap-northeast-3: ami-056531169587ffc23
+ap-south-1: ami-03a57d339b58b2f23
+ap-southeast-1: ami-00063e5635939c9d1
+ap-southeast-2: ami-015e730525001bf53
+ca-central-1: ami-0fea713db092ac77f
+eu-central-1: ami-0f6b91d011f44aaf6
+eu-west-1: ami-029accac823a2a844
+eu-west-2: ami-032b6b895dab544c6
+eu-west-3: ami-039be4bd46c878e2c
+sa-east-1: ami-0e507bdec09b8a091
+us-east-1: ami-091bbc83b27f92634
+us-east-2: ami-0fa89e68b0222f50b
+us-west-1: ami-06f3909be633646b9
+us-west-2: ami-087d4f29315fd19d2
 # centos7
-ap-northeast-1: ami-03e99430742e7d834
-ap-northeast-2: ami-0a3ad3da5f45318a4
-ap-northeast-3: ami-0804a0d2dcf1e675f
-ap-south-1: ami-06b1a64ab30690802
-ap-southeast-1: ami-06eec4db7a3965f04
-ap-southeast-2: ami-0ab1def1c27a54ada
-ca-central-1: ami-089b3d6d4ef8c5772
-eu-central-1: ami-0842b79de396801b5
-eu-west-1: ami-019ecc5629a990951
-eu-west-2: ami-03e914c8e1b836bcf
-eu-west-3: ami-099b3883ce5e219b3
-sa-east-1: ami-0edbfab1cbd73ac82
-us-east-1: ami-00976a3c38aaac1a2
-us-east-2: ami-0f7dbb8d94e96b0ab
-us-west-1: ami-0fae379328b9061bc
-us-west-2: ami-0df85f5bc2d980e57
+ap-northeast-1: ami-09dc3b175ed2905f8
+ap-northeast-2: ami-03f494fed9f0f9c98
+ap-northeast-3: ami-04094dd3b4013b75f
+ap-south-1: ami-0c20297a2c00b0528
+ap-southeast-1: ami-0f610508513f2012f
+ap-southeast-2: ami-09d0b89439c0c1b65
+ca-central-1: ami-0362a3ef26ab5dfce
+eu-central-1: ami-03d5519f5f4e7619f
+eu-west-1: ami-0575304add4986551
+eu-west-2: ami-08263362b21b98916
+eu-west-3: ami-0befb803ea7ce1b83
+sa-east-1: ami-0860c11596f954bbc
+us-east-1: ami-06c93be112f880acd
+us-east-2: ami-0493261a1b2adfd27
+us-west-1: ami-0af95249ebb8de7a5
+us-west-2: ami-0a259cf8220fabdcb
 # ubuntu1404
-ap-northeast-1: ami-068214c5cfb518070
-ap-northeast-2: ami-0d55e5524b8536518
-ap-northeast-3: ami-0ad3400658fbc51fa
-ap-south-1: ami-0b54a1bd3d82929e6
-ap-southeast-1: ami-0d3382bea35764ad2
-ap-southeast-2: ami-0a2a8aa487ce39e8c
-ca-central-1: ami-0bc510154aa82211d
-eu-central-1: ami-0174c3f5501fc441b
-eu-west-1: ami-01848c6ae1a9c6460
-eu-west-2: ami-0b867b5579d0f7a08
-eu-west-3: ami-0e8f93d19635c5d49
-sa-east-1: ami-048422e42509a1596
-us-east-1: ami-0ba2fcbf9fa378762
-us-east-2: ami-05aea9ea89d577194
-us-gov-west-1: ami-4aca572b
-us-west-1: ami-0089ace30f2612d13
-us-west-2: ami-06142a55749b7912b
+ap-northeast-1: ami-09dc9e9da730c0248
+ap-northeast-2: ami-06346bcc8b825a458
+ap-northeast-3: ami-0ae6e02016436404a
+ap-south-1: ami-0e4294a5b9d5de425
+ap-southeast-1: ami-01e6359021e3bac23
+ap-southeast-2: ami-0cfebd51a3470f1d4
+ca-central-1: ami-0097472502a04776d
+eu-central-1: ami-0a82ac2872821deef
+eu-west-1: ami-0d6c90f092266b787
+eu-west-2: ami-0f23dffc4db122b53
+eu-west-3: ami-02b6ca8fd14f58265
+sa-east-1: ami-04e943b561cd32659
+us-east-1: ami-0920fa86d5a85a07a
+us-east-2: ami-0d63a602f26709e70
+us-gov-west-1: ami-c2d34da3 
+us-west-1: ami-06fe4f837d8715dbb
+us-west-2: ami-01170828a21d86a72
 # ubuntu1604
-ap-northeast-1: ami-03f6879ede5f51a6f
-ap-northeast-2: ami-0036ef722f2d0a331
-ap-northeast-3: ami-0a8db47e6bc6f585b
-ap-south-1: ami-08870f6ce8227e90f
-ap-southeast-1: ami-04950204e84f8bf64
-ap-southeast-2: ami-02b4175018a85c03d
-ca-central-1: ami-04a4a63323b55af6e
-eu-central-1: ami-089e9903d9c3bee1b
-eu-west-1: ami-033b9190bab9d42aa
-eu-west-2: ami-049d2247a1c3d582c
-eu-west-3: ami-09c4322a65746f77b
-sa-east-1: ami-0740b75b4caaa3b77
-us-east-1: ami-05790a6a634720a0d
-us-east-2: ami-0a0fcd1219e40a8f5
-us-gov-west-1: ami-3dc8555c
-us-west-1: ami-056472d51cf2fed45
-us-west-2: ami-0702d59e57ada76b8
+ap-northeast-1: ami-0b0bf645cde38fe52
+ap-northeast-2: ami-0b2c6c9e988689d7c
+ap-northeast-3: ami-0dae0e941e3bce10b
+ap-south-1: ami-011d655005fe854e9
+ap-southeast-1: ami-03428422e2cb20c9f
+ap-southeast-2: ami-0bbef9e2b1170757f
+ca-central-1: ami-0916119fde83c29b7
+eu-central-1: ami-003273c4072efbe43
+eu-west-1: ami-009af0d819973e1d6
+eu-west-2: ami-029f01bbfec52a10e
+eu-west-3: ami-0ae405adcb3d1477c
+sa-east-1: ami-0b0057ecd5c465d09
+us-east-1: ami-0dfb1662fcc845c55
+us-east-2: ami-01bd1361428c384c3
+us-gov-west-1: ami-0fd6486e
+us-west-1: ami-0e144dd24634766a2
+us-west-2: ami-02087bf0ff99cc127

cli/README

@@ -1,26 +1,27 @@
 cfncluster is an example framework for launching clusters.
 
-    usage: cfncluster [-h] [--config CONFIG_FILE] [--region REGION] [--nowait]
-                      {create,update,delete,status,list,instances,version}
+    usage: cfncluster [-h]
+                      {create,update,delete,start,stop,status,list,instances,ssh,configure,version}
                       ...
 
-    cfncluster is the a tool to launch and manage cluster.
+    cfncluster is a tool to launch and manage a cluster.
 
     positional arguments:
-      {create,update,stop,delete,status,list,instances,sshmaster}
+      {create,update,delete,start,stop,status,list,instances,ssh,configure,version}
         create              creates a cluster
         update              update a running cluster
         delete              delete a cluster
+        start               start the compute fleet that has been stopped
+        stop                stop the compute fleet, but leave the master server
+                            running for debugging/development
         status              pull the current status of the cluster
         list                display a list of stacks associated with cfncluster
         instances           display a list of all instances in a cluster
-        version             display the version of cfncluster cli
+        ssh                 connect to the master server using SSH
+        configure           creating initial cfncluster configuration
+        version             display version of cfncluster
 
     optional arguments:
       -h, --help            show this help message and exit
-      --config CONFIG_FILE, -c CONFIG_FILE
-                            specify a alternative config file
-      --region REGION, -r REGION
-                            specify a specific region to connect to
-      --nowait, -nw         do not wait for stack events, after executing stack
-                            command
+
+    For command specific flags run cfncluster [command] --help

cli/cfncluster/cfncluster.py

@@ -16,6 +16,7 @@
 import logging
 import boto3
 import os
+import json
 from botocore.exceptions import ClientError
 
 from . import cfnconfig
@@ -103,7 +104,10 @@ def create(args):
                                      event.get('ResourceStatusReason')))
             logger.info('')
             outputs = cfn.describe_stacks(StackName=stack_name).get("Stacks")[0].get('Outputs', [])
+            ganglia_enabled = is_ganglia_enabled(config.parameters)
             for output in outputs:
+                if not ganglia_enabled and output.get('OutputKey').startswith('Ganglia'):
+                    continue
                 logger.info("%s: %s" % (output.get('OutputKey'), output.get('OutputValue')))
         else:
             status = cfn.describe_stacks(StackName=stack_name).get("Stacks")[0].get('StackStatus')
@@ -119,6 +123,14 @@ def create(args):
         logger.critical(e)
         sys.exit(1)
 
+def is_ganglia_enabled(parameters):
+    extra_json = dict(filter(lambda x: x[0] == 'ExtraJson', parameters))
+    try:
+        extra_json = json.loads(extra_json.get('ExtraJson')).get('cfncluster')
+        return extra_json.get('ganglia_enabled') == 'yes'
+    except:
+        pass
+    return False
 
 def update(args):
     logger.info('Updating: %s' % (args.cluster_name))
@@ -139,7 +151,7 @@ def update(args):
         desired_capacity = asg.describe_auto_scaling_groups(AutoScalingGroupNames=[asg_name])\
             .get('AutoScalingGroups')[0]\
             .get('DesiredCapacity')
-        config.parameters.append(('InitialQueueSize', desired_capacity))
+        config.parameters.append(('InitialQueueSize', str(desired_capacity)))
 
     # Get the MasterSubnetId and use it to determine AvailabilityZone
     try:
@@ -162,7 +174,7 @@ def update(args):
     try:
         logger.debug((config.template_url, config.parameters))
 
-        cfn_params = [{'ParameterKey': param[0], 'ParameterValue': str(param[1])} for param in config.parameters]
+        cfn_params = [{'ParameterKey': param[0], 'ParameterValue': param[1]} for param in config.parameters]
         cfn.update_stack(StackName=stack_name,TemplateURL=config.template_url,
                          Parameters=cfn_params, Capabilities=capabilities)
         status = cfn.describe_stacks(StackName=stack_name).get("Stacks")[0].get('StackStatus')

cli/cfncluster/cli.py

@@ -141,17 +141,17 @@ def main():
     addarg_nowait(pdelete)
     pdelete.set_defaults(func=delete)
 
-    pstart = subparsers.add_parser('start', help='start the compute-fleet that has been stopped')
+    pstart = subparsers.add_parser('start', help='start the compute fleet that has been stopped')
     pstart.add_argument("cluster_name", type=str, default=None,
-                        help='starts the compute-fleet of the provided cluster name.')
+                        help='starts the compute fleet of the provided cluster name.')
     addarg_config(pstart)
     addarg_region(pstart)
     pstart.set_defaults(func=start)
 
-    pstop = subparsers.add_parser('stop', help='stop the compute-fleet, but leave the MasterServer running for '
+    pstop = subparsers.add_parser('stop', help='stop the compute fleet, but leave the master server running for '
                                                'debugging/development')
     pstop.add_argument("cluster_name", type=str, default=None,
-                        help='stops the compute-fleet of the provided cluster name.')
+                        help='stops the compute fleet of the provided cluster name.')
     addarg_config(pstop)
     addarg_region(pstop)
     pstop.set_defaults(func=stop)
@@ -176,22 +176,23 @@ def main():
     addarg_region(pinstances)
     pinstances.set_defaults(func=instances)
 
+    pssh = subparsers.add_parser('ssh', help='connect to the master server using SSH',
+                                 description='run ssh command with username and ip address pre-filled. ' \
+                                             'Arbitrary arguments are appended to the end of the ssh commmand. ' \
+                                             'This command may be customized in the aliases section of the config file.')
+    pssh.add_argument("cluster_name", type=str, default=None,
+                        help='name of the cluster to set variables for.')
+    pssh.add_argument("--dryrun", "-d", action='store_true', dest="dryrun", default=False,
+                         help='print command and exit.')
+    pssh.set_defaults(func=command)
+
     pconfigure = subparsers.add_parser('configure', help='creating initial cfncluster configuration')
     addarg_config(pconfigure)
     pconfigure.set_defaults(func=configure)
 
     pversion = subparsers.add_parser('version', help='display version of cfncluster')
     pversion.set_defaults(func=version)
 
-    pssh = subparsers.add_parser('ssh', description='run ssh command with username and ip address pre-filled. ' \
-                                                    'Arbitrary arguments are appended to the end of the ssh commmand. ' \
-                                                    'This command may be customized in the aliases section of the config file.')
-    pssh.add_argument("cluster_name", type=str, default=None,
-                        help='name of the cluster to set variables for.')
-    pssh.add_argument("--dryrun", "-d", action='store_true', dest="dryrun", default=False,
-                         help='print command and exit.')
-    pssh.set_defaults(func=command)
-
     args, extra_args = parser.parse_known_args()
     logger.debug(args)
     if args.func.__name__ == 'command':

cli/cfncluster/config_sanity.py

@@ -40,14 +40,17 @@ def check_resource(region, aws_access_key_id, aws_secret_access_key, resource_ty
                                 aws_secret_access_key=aws_secret_access_key)
 
             arn = iam.get_role(RoleName=resource_value).get('Role').get('Arn')
-            accountid = boto3.client('sts').get_caller_identity().get('Account')
+            accountid = boto3.client('sts', region_name=region,
+                                        aws_access_key_id=aws_access_key_id,
+                                        aws_secret_access_key=aws_secret_access_key).get_caller_identity().get('Account')
 
             iam_policy = [(['ec2:DescribeVolumes', 'ec2:AttachVolume', 'ec2:DescribeInstanceAttribute', 'ec2:DescribeInstanceStatus', 'ec2:DescribeInstances'], "*"),
                         (['dynamodb:ListTables'], "*"),
                         (['sqs:SendMessage', 'sqs:ReceiveMessage', 'sqs:ChangeMessageVisibility', 'sqs:DeleteMessage', 'sqs:GetQueueUrl'], "arn:aws:sqs:%s:%s:cfncluster-*" % (region, accountid)),
                         (['autoscaling:DescribeAutoScalingGroups', 'autoscaling:TerminateInstanceInAutoScalingGroup', 'autoscaling:SetDesiredCapacity'], "*"),
                         (['cloudwatch:PutMetricData'], "*"),
                         (['dynamodb:PutItem', 'dynamodb:Query', 'dynamodb:GetItem', 'dynamodb:DeleteItem', 'dynamodb:DescribeTable'], "arn:aws:dynamodb:%s:%s:table/cfncluster-*" % (region, accountid)),
+                        (['s3:GetObject'], "arn:aws:s3:::%s-cfncluster/*" % region),
                         (['sqs:ListQueues'], "*"),
                         (['logs:*'], "arn:aws:logs:*:*:*")]