From 57923d46ea9c9de088c2f659a6907f703bfc6d1b Mon Sep 17 00:00:00 2001 From: Sean Smith Date: Fri, 27 Jul 2018 09:03:26 -0700 Subject: [PATCH 1/7] Update generate-ami-list to append Signed-off-by: Sean Smith --- cli/cfncluster/cfnconfig.py | 2 +- util/generate-ami-list.py | 20 +++++++++++++++----- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/cli/cfncluster/cfnconfig.py b/cli/cfncluster/cfnconfig.py index 7a342a8ccf..6cb17518b2 100644 --- a/cli/cfncluster/cfnconfig.py +++ b/cli/cfncluster/cfnconfig.py @@ -177,7 +177,7 @@ def __init__(self, args): 'URL', self.template_url) except configparser.NoOptionError: if self.region == 'us-gov-west-1': - self.template_url = ('https://s3-%s.amazonaws.com/cfncluster-%s/templates/cfncluster-%s.cfn.json' + self.template_url = ('https://s3-%s.amazonaws.com/%s-cfncluster/templates/cfncluster-%s.cfn.json' % (self.region, self.region, self.version)) else: self.template_url = ('https://s3.amazonaws.com/%s-cfncluster/templates/cfncluster-%s.cfn.json' diff --git a/util/generate-ami-list.py b/util/generate-ami-list.py index a42f1cfc14..0a0fea512d 100644 --- a/util/generate-ami-list.py +++ b/util/generate-ami-list.py @@ -25,18 +25,16 @@ import sys from collections import OrderedDict - -owners = ["247102896272"] distros = OrderedDict([("alinux", "amzn"), ("centos6", "centos6"), ("centos7", "centos7"), ("ubuntu1404", "ubuntu-1404"), ("ubuntu1604", "ubuntu-1604")]) -def get_ami_list(regions, date, version): +def get_ami_list(regions, date, version, owner): amis_json = {} for region_name in regions: try: ec2 = boto3.client('ec2', region_name=region_name) - images = ec2.describe_images(Owners=owners, Filters=[{'Name': 'name', "Values": ["cfncluster-%s*%s" % (version, date)]}]) + images = ec2.describe_images(Owners=[owner], Filters=[{'Name': 'name', "Values": ["cfncluster-%s*%s" % (version, date)]}]) amis = OrderedDict() for image in images.get('Images'): @@ -76,6 +74,9 @@ def convert_json_to_txt(regions, amis_json): parser.add_argument('--date', type=str, help='release date [timestamp] (e.g. 201801112350)', required=True) parser.add_argument('--json-file', type=str, help='json output file path', required=False, default="amis.json") parser.add_argument('--txt-file', type=str, help='txt output file path', required=False, default="amis.txt") + parser.add_argument('--account-id', type=str, help='account id that owns the amis', required=False, default="247102896272") + parser.add_argument('--append', type=str, help='append new amis to current amis.txt', required=False, default=False) + parser.add_argument('--cloudformation-template', type=str, help='path to cloudfomation template', required=False, default='cloudformation/cfncluster.cfn.json') args = parser.parse_args() # get all regions @@ -83,13 +84,22 @@ def convert_json_to_txt(regions, amis_json): regions = sorted(r.get('RegionName') for r in ec2.describe_regions().get('Regions')) # get ami list - amis_json = get_ami_list(regions=regions, date=args.date, version=args.version) + amis_json = get_ami_list(regions=regions, date=args.date, version=args.version, owner=args.account_id) # write amis.json file amis_json_file = open(args.json_file, "w") json.dump(amis_json, amis_json_file, indent=2, sort_keys=True) amis_json_file.close() + # append to amis.txt file + if args.append: + with open(args.cloudformation_template) as f: + data = json.load(f) + amis = data.get('Mappings').get('AWSRegionOS2AMI') + amis.update(amis_json) + amis_json = amis + regions = sorted(amis_json) + # convert json to txt amis_txt = convert_json_to_txt(regions=regions, amis_json=amis_json) From 10f2ec71a90b404f437ded8c8845e321e020fc7a Mon Sep 17 00:00:00 2001 From: CfnCluster AMI bot Date: Fri, 27 Jul 2018 21:23:47 +0000 Subject: [PATCH 2/7] Update AMI list Signed-off-by: CfnCluster AMI bot --- amis.txt | 3 +++ cloudformation/cfncluster.cfn.json | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/amis.txt b/amis.txt index 06e0460a10..a54288c050 100644 --- a/amis.txt +++ b/amis.txt @@ -13,6 +13,7 @@ eu-west-3: ami-0b1472bf5fed029bd sa-east-1: ami-0a390e1998fb6b22f us-east-1: ami-0fd6854902340b407 us-east-2: ami-04fc90a3f60cb3dc7 +us-gov-west-1: ami-af2cbece us-west-1: ami-0054acc8361046ccc us-west-2: ami-01382f63e2667a7d1 # centos6 @@ -64,6 +65,7 @@ eu-west-3: ami-0e8f93d19635c5d49 sa-east-1: ami-048422e42509a1596 us-east-1: ami-0ba2fcbf9fa378762 us-east-2: ami-05aea9ea89d577194 +us-gov-west-1: ami-4aca572b us-west-1: ami-0089ace30f2612d13 us-west-2: ami-06142a55749b7912b # ubuntu1604 @@ -81,5 +83,6 @@ eu-west-3: ami-09c4322a65746f77b sa-east-1: ami-0740b75b4caaa3b77 us-east-1: ami-05790a6a634720a0d us-east-2: ami-0a0fcd1219e40a8f5 +us-gov-west-1: ami-3dc8555c us-west-1: ami-056472d51cf2fed45 us-west-2: ami-0702d59e57ada76b8 diff --git a/cloudformation/cfncluster.cfn.json b/cloudformation/cfncluster.cfn.json index 47834ee04f..e9f623f686 100644 --- a/cloudformation/cfncluster.cfn.json +++ b/cloudformation/cfncluster.cfn.json @@ -1741,6 +1741,11 @@ "centos7": "ami-0df85f5bc2d980e57", "ubuntu1404": "ami-06142a55749b7912b", "ubuntu1604": "ami-0702d59e57ada76b8" + }, + "us-gov-west-1": { + "alinux": "ami-af2cbece", + "ubuntu1404": "ami-4aca572b", + "ubuntu1604": "ami-3dc8555c" } }, "OSFeatures": { From 7eddac163e84c15b0a8c6d2867d4938a767d7a7b Mon Sep 17 00:00:00 2001 From: Sean Smith Date: Fri, 27 Jul 2018 14:43:52 -0700 Subject: [PATCH 3/7] Release 1.5.3rc1 Version Signed-off-by: Sean Smith --- CHANGELOG.rst | 4 ++++ cli/setup.py | 2 +- cloudformation/cfncluster.cfn.json | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 406938ba00..22b96cad72 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,10 @@ CHANGELOG ========= +1.5.3 +===== +* Add support for GovCloud, us-gov-west-1 region + 1.5.2 ===== * feature:``cfncluster``: Added ClusterUser as a stack output. This makes it easier to get the username of the head node. diff --git a/cli/setup.py b/cli/setup.py index 196279c480..cdc089ff28 100644 --- a/cli/setup.py +++ b/cli/setup.py @@ -20,7 +20,7 @@ def read(fname): return open(os.path.join(os.path.dirname(__file__), fname)).read() console_scripts = ['cfncluster = cfncluster.cli:main'] -version = "1.5.2" +version = "1.5.3rc1" requires = ['boto3>=1.7.33', 'awscli>=1.11.175', 'future>=0.16.0'] if sys.version_info[:2] == (2, 6): diff --git a/cloudformation/cfncluster.cfn.json b/cloudformation/cfncluster.cfn.json index e9f623f686..beca653716 100644 --- a/cloudformation/cfncluster.cfn.json +++ b/cloudformation/cfncluster.cfn.json @@ -1772,7 +1772,7 @@ }, "CfnClusterVersions": { "default": { - "cfncluster": "cfncluster-1.5.2", + "cfncluster": "cfncluster-1.5.3rc1", "cookbook": "cfncluster-cookbook-1.5.2", "chef": "14.2.0", "ridley": "5.1.1", From b68b29b6fe3bbac01799e2a06f035f0d8f385fdf Mon Sep 17 00:00:00 2001 From: Maurizio Melato Date: Tue, 31 Jul 2018 17:18:08 +0200 Subject: [PATCH 4/7] Fix integration test for Slurm: get num of slots After fixing the configuration of the compute nodes in a Slurm cluster and set the CPU as consumable resource we should also fix job submission in the integration tests. In order to properly test the scale up a single job submission should allocate all the slots available in a compute node. The fix has been tested. Stage 1: two jobs submitted (one running and one pending) JOBID PARTITION NAME USER ST TIME NODES NODELIST(REASON) 3 compute job2.sh centos PD 0:00 1 (Resources) 2 compute job1.sh centos R 5:18 1 ip-10-0-82-245 - one nodes with the 2 CPUs allocated [centos@ip-10-0-235-160 ~]$ scontrol show nodes --all NodeName=ip-10-0-82-245 Arch=x86_64 CoresPerSocket=1 CPUAlloc=2 CPUErr=0 CPUTot=2 CPULoad=0.11 AvailableFeatures=(null) ActiveFeatures=(null) Gres=(null) NodeAddr=ip-10-0-82-245 NodeHostName=ip-10-0-82-245 Version=16.05 OS=Linux RealMemory=3711 AllocMem=0 FreeMem=3022 Sockets=2 Boards=1 State=ALLOCATED ThreadsPerCore=1 TmpDisk=14989 Weight=1 Owner=N/A MCS_label=N/A BootTime=2018-07-31T14:37:49 SlurmdStartTime=2018-07-31T14:41:31 CapWatts=n/a CurrentWatts=0 LowestJoules=0 ConsumedJoules=0 ExtSensorsJoules=n/s ExtSensorsWatts=0 ExtSensorsTemp=n/s Stage 2: the second compute node join the cluster and the two jobs are both running on two different hosts: [centos@ip-10-0-235-160 ~]$ squeue --states=all JOBID PARTITION NAME USER ST TIME NODES NODELIST(REASON) 2 compute job1.sh centos R 6:14 1 ip-10-0-82-245 3 compute job2.sh centos R 0:34 1 ip-10-0-121-16 [centos@ip-10-0-235-160 ~]$ scontrol show nodes --all NodeName=ip-10-0-82-245 Arch=x86_64 CoresPerSocket=1 CPUAlloc=2 CPUErr=0 CPUTot=2 CPULoad=0.11 AvailableFeatures=(null) ActiveFeatures=(null) Gres=(null) NodeAddr=ip-10-0-82-245 NodeHostName=ip-10-0-82-245 Version=16.05 OS=Linux RealMemory=3711 AllocMem=0 FreeMem=3022 Sockets=2 Boards=1 State=ALLOCATED ThreadsPerCore=1 TmpDisk=14989 Weight=1 Owner=N/A MCS_label=N/A BootTime=2018-07-31T14:37:49 SlurmdStartTime=2018-07-31T14:41:31 CapWatts=n/a CurrentWatts=0 LowestJoules=0 ConsumedJoules=0 ExtSensorsJoules=n/s ExtSensorsWatts=0 ExtSensorsTemp=n/s NodeName=ip-10-0-121-16 Arch=x86_64 CoresPerSocket=1 CPUAlloc=2 CPUErr=0 CPUTot=2 CPULoad=0.37 AvailableFeatures=(null) ActiveFeatures=(null) Gres=(null) NodeAddr=ip-10-0-121-16 NodeHostName=ip-10-0-121-16 Version=(null) OS=Linux RealMemory=3711 AllocMem=0 FreeMem=3035 Sockets=2 Boards=1 State=ALLOCATED ThreadsPerCore=1 TmpDisk=14989 Weight=1 Owner=N/A MCS_label=N/A BootTime=2018-07-31T14:43:46 SlurmdStartTime=2018-07-31T14:47:35 CapWatts=n/a CurrentWatts=0 LowestJoules=0 ConsumedJoules=0 ExtSensorsJoules=n/s ExtSensorsWatts=0 ExtSensorsTemp=n/s Signed-off-by: Maurizio Melato --- tests/cluster-check.sh | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/tests/cluster-check.sh b/tests/cluster-check.sh index 3e07b79be9..4dd151ef6b 100755 --- a/tests/cluster-check.sh +++ b/tests/cluster-check.sh @@ -43,6 +43,19 @@ sge_get_slots() { echo ${ppn} } +slurm_get_slots() { + local -- ppn i=0 + ppn=$(scontrol show nodes -o | head -n 1 | sed -n -e 's/^.* CPUTot=\([0-9]\+\) .*$/\1/p') + # wait 15 secs before giving up retrieving the slots per host + while [ -z "${ppn}" -a $i -lt 15 ]; do + sleep 1 + i=$((i+1)) + ppn=$(scontrol show nodes -o | head -n 1 | sed -n -e 's/^.* CPUTot=\([0-9]\+\) .*$/\1/p') + done + + echo ${ppn} +} + torque_get_slots() { local -- chost ppn i=0 @@ -76,6 +89,12 @@ set -e # less than 8 minutes in order for the test to succeed. if test "$scheduler" = "slurm" ; then + _ppn=$(slurm_get_slots) + if [ -z "${_ppn}" ]; then + >&2 echo "The number of slots per instance couldn't be retrieved, no compute nodes available in Slurm cluster" + exit 1 + fi + cat > job1.sh < Date: Thu, 2 Aug 2018 14:27:18 +0200 Subject: [PATCH 5/7] Add missing policy for CfnCluster user profile "ec2:CreatePlacementGroup" and "ec2:DeletePlacementGroup" used when setting the placement group config to be DYNAMIC "iam:GetRole" and "iam:SimulatePrincipalPolicy" used when setting a custom instance role Signed-off-by: Luca Carrogu --- docs/source/iam.rst | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/source/iam.rst b/docs/source/iam.rst index 46b5dde5c0..9f9f86e040 100644 --- a/docs/source/iam.rst +++ b/docs/source/iam.rst @@ -8,7 +8,7 @@ IAM in CfnCluster CfnCluster utilizes multiple AWS services to deploy and operate a cluster. The services used are listed in the :ref:`AWS Services used in CfnCluster ` section of the documentation. - + CfnCluster uses EC2 IAM roles to enable instances access to AWS services for the deployment and operation of the cluster. By default the EC2 IAM role is created as part of the cluster creation by CloudFormation. This means that the user creating the cluster must have the appropriate level of permissions Defaults @@ -176,7 +176,9 @@ CfnClusterUserPolicy "ec2:DeleteSecurityGroup", "ec2:DisassociateAddress", "ec2:RevokeSecurityGroupIngress", - "ec2:ReleaseAddress" + "ec2:ReleaseAddress", + "ec2:CreatePlacementGroup", + "ec2:DeletePlacementGroup" ], "Effect": "Allow", "Resource": "*" @@ -308,7 +310,9 @@ CfnClusterUserPolicy "Action": [ "iam:PassRole", "iam:CreateRole", - "iam:DeleteRole" + "iam:DeleteRole", + "iam:GetRole", + "iam:SimulatePrincipalPolicy" ], "Effect": "Allow", "Resource": "arn:aws:iam:::role/" From a4fd967fd2e0733795514af506869a770e917df6 Mon Sep 17 00:00:00 2001 From: Sean Smith Date: Mon, 6 Aug 2018 15:43:08 -0700 Subject: [PATCH 6/7] Remove unsupported regions from cfncluster configure Signed-off-by: Sean Smith --- cli/cfncluster/easyconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/cfncluster/easyconfig.py b/cli/cfncluster/easyconfig.py index 31b6021cd7..61edbc6989 100644 --- a/cli/cfncluster/easyconfig.py +++ b/cli/cfncluster/easyconfig.py @@ -25,7 +25,7 @@ from . import cfnconfig logger = logging.getLogger('cfncluster.cfncluster') -unsupported_regions = ['ap-northeast-3'] +unsupported_regions = ['ap-northeast-3', 'cn-north-1', 'cn-northwest-1'] def prompt(prompt, default_value=None, hidden=False, options=None): if hidden and default_value is not None: From f6253b4f3cf44cccd069165fc179996006acfc29 Mon Sep 17 00:00:00 2001 From: Sean Smith Date: Wed, 8 Aug 2018 14:14:44 -0700 Subject: [PATCH 7/7] Release 1.5.3 Adds support for GovCloud (us-gov-west-1) region Signed-off-by: Sean Smith --- cli/setup.py | 2 +- cloudformation/cfncluster.cfn.json | 2 +- docs/source/conf.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/setup.py b/cli/setup.py index cdc089ff28..1c6475cc0e 100644 --- a/cli/setup.py +++ b/cli/setup.py @@ -20,7 +20,7 @@ def read(fname): return open(os.path.join(os.path.dirname(__file__), fname)).read() console_scripts = ['cfncluster = cfncluster.cli:main'] -version = "1.5.3rc1" +version = "1.5.3" requires = ['boto3>=1.7.33', 'awscli>=1.11.175', 'future>=0.16.0'] if sys.version_info[:2] == (2, 6): diff --git a/cloudformation/cfncluster.cfn.json b/cloudformation/cfncluster.cfn.json index beca653716..c98c9b8be0 100644 --- a/cloudformation/cfncluster.cfn.json +++ b/cloudformation/cfncluster.cfn.json @@ -1772,7 +1772,7 @@ }, "CfnClusterVersions": { "default": { - "cfncluster": "cfncluster-1.5.3rc1", + "cfncluster": "cfncluster-1.5.3", "cookbook": "cfncluster-cookbook-1.5.2", "chef": "14.2.0", "ridley": "5.1.1", diff --git a/docs/source/conf.py b/docs/source/conf.py index e41f989463..99d5f2b405 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -52,7 +52,7 @@ # The short X.Y version. version = '1.5' # The full version, including alpha/beta/rc tags. -release = '1.5.2' +release = '1.5.3' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages.