Merge pull request #169 from ajfriedman18/master

Add EFS support in Nextflow and other minor changes

README.md

@@ -55,6 +55,15 @@ aws cloudformation create-stack \
 
 ```
 
+## Shared File System Support
+
+Amazon EFS is supported out of the box for `GWFCore` and `Nextflow`. You have two options to use EFS.
+
+1. **Create a new EFS File System:** Be sure to have `CreateEFS` set to `Yes` and also include the total number of subnets.
+2. **Use an Exisitng EFS File System:** Be sure to specify the EFS ID in the `ExistingEFS` parameter. This file system should be accessible from every subnet you specify.
+
+Following successful deployment of `GWFCore`, when creating your Nextflow Resources, set `MountEFS` to `Yes`.
+
 ## Building the documentation
 
 The documentation is built using mkdocs.

src/containers/nextflow/nextflow.aws.sh

@@ -40,13 +40,19 @@ cd /opt/work/$GUID
 NF_CONFIG=./nextflow.config
 echo "Creating config file: $NF_CONFIG"
 
+# To figure out - batch volumes 
 cat << EOF > $NF_CONFIG
 workDir = "$NF_WORKDIR"
 process.executor = "awsbatch"
 process.queue = "$NF_JOB_QUEUE"
 aws.batch.cliPath = "$AWS_CLI_PATH"
 EOF
 
+if [[ "$EFS_MOUNT" != "" ]]
+then
+    echo aws.batch.volumes = [\"/mnt/efs\"] >> $NF_CONFIG
+fi
+
 echo "=== CONFIGURATION ==="
 cat ./nextflow.config
 

src/templates/gwfcore/gwfcore-code.template.yaml

@@ -36,9 +36,7 @@ Mappings:
           Value: "default"
 
 # Conditions:
-
 Resources:
-
   IAMCodeBuildRole:
     Type: AWS::IAM::Role
     Properties:
@@ -204,7 +202,6 @@ Resources:
       RepositoryName: !Sub ecs-additions-${Namespace}
       RepositoryDescription: |
         Provisioning scripts for Batch instances
-      
       Code:
         S3:
           Bucket: !Ref ArtifactBucketName

src/templates/gwfcore/gwfcore-efs.template.yaml

@@ -0,0 +1,112 @@
+---
+AWSTemplateFormatVersion: '2010-09-09'
+Description: >-
+  (WWPS-GLS-WF-GWFCORE-EFS) Creates EFS file system and mount targets to a list of subnets
+
+Parameters:
+  VpcId:
+    Type: AWS::EC2::VPC::Id
+    Description: 'The VPC to create security groups and deploy AWS Batch to. NOTE: Must be the same VPC as the provided subnet IDs.'
+  SubnetIds:
+    Type: List<AWS::EC2::Subnet::Id>
+    Description: 'Subnets you want your batch compute environment to launch in. We recommend private subnets. NOTE: Must be from the VPC provided.'
+  NumberOfSubnets:
+    Type: Number
+    Description: Number of subnets to launch into. Should correspond to the length of Subnet Ids
+    MinValue: 1
+    MaxValue: 6
+
+Conditions:
+  2SubnetCondition: !Or
+    - !Equals [!Ref 'NumberOfSubnets', '2']
+    - !Condition '3SubnetCondition'
+  3SubnetCondition: !Or
+    - !Equals [!Ref 'NumberOfSubnets', '3']
+    - !Condition '4SubnetCondition'
+  4SubnetCondition: !Or
+    - !Equals [!Ref 'NumberOfSubnets', '4']
+    - !Condition '5SubnetCondition'
+  5SubnetCondition: !Or
+    - !Equals [!Ref 'NumberOfSubnets', '5']
+    - !Condition '6SubnetCondition'
+  6SubnetCondition: !Equals [!Ref NumberOfSubnets, '6']
+
+Resources:
+  SharedDataFileSystem:
+    Type: AWS::EFS::FileSystem
+    Properties:
+      PerformanceMode: generalPurpose
+      Encrypted: true
+      FileSystemTags:
+      - Key: Name
+        Value: SharedDataGenomics
+
+  MountTargetSecurityGroup:
+    Type: AWS::EC2::SecurityGroup
+    Properties:
+      VpcId: !Ref VpcId
+      GroupDescription: Security group for mount target
+      SecurityGroupIngress:
+      - IpProtocol: tcp
+        FromPort: 2049
+        ToPort: 2049
+        CidrIp: 0.0.0.0/0
+
+  MountTargetSubnet1:
+    Type: AWS::EFS::MountTarget
+    Properties:
+      FileSystemId: !Ref SharedDataFileSystem
+      SubnetId: !Select [0, !Ref SubnetIds]
+      SecurityGroups: 
+      - !Ref MountTargetSecurityGroup
+
+  MountTargetSubnet2:
+    Type: AWS::EFS::MountTarget
+    Condition: 2SubnetCondition
+    Properties:
+      FileSystemId: !Ref SharedDataFileSystem
+      SubnetId: !Select [1, !Ref SubnetIds]
+      SecurityGroups: 
+      - !Ref MountTargetSecurityGroup
+
+  MountTargetSubnet3:
+    Type: AWS::EFS::MountTarget
+    Condition: 3SubnetCondition
+    Properties:
+      FileSystemId: !Ref SharedDataFileSystem
+      SubnetId: !Select [2, !Ref SubnetIds]
+      SecurityGroups: 
+      - !Ref MountTargetSecurityGroup
+
+  MountTargetSubnet4:
+    Type: AWS::EFS::MountTarget
+    Condition: 4SubnetCondition
+    Properties:
+      FileSystemId: !Ref SharedDataFileSystem
+      SubnetId: !Select [3, !Ref SubnetIds]
+      SecurityGroups: 
+      - !Ref MountTargetSecurityGroup
+
+  MountTargetSubnet5:
+    Type: AWS::EFS::MountTarget
+    Condition: 5SubnetCondition
+    Properties:
+      FileSystemId: !Ref SharedDataFileSystem
+      SubnetId: !Select [4, !Ref SubnetIds]
+      SecurityGroups: 
+      - !Ref MountTargetSecurityGroup
+
+  MountTargetSubnet6:
+    Type: AWS::EFS::MountTarget
+    Condition: 6SubnetCondition
+    Properties:
+      FileSystemId: !Ref SharedDataFileSystem
+      SubnetId: !Select [5, !Ref SubnetIds]
+      SecurityGroups: 
+      - !Ref MountTargetSecurityGroup
+
+Outputs:
+  EfsId:
+    Value: !Ref SharedDataFileSystem
+    Description: EFS ID
+

src/templates/gwfcore/gwfcore-iam.template.yaml

@@ -21,16 +21,16 @@ Resources:
             Version: 2012-10-17
             Statement:
               - Effect: Deny
-                Resource: !Join ["", ["arn:aws:s3:::", !Ref S3BucketName]]
+                Resource: !Sub "arn:aws:s3:::${S3BucketName}"
                 Action:
                   - "s3:Delete*"
                   - "s3:PutBucket*"
               - Effect: Allow
-                Resource: !Join ["", ["arn:aws:s3:::", !Ref S3BucketName]]
+                Resource: !Sub "arn:aws:s3:::${S3BucketName}"
                 Action:
                   - "s3:ListBucket*"
               - Effect: Allow
-                Resource: !Join ["", ["arn:aws:s3:::", !Ref S3BucketName, "/*"]]
+                Resource: !Sub "arn:aws:s3:::${S3BucketName}/*"
                 Action:
                   - "s3:*"
         - PolicyName: !Sub SecretManager-Access-${AWS::Region}
@@ -92,10 +92,16 @@ Resources:
               - Sid: S3BucketAllowAllObjectOps
                 Effect: Allow
                 Resource: 
-                  - !Join ["", ["arn:aws:s3:::", !Ref S3BucketName]]
-                  - !Join ["", ["arn:aws:s3:::", !Ref S3BucketName, "/*"]]
+                  - !Sub "arn:aws:s3:::${S3BucketName}"
+                  - !Sub "arn:aws:s3:::${S3BucketName}/*"
                 Action:
                   - "s3:*"
+              - Sid: DenyDeleteBucket
+                Effect: Deny
+                Resource: !Sub "arn:aws:s3:::${S3BucketName}"
+                Action:
+                - "s3:DeleteBucket*"
+                - "s3:CreateBucket"
 
         # required for amazon-ebs-autoscale to resize filesystems
         - PolicyName: !Sub Autoscale-EBS-${AWS::Region}