From b9c40872b4a88750979eddc0031fef4285a4d579 Mon Sep 17 00:00:00 2001 From: Allan Carter Date: Thu, 24 Oct 2024 18:53:29 +0000 Subject: [PATCH] Update RES templates Get latest RES templates Create separate BI templates and RES-only templates. Change default keycloak instance type from t3.micro to c7a.medium. Resolves #275 --- res/Makefile | 13 + res/download-res-templates.sh | 8 +- res/res-demo-original/bi.yaml | 20 +- res/res-demo-original/keycloak.yaml | 361 +++++++++++++ res/res-demo-original/networking.yaml | 38 ++ res/res-demo-original/res-demo-stack.yaml | 126 ++++- res/res-demo-original/res-sso-keycloak.yaml | 405 +++++++++++++++ res/res-demo-with-cidr/bi.yaml | 27 +- res/res-demo-with-cidr/keycloak.yaml | 377 ++++++++++++++ res/res-demo-with-cidr/res-bi-only.yaml | 513 +++++++++++++++++++ res/res-demo-with-cidr/res-demo-stack.yaml | 167 +++++- res/res-demo-with-cidr/res-only.yaml | 417 +++++++++++++++ res/res-demo-with-cidr/res-sso-keycloak.yaml | 420 +++++++++++++++ res/upload-res-templates.py | 19 +- 14 files changed, 2868 insertions(+), 43 deletions(-) create mode 100644 res/Makefile create mode 100644 res/res-demo-original/keycloak.yaml create mode 100644 res/res-demo-original/res-sso-keycloak.yaml create mode 100644 res/res-demo-with-cidr/keycloak.yaml create mode 100644 res/res-demo-with-cidr/res-bi-only.yaml create mode 100644 res/res-demo-with-cidr/res-only.yaml create mode 100644 res/res-demo-with-cidr/res-sso-keycloak.yaml diff --git a/res/Makefile b/res/Makefile new file mode 100644 index 00000000..a8748cec --- /dev/null +++ b/res/Makefile @@ -0,0 +1,13 @@ + +diff-base: + meld res-demo-original/bi.yaml res-demo-with-cidr/bi.yaml & + meld res-demo-original/keycloak.yaml res-demo-with-cidr/keycloak.yaml & + meld res-demo-original/res-sso-keycloak.yaml res-demo-with-cidr/res-sso-keycloak.yaml & + meld res-demo-original/res.ldif res-demo-with-cidr/res.ldif & + +diff-top: + meld res-demo-original/res-demo-stack.yaml res-demo-with-cidr/res-demo-stack.yaml & + meld res-demo-with-cidr/res-demo-stack.yaml res-demo-with-cidr/res-bi-only.yaml & + meld res-demo-with-cidr/res-demo-stack.yaml res-demo-with-cidr/res-only.yaml & + +diff: diff-base diff-top diff --git a/res/download-res-templates.sh b/res/download-res-templates.sh index 9f7440a8..55ac6837 100755 --- a/res/download-res-templates.sh +++ b/res/download-res-templates.sh @@ -7,8 +7,10 @@ script_dir=$(dirname $(realpath $0)) cd $script_dir -aws s3 cp s3://aws-hpc-recipes/main/recipes/res/res_demo_env/assets/res-demo-stack.yaml res-demo-original/. -aws s3 cp s3://aws-hpc-recipes/main/recipes/res/res_demo_env/assets/bi.yaml res-demo-original/. -aws s3 cp s3://aws-hpc-recipes/main/recipes/net/hpc_large_scale/assets/main.yaml res-demo-original/networking.yaml +aws s3 cp s3://aws-hpc-recipes/main/recipes/res/res_demo_env/assets/bi.yaml res-demo-original/. +aws s3 cp s3://aws-hpc-recipes/main/recipes/res/res_demo_env/assets/keycloak.yaml res-demo-original/. +aws s3 cp s3://aws-hpc-recipes/main/recipes/res/res_demo_env/assets/res-demo-stack.yaml res-demo-original/. +aws s3 cp s3://aws-hpc-recipes/main/recipes/res/res_demo_env/assets/res-sso-keycloak.yaml res-demo-original/. +aws s3 cp s3://aws-hpc-recipes/main/recipes/net/hpc_large_scale/assets/main.yaml res-demo-original/networking.yaml aws s3 cp s3://aws-hpc-recipes/main/recipes/res/res_demo_env/assets/res.ldif res-demo-original/. diff --git a/res/res-demo-original/bi.yaml b/res/res-demo-original/bi.yaml index c945fdb3..9e615b91 100644 --- a/res/res-demo-original/bi.yaml +++ b/res/res-demo-original/bi.yaml @@ -36,21 +36,22 @@ Parameters: EnvironmentName: Description: (Optional) EnvironmentName must start with "res-"and should be less than or equal to 11 characters. Required to generate certificates. Type: String - AllowedPattern: ^res-[A-Za-z\-\_0-9]{0,7}$ + AllowedPattern: ^$|^res-[A-Za-z\-\_0-9]{0,7}$ + Default: res-demo AdminPassword: Description: Provide the Active Directory Administrator Account Password Directly or Resource ARN to Secret Containing Password. Type: String MinLength: 8 MaxLength: 2048 - AllowedPattern: (arn:(aws(-cn|-us-gov)?):secretsmanager:(us(-gov)?|ap|ca|cn|eu|sa)-(central|(north|south)?(east|west)?)-\d:\d{12}:secret:[a-zA-Z0-9/_+=.@-]+)|(?=^.{8,64}$)((?=.*\d)(?=.*[A-Z])(?=.*[a-z])|(?=.*\d)(?=.*[^A-Za-z0-9\s])(?=.*[a-z])|(?=.*[^A-Za-z0-9\s])(?=.*[A-Z])(?=.*[a-z])|(?=.*\d)(?=.*[A-Z])(?=.*[^A-Za-z0-9\s]))^.* + AllowedPattern: (arn:(aws(-cn|-us-gov)?):secretsmanager:(us(-gov)?|ap|ca|cn|eu|il|sa)-(central|(north|south)?(east|west)?)-\d:\d{12}:secret:[a-zA-Z0-9/_+=.@-]+)|(?=^.{8,64}$)((?=.*\d)(?=.*[A-Z])(?=.*[a-z])|(?=.*\d)(?=.*[^A-Za-z0-9\s])(?=.*[a-z])|(?=.*[^A-Za-z0-9\s])(?=.*[A-Z])(?=.*[a-z])|(?=.*\d)(?=.*[A-Z])(?=.*[^A-Za-z0-9\s]))^.* NoEcho: true ServiceAccountPassword: Description: Provide the Active Directory Service Account Password Directly or Resource ARN to Secret Containing Password. Type: String MinLength: 8 MaxLength: 2048 - AllowedPattern: (arn:(aws(-cn|-us-gov)?):secretsmanager:(us(-gov)?|ap|ca|cn|eu|sa)-(central|(north|south)?(east|west)?)-\d:\d{12}:secret:[a-zA-Z0-9/_+=.@-]+)|(?=^.{8,64}$)((?=.*\d)(?=.*[A-Z])(?=.*[a-z])|(?=.*\d)(?=.*[^A-Za-z0-9\s])(?=.*[a-z])|(?=.*[^A-Za-z0-9\s])(?=.*[A-Z])(?=.*[a-z])|(?=.*\d)(?=.*[A-Z])(?=.*[^A-Za-z0-9\s]))^.* + AllowedPattern: (arn:(aws(-cn|-us-gov)?):secretsmanager:(us(-gov)?|ap|ca|cn|eu|il|sa)-(central|(north|south)?(east|west)?)-\d:\d{12}:secret:[a-zA-Z0-9/_+=.@-]+)|(?=^.{8,64}$)((?=.*\d)(?=.*[A-Z])(?=.*[a-z])|(?=.*\d)(?=.*[^A-Za-z0-9\s])(?=.*[a-z])|(?=.*[^A-Za-z0-9\s])(?=.*[A-Z])(?=.*[a-z])|(?=.*\d)(?=.*[A-Z])(?=.*[^A-Za-z0-9\s]))^.* NoEcho: true LDIFS3Path: Description: (Optional) An S3 Path (without the s3://) to an LDIF file that will be used during stack creation. @@ -482,7 +483,7 @@ Resources: response_data['Message'] = 'Resource creation successful!' physical_resource_id = create_physical_resource_id() - secretsmanager_arn_regex_pattern = r"(arn:(aws(-cn|-us-gov)?):secretsmanager:(us(-gov)?|ap|ca|cn|eu|sa)-(central|(north|south)?(east|west)?)-\d:\d{12}:secret:[a-zA-Z0-9/_+=.@-]+)" + secretsmanager_arn_regex_pattern = r"(arn:(aws(-cn|-us-gov)?):secretsmanager:(us(-gov)?|ap|ca|cn|eu|il|sa)-(central|(north|south)?(east|west)?)-\d:\d{12}:secret:[a-zA-Z0-9/_+=.@-]+)" admin_arn_match = re.search(secretsmanager_arn_regex_pattern, admin_password) service_account_arn_match = re.search(secretsmanager_arn_regex_pattern, service_account_password) @@ -547,10 +548,8 @@ Outputs: Value: !Sub - dc=${dc} - { dc: !Join [",dc=", !Split [".", !If [ SubDomainNotProvided, !Ref DomainName, !Join [ ".", [ !Ref SubDomain, !Ref DomainName] ] ] ]] } - ServiceAccountUsername: - Value: ServiceAccount - ServiceAccountPasswordSecretArn: - Value: !GetAtt [ DirectoryService, Outputs.PasswordSecretArn ] + ServiceAccountCredentialsSecretArn: + Value: !GetAtt [ DirectoryService, Outputs.CredentialsSecretArn ] ServiceAccountUserDN: Description: The Distinguished Name (DN) of the ServiceAccount user in your Active Directory Value: !Sub @@ -568,11 +567,6 @@ Outputs: Value: !Sub - OU=Users,OU=RES,OU=${ou},DC=${dc} - { dc: !Join [",DC=", !Split [".", !If [ SubDomainNotProvided, !Ref DomainName, !Join [ ".", [ !Ref SubDomain, !Ref DomainName]]]]], ou: !GetAtt [ DirectoryService, Outputs.DomainShortName ]} - SudoersOU: - Description: The OU for users who should have sudoers permission across all projects. The value provided here is based off of a supplied LDIF file. - Value: !Sub - - OU=Users,OU=RES,OU=${ou},DC=${dc} - - { dc: !Join [",DC=", !Split [".", !If [ SubDomainNotProvided, !Ref DomainName, !Join [ ".", [ !Ref SubDomain, !Ref DomainName]]]]], ou: !GetAtt [ DirectoryService, Outputs.DomainShortName ]} ComputersOU: Description: The OU for computers that join the AD. The value provided here is based off of a supplied LDIF file. Value: !Sub diff --git a/res/res-demo-original/keycloak.yaml b/res/res-demo-original/keycloak.yaml new file mode 100644 index 00000000..1020ce2a --- /dev/null +++ b/res/res-demo-original/keycloak.yaml @@ -0,0 +1,361 @@ +Description: Keycloak Server + +Metadata: + AWS::CloudFormation::Interface: + ParameterGroups: + - Label: + default: Keycloak Configuration + Parameters: + - Keypair + - ServiceAccountCredentialsSecretArn + - VpcId + - PublicSubnet + - ServiceAccountUserDN + - UsersDN + - LDAPConnectionURI + - CogntioUserPoolId + - EnvironmentBaseURL + - SAMLRedirectUrl + +Parameters: + Keypair: + Description: EC2 Keypair to access management instance. + Type: AWS::EC2::KeyPair::KeyName + + ServiceAccountCredentialsSecretArn: + Type: String + AllowedPattern: ^(?:arn:(?:aws|aws-us-gov|aws-cn):secretsmanager:[a-z0-9-]{1,20}:[0-9]{12}:secret:[A-Za-z0-9-_+=,\.@]{1,128})?$ + Description: Directory Service Service Account Credentials Secret ARN. The username and password for the Active Directory ServiceAccount user formatted as a username:password key/value pair. + + VpcId: + Type: AWS::EC2::VPC::Id + AllowedPattern: vpc-[0-9a-f]{17} + ConstraintDescription: VpcId must begin with 'vpc-', only contain letters (a-f) or numbers(0-9) and must be 21 characters in length + + PublicSubnet: + Type: AWS::EC2::Subnet::Id + AllowedPattern: subnet-.+ + Description: Select a public subnet from the already selected VPC + + ServiceAccountUserDN: + Type: String + AllowedPattern: .+ + Description: Provide the Distinguished name (DN) of the service account user in the Active Directory + + UsersDN: + Type: String + AllowedPattern: .+ + Description: Please provide Users Organization Unit in your active directory under which all of your users exist. For example, OU=Users,DC=RES,DC=example,DC=internal + + LDAPConnectionURI: + Type: String + AllowedPattern: .+ + Description: Please provide the active directory connection URI (e.g. ldap://www.example.com) + + CogntioUserPoolId: + Type: String + AllowedPattern: .+ + Description: Please provide the Cognito user pool id (e.g. us-east-1_ababab) + + EnvironmentBaseURL: + Type: String + AllowedPattern: https?://.+ + Description: Please provide your base URL for your environment + + SAMLRedirectUrl: + Type: String + AllowedPattern: https://.+\.amazoncognito\.com/saml2/idpresponse + Description: Please provide the SAML redirect URL + +Mappings: + Keycloak: + Config: + Version: "24.0.3" + + +Resources: + KeycloakSecret: + Type: AWS::SecretsManager::Secret + Properties: + Name: !Sub + - KeycloakSecret-${AWS::StackName}-${StackIdSuffix} + - StackIdSuffix: !Select [2, !Split ['/', !Ref 'AWS::StackId']] + Description: Keycloak secret + GenerateSecretString: + PasswordLength: 14 + ExcludePunctuation: true + + KeycloakEC2InstanceRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - ec2.amazonaws.com + Action: + - sts:AssumeRole + ManagedPolicyArns: + - !Sub arn:${AWS::Partition}:iam::aws:policy/AmazonSSMManagedInstanceCore + Policies: + - PolicyName: KeycloakEC2InstancePolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: secretsmanager:GetSecretValue + Resource: + - !Ref KeycloakSecret + - !Ref ServiceAccountCredentialsSecretArn + - Effect: Allow + Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogEvents + Resource: '*' + + KeycloakEC2InstanceProfile: + Type: AWS::IAM::InstanceProfile + Properties: + Roles: + - !Ref KeycloakEC2InstanceRole + + KeycloakSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + GroupDescription: Keycloak security group + VpcId: !Ref VpcId + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 80 + ToPort: 80 + CidrIp: "0.0.0.0/0" + + KeycloakEC2Instance: + Type: AWS::EC2::Instance + DependsOn: + - KeycloakSecurityGroup + - KeycloakEC2InstanceProfile + - KeycloakSecret + CreationPolicy: + ResourceSignal: + Timeout: PT15M + Properties: + ImageId: '{{resolve:ssm:/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-6.1-x86_64:75}}' + InstanceType: t3.micro + KeyName: !Ref Keypair + IamInstanceProfile: !Ref KeycloakEC2InstanceProfile + SubnetId: !Ref PublicSubnet + SecurityGroupIds: + - !Ref KeycloakSecurityGroup + Tags: + - Key: Name + Value: !Sub keycloak-${AWS::StackName} + UserData: + Fn::Base64: + Fn::Sub: + - | + #!/bin/sh -x + mkdir -p /root/bootstrap && cd /root/bootstrap + mkdir -p /root/bootstrap/logs/ + exec > /root/bootstrap/logs/userdata.log 2>&1 + + # Create utils.sh script + echo -e "#!/bin/sh + wait_for_server() { + SERVER_URL=\$1 + MAX_ATTEMPTS=\$2 + RETRY_INTERVAL=\$3 + attempt=0 + while [ \$attempt -lt \$MAX_ATTEMPTS ]; do + response=\$(curl -s -o /dev/null -w \"%{http_code}\" \"\$SERVER_URL\") + if [ \"\$response\" == \"200\" ] || [ \"\$response\" == \"302\" ]; then + echo \"Server is up!\" + return 0 + else + echo \"Server is not yet up. Retrying in \$RETRY_INTERVAL seconds...\" + sleep \$RETRY_INTERVAL + ((attempt++)) + fi + done + echo \"Server is not up after \$MAX_ATTEMPTS attempts, exiting...\" + return 1 + } + " > /root/bootstrap/utils.sh + + #Install java17 + MAX_ATTEMPTS=5 + RETRY_INTERVAL=5 + attempt=0 + while [ $attempt -lt $MAX_ATTEMPTS ]; do + #Clean yum cache + sudo yum clean packages + #Install java-17 + sudo yum -y install java-17-amazon-corretto-headless + which java && break + sleep $RETRY_INTERVAL + ((attempt++)) + done + + export KEYCLOAK_VERSION=${KeycloakVersion} + wget https://github.com/keycloak/keycloak/releases/download/$KEYCLOAK_VERSION/keycloak-$KEYCLOAK_VERSION.zip + unzip keycloak-$KEYCLOAK_VERSION.zip + + cd keycloak-$KEYCLOAK_VERSION + + export KC_HTTP_PORT=80 + export KEYCLOAK_ADMIN=admin + set +x + export KEYCLOAK_ADMIN_PASSWORD=$(aws secretsmanager get-secret-value --secret-id ${KeycloakSecret} --query SecretString --region ${AWS::Region} --output text) + set -x + + # Start Keycloak + sudo -E nohup ./bin/kc.sh start-dev --http-port 80 > keycloak.log & + sleep 30 + + SERVER_URL="http://0.0.0.0:80" + MAX_ATTEMPTS=15 + RETRY_INTERVAL=10 + + # Initial setup to wait for the server to be up + . /root/bootstrap/utils.sh + wait_for_server "$SERVER_URL" $MAX_ATTEMPTS $RETRY_INTERVAL + if [ $? -ne 0 ]; then + /opt/aws/bin/cfn-signal -e 1 --stack "${AWS::StackName}" --resource "KeycloakEC2Instance" --region "${AWS::Region}" + sleep 30 + fi + + echo "Keycloak server is up" + # Login to Keycloak + set +x + ./bin/kcadm.sh config credentials --server $SERVER_URL --realm master --user admin --password $KEYCLOAK_ADMIN_PASSWORD + set -x + + # Create realm named 'res' + ./bin/kcadm.sh create realms -s realm=res -s id=res -s enabled=true -o + + # Set sslRequired to NONE + ./bin/kcadm.sh update realms/master -s sslRequired=NONE --server $SERVER_URL + ./bin/kcadm.sh update realms/res -s sslRequired=NONE --server $SERVER_URL + + #Configure Keycloak + #Get ServiceAccount passsword + set +x + serviceAccountPassword=$(aws secretsmanager get-secret-value --secret-id ${ServiceAccountCredentialsSecretArn} --query SecretString --region ${AWS::Region} --output text | jq -r 'to_entries[] | .value') + + #Create storage component to sync from AD + componentId=$(./bin/kcadm.sh create components -s name=ldap -s parentId=res -s providerId=ldap -s providerType=org.keycloak.storage.UserStorageProvider \ + -s 'config.authType=["simple"]' -s "config.bindCredential=[\"$serviceAccountPassword\"]" -s 'config.bindDn=["${ServiceAccountUserDN}"]' \ + -s 'config.connectionUrl=["${LDAPConnectionURI}"]' -s 'config.editMode=["READ_ONLY"]' -s 'config.enabled=["true"]' -s 'config.rdnLDAPAttribute=["cn"]' \ + -s 'config.searchScope=["2"]' -s 'config.usernameLDAPAttribute=["sAMAccountName"]' \ + -s 'config.usersDn=["${UsersDN}"]' -s 'config.uuidLDAPAttribute=["objectGUID"]' \ + -s 'config.vendor=["ad"]' -s 'config.userObjectClasses=["person, organizationalPerson, user"]' -r res -i) + set -x + + # Trigger user sync + ./bin/kcadm.sh create user-storage/$componentId/sync?action=triggerFullSync -r res + + #Create SSO SAML client for SSO + clientId=$(./bin/kcadm.sh create clients -r res -s baseUrl=${EnvironmentBaseURL} \ + -s clientId=urn:amazon:cognito:sp:${CogntioUserPoolId} -s name=saml -s protocol=saml -s 'redirectUris=["*"]' -s rootUrl=${EnvironmentBaseURL} \ + -s 'attributes.saml_name_id_format=email' -s 'attributes."post.logout.redirect.uris"=${EnvironmentBaseURL}' \ + -s 'attributes."saml.client.signature"=false' -s 'attributes."saml.force.post.binding"=true' -s 'attributes."saml.authnstatement"=true' \ + -s 'attributes."saml_assertion_consumer_url_post"=${SAMLRedirectUrl}' \ + -s 'attributes.saml_single_logout_service_url_redirect=${EnvironmentBaseURL}' -i) + + # Create email mapper + ./bin/kcadm.sh create clients/$clientId/protocol-mappers/models -s name=email_mapper -s protocol=saml -s protocolMapper=saml-user-property-mapper \ + -s 'config."attribute.name"=email' -s 'config."attribute.nameformat"=Unspecified' -s 'config."friendly.name"=email_mapper' -s 'config."user.attribute"=email' -r res + + ##Schedule crontabs + #Install crontab on al3 + sudo yum -y install cronie + sudo systemctl enable crond.service + sudo systemctl start crond.service + + #Crontab1 - service account password rotation - script + echo -e "#!/bin/sh -x + exec >> /root/bootstrap/logs/userdata.log 2>&1 + echo Updating service account password + cd /root/bootstrap/keycloak-$KEYCLOAK_VERSION + SERVER_URL=\"http://0.0.0.0:80\" + set +x + kc_admin_password=\$(aws secretsmanager get-secret-value --secret-id ${KeycloakSecret} --query SecretString --region ${AWS::Region} --output text) + serviceAccountPassword=\$(aws secretsmanager get-secret-value --secret-id ${ServiceAccountCredentialsSecretArn} --query SecretString --region ${AWS::Region} --output text | jq -r 'to_entries[] | .value') + ./bin/kcadm.sh config credentials --server \$SERVER_URL --realm master --user admin --password \$kc_admin_password + ./bin/kcadm.sh update components/$componentId -s name=ldap -s parentId=res -s providerId=ldap -s providerType=org.keycloak.storage.UserStorageProvider \\ + -s 'config.authType=[\"simple\"]' -s \"config.bindCredential=[\\\"\$serviceAccountPassword\\\"]\" -s 'config.bindDn=[\"${ServiceAccountUserDN}\"]' \\ + -s 'config.connectionUrl=[\"${LDAPConnectionURI}\"]' -s 'config.editMode=[\"READ_ONLY\"]' -s 'config.enabled=[\"true\"]' -s 'config.rdnLDAPAttribute=[\"cn\"]' \\ + -s 'config.searchScope=[\"2\"]' -s 'config.usernameLDAPAttribute=[\"sAMAccountName\"]' \\ + -s 'config.usersDn=[\"${UsersDN}\"]' -s 'config.uuidLDAPAttribute=[\"objectGUID\"]' \\ + -s 'config.vendor=[\"ad\"]' -s 'config.userObjectClasses=[\"person, organizationalPerson, user\"]' -r res + set -x + ./bin/kcadm.sh create user-storage/$componentId/sync?action=triggerFullSync -r res + " > /root/bootstrap/password_rotation.sh + chmod +x /root/bootstrap/password_rotation.sh + + #Crontab2 - user sync - script + echo -e "#!/bin/sh -x + exec >> /root/bootstrap/logs/userdata.log 2>&1 + echo Syncing users + cd /root/bootstrap/keycloak-$KEYCLOAK_VERSION + SERVER_URL=\"http://0.0.0.0:80\" + set +x + kc_admin_password=\$(aws secretsmanager get-secret-value --secret-id ${KeycloakSecret} --query SecretString --region ${AWS::Region} --output text) + ./bin/kcadm.sh config credentials --server \$SERVER_URL --realm master --user admin --password \$kc_admin_password + set -x + ./bin/kcadm.sh create user-storage/$componentId/sync?action=triggerFullSync -r res + " > /root/bootstrap/user_sync.sh + chmod +x /root/bootstrap/user_sync.sh + + (crontab -l; echo "*/30 * * * * /root/bootstrap/password_rotation.sh") | crontab - + (crontab -l; echo "*/5 * * * * /root/bootstrap/user_sync.sh") | crontab - + + # Monitoring script to restart Keycloak if it goes down + echo -e "#!/bin/sh -x + exec >> /root/bootstrap/logs/userdata.log 2>&1 + . /root/bootstrap/utils.sh + SERVER_URL=\"http://0.0.0.0:80\" + MAX_ATTEMPTS=15 + RETRY_INTERVAL=10 + + while true; do + echo \"Start monitoring keycloak server...\" + response=\$(curl -s -o /dev/null -w \"%{http_code}\" \"\$SERVER_URL\") + if [ \"\$response\" == \"200\" ] || [ \"\$response\" == \"302\" ]; then + echo \"Keycloak server is running.\" + else + # Check for running Keycloak processes and kill them if found + if pgrep -f \"keycloak\" > /dev/null; then + pkill -f \"keycloak\" + echo \"Killed existing Keycloak processes.\" + else + echo \"No Keycloak processes found.\" + fi + echo \"Keycloak server is down. Restarting...\" + + cd /root/bootstrap/keycloak-$KEYCLOAK_VERSION + sudo -E nohup ./bin/kc.sh start-dev --http-port 80 > keycloak.log & + wait_for_server \"\$SERVER_URL\" \$MAX_ATTEMPTS \$RETRY_INTERVAL + fi + sleep 60 + done + " > /root/bootstrap/monitor.sh + chmod +x /root/bootstrap/monitor.sh + + # Start the monitoring script in the background + nohup /root/bootstrap/monitor.sh & + + # Signal stack to continue based on last command output + /opt/aws/bin/cfn-signal -e $? --stack "${AWS::StackName}" --resource "KeycloakEC2Instance" --region "${AWS::Region}" + - KeycloakVersion: !FindInMap [Keycloak, Config, Version] + +Outputs: + KeycloakUrl: + Description: Keycloak administrator URL + Value: !Sub http://${KeycloakEC2Instance.PublicIp}:80 + KeycloakAdminPasswordSecretArn: + Description: Keycloak password for admin user + Value: !Sub ${KeycloakSecret} diff --git a/res/res-demo-original/networking.yaml b/res/res-demo-original/networking.yaml index e96d0bfe..731aee39 100644 --- a/res/res-demo-original/networking.yaml +++ b/res/res-demo-original/networking.yaml @@ -202,6 +202,44 @@ Resources: Tags: - Key: "Name" Value: !Sub '${AWS::StackName}:Large-Scale-HPC' + + VPCFlowLog: + Type: AWS::EC2::FlowLog + Properties: + ResourceId: !Ref VPC + ResourceType: VPC + TrafficType: ALL + LogDestinationType: cloud-watch-logs + LogGroupName: !Sub '${AWS::StackName}-VPCFlowLogs' + DeliverLogsPermissionArn: !GetAtt FlowLogRole.Arn + + FlowLogRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: + - vpc-flow-logs.amazonaws.com + Action: + - "sts:AssumeRole" + ManagedPolicyArns: + - !Ref AWS::NoValue + Policies: + - PolicyName: FlowLogPolicy + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: + - "logs:CreateLogGroup" + - "logs:CreateLogStream" + - "logs:PutLogEvents" + - "logs:DescribeLogGroups" + - "logs:DescribeLogStreams" + Resource: !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:${AWS::StackName}-VPCFlowLogs:*" PublicSubnetA: Type: AWS::EC2::Subnet diff --git a/res/res-demo-original/res-demo-stack.yaml b/res/res-demo-original/res-demo-stack.yaml index c091303b..f82e890f 100644 --- a/res/res-demo-original/res-demo-stack.yaml +++ b/res/res-demo-original/res-demo-stack.yaml @@ -35,7 +35,6 @@ Parameters: ClientIpCidr: Description: Default IP(s) allowed to directly access the Web UI, SSH into the bastion host, and access the Windows AD admin host. We recommend that you restrict it with your own IP/subnet (x.x.x.x/32 for your own ip or x.x.x.x/24 for range. Replace x.x.x.x with your own PUBLIC IP. You can get your public IP using tools such as https://ifconfig.co/) - Default: 0.0.0.0/0 Type: String AllowedPattern: (\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2}) ConstraintDescription: Value must be a valid IP or network range of the form x.x.x.x/x. @@ -122,11 +121,9 @@ Resources: LDAPBase: !GetAtt [ RESExternal, Outputs.LDAPBase ] LDAPConnectionURI: !GetAtt [ RESExternal, Outputs.LDAPConnectionURI ] SudoersGroupName: RESAdministrators - ServiceAccountUsername: !GetAtt [ RESExternal, Outputs.ServiceAccountUsername ] - ServiceAccountPasswordSecretArn: !GetAtt [ RESExternal, Outputs.ServiceAccountPasswordSecretArn ] + ServiceAccountCredentialsSecretArn: !GetAtt [ RESExternal, Outputs.ServiceAccountCredentialsSecretArn ] UsersOU: !GetAtt [ RESExternal, Outputs.UsersOU ] GroupsOU: !GetAtt [ RESExternal, Outputs.GroupsOU ] - SudoersOU: !GetAtt [ RESExternal, Outputs.SudoersOU ] ComputersOU: !GetAtt [ RESExternal, Outputs.ComputersOU ] SharedHomeFileSystemId: !GetAtt [ RESExternal, Outputs.SharedHomeFilesystemId ] InfrastructureHostAMI: "" @@ -143,7 +140,7 @@ Resources: Parameters: EnvironmentName: !Ref EnvironmentName Keypair: !Ref Keypair - ServiceAccountPasswordSecretArn: !GetAtt [ RESExternal, Outputs.ServiceAccountPasswordSecretArn ] + ServiceAccountCredentialsSecretArn: !GetAtt [ RESExternal, Outputs.ServiceAccountCredentialsSecretArn ] VpcId: !GetAtt [ RESExternal, Outputs.VpcId ] PublicSubnet: !Select [0, !Split [",", !GetAtt RESExternal.Outputs.PublicSubnets]] ServiceAccountUserDN: !GetAtt [ RESExternal, Outputs.ServiceAccountUserDN ] @@ -239,6 +236,125 @@ Resources: Properties: ServiceToken: !GetAtt InvokeDeleteSharedSecurityGroupHandlerFunction.Arn + RESPostDeploymentConfiguationFunctionRole: + Type: 'AWS::IAM::Role' + DependsOn: RES + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: lambda.amazonaws.com + Action: 'sts:AssumeRole' + Policies: + - PolicyName: LogOutput + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogEvents + Resource: '*' + - PolicyName: DynamoDBReadWritePolicy + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - dynamodb:GetItem + - dynamodb:UpdateItem + Resource: + - !Sub arn:${AWS::Partition}:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${EnvironmentName}.cluster-settings + - !Sub arn:${AWS::Partition}:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${EnvironmentName}.cluster-settings/stream/* + Condition: + ForAllValues:StringLike: + dynamodb:LeadingKeys: + - shared-storage.* + + RESPostDeploymentConfiguationFunction: + Type: 'AWS::Lambda::Function' + DependsOn: + - RES + - RESPostDeploymentConfiguationFunctionRole + Properties: + Description: 'Post configuration of RES for demo purposes' + FunctionName: !Sub ${EnvironmentName}-RESPostDeploymentConfiguationFunction-${AWS::StackName} + Timeout: 60 + Role: !GetAtt RESPostDeploymentConfiguationFunctionRole.Arn + Handler: index.handler + Runtime: python3.11 + Code: + ZipFile: | + import boto3 + import os + import logging + import cfnresponse + + logger = logging.getLogger() + logger.setLevel(logging.INFO) + + def handler(event, context): + logger.info(f"Received event: {event}") + response = {} + + if event["RequestType"] == "Create": + try: + dynamodb = boto3.resource('dynamodb') + cluster_settings_table = dynamodb.Table(f"{os.environ['ENVIRONMENT_NAME']}.cluster-settings") + + demo_config = { + 'shared-storage.enable_file_browser': True + } + + for key, value in demo_config.items(): + item_response = cluster_settings_table.get_item( + Key={ + 'key': key + } + ) + + if 'Item' in item_response: + logger.info(f"Item found: {item_response['Item']}") + + # Update the item + update_response = cluster_settings_table.update_item( + Key={ + 'key': key + }, + UpdateExpression="SET #val = :val", + ExpressionAttributeNames={ + '#val': 'value' + }, + ExpressionAttributeValues={ + ':val': value + }, + ReturnValues="UPDATED_NEW" + ) + + logger.info(f"Item updated: {update_response['Attributes']}") + else: + logger.info(f"Item with key '{key}' not found") + + response['Output'] = 'RES demo environment has been pre-configured.' + cfnresponse.send(event, context, cfnresponse.SUCCESS, response) + except Exception as e: + logger.error(f"Error: Unable to pre-configure RES demo environment: {e}") + response['Output'] = f"Error: Unable to pre-configure RES demo environment: {e}" + cfnresponse.send(event, context, cfnresponse.FAILED, response) + else: + cfnresponse.send(event, context, cfnresponse.SUCCESS, response) + Environment: + Variables: + ENVIRONMENT_NAME: !Ref EnvironmentName + + RESPostDeploymentConfiguation: + Type: Custom::RESPostDeploymentConfiguation + Properties: + ServiceToken: !GetAtt RESPostDeploymentConfiguationFunction.Arn + Outputs: KeycloakUrl: Description: Keycloak Administrator Url diff --git a/res/res-demo-original/res-sso-keycloak.yaml b/res/res-demo-original/res-sso-keycloak.yaml new file mode 100644 index 00000000..feda6671 --- /dev/null +++ b/res/res-demo-original/res-sso-keycloak.yaml @@ -0,0 +1,405 @@ +Description: Research and Engineering Studio SSO setup with Keycloak + +Metadata: + AWS::CloudFormation::Interface: + ParameterGroups: + - Label: + default: RES SSO Configuration + Parameters: + - EnvironmentName + - Keypair + - ServiceAccountCredentialsSecretArn + - VpcId + - PublicSubnet + - ServiceAccountUserDN + - UsersDN + - LDAPConnectionURI + +Parameters: + + EnvironmentName: + Description: Provide name of the RES Environment. Must be unique for your account and AWS Region. + Type: String + Default: res-demo + AllowedPattern: ^res-[A-Za-z\-\_0-9]{0,7}$ + ConstraintDescription: EnvironmentName must start with "res-" and should be less than or equal to 11 characters. + + Keypair: + Description: EC2 Keypair to access management instance. + Type: AWS::EC2::KeyPair::KeyName + + ServiceAccountCredentialsSecretArn: + Type: String + AllowedPattern: ^(?:arn:(?:aws|aws-us-gov|aws-cn):secretsmanager:[a-z0-9-]{1,20}:[0-9]{12}:secret:[A-Za-z0-9-_+=,\.@]{1,128})?$ + Description: Directory Service Service Account Credentials Secret ARN. The username and password for the Active Directory ServiceAccount user formatted as a username:password key/value pair. + + VpcId: + Type: AWS::EC2::VPC::Id + AllowedPattern: vpc-[0-9a-f]{17} + ConstraintDescription: VpcId must begin with 'vpc-', only contain letters (a-f) or numbers(0-9) and must be 21 characters in length + + PublicSubnet: + Type: AWS::EC2::Subnet::Id + AllowedPattern: subnet-.+ + Description: Select a public subnet from the already selected VPC + + ServiceAccountUserDN: + Type: String + AllowedPattern: .+ + Description: Provide the Distinguished name (DN) of the service account user in the Active Directory + + UsersDN: + Type: String + AllowedPattern: .+ + Description: Please provide Users Organization Unit in your active directory under which all of your users exist. For example, OU=Users,DC=RES,DC=example,DC=internal + + LDAPConnectionURI: + Type: String + AllowedPattern: .+ + Description: Please provide the active directory connection URI (e.g. ldap://www.example.com) + +Resources: + + Keycloak: + Type: AWS::CloudFormation::Stack + Properties: + Parameters: + Keypair: !Ref Keypair + ServiceAccountCredentialsSecretArn: !Ref ServiceAccountCredentialsSecretArn + VpcId: !Ref VpcId + PublicSubnet: !Ref PublicSubnet + ServiceAccountUserDN: !Ref ServiceAccountUserDN + UsersDN: !Ref UsersDN + LDAPConnectionURI: !Ref LDAPConnectionURI + CogntioUserPoolId: !Sub ${DataGatherCustomResource.UserPoolId} + EnvironmentBaseURL: !Sub ${DataGatherCustomResource.LoadBalancerDnsName} + SAMLRedirectUrl: !Sub ${DataGatherCustomResource.SAMLRedirectUrl} + TemplateURL: https://aws-hpc-recipes.s3.us-east-1.amazonaws.com/main/recipes/res/res_demo_env/assets/keycloak.yaml + + KeycloakDataGatherLambdaExecutionRole: + Type: 'AWS::IAM::Role' + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: lambda.amazonaws.com + Action: 'sts:AssumeRole' + Policies: + - PolicyName: QueryCognitoAndELBv2 + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - elasticloadbalancing:DescribeLoadBalancers + - elasticloadbalancing:DescribeTags + - cognito-idp:ListUserPools + Resource: '*' + - Effect: Allow + Action: + - cognito-idp:DescribeUserPool + Resource: + - !Sub arn:${AWS::Partition}:cognito-idp:${AWS::Region}:${AWS::AccountId}:userpool/${AWS::Region}* + - Effect: Allow + Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogEvents + Resource: '*' + + KeycloakDataGatherHandlerFunction: + Type: 'AWS::Lambda::Function' + DependsOn: + - KeycloakDataGatherLambdaExecutionRole + Properties: + Description: 'Keycloak Data Gather Handler' + FunctionName: !Sub KeycloakDataGatherHandler-${EnvironmentName} + Timeout: 300 # 5 minutes + Role: !GetAtt KeycloakDataGatherLambdaExecutionRole.Arn + Handler: index.handler + Runtime: python3.11 + Code: + ZipFile: | + import os + import boto3 + import urllib.error + import urllib.parse + import urllib.request + import json + from typing import Any, Dict, TypedDict, Union + from itertools import chain + import boto3 + import botocore.exceptions + import logging + from typing import TypedDict + + logger = logging.getLogger() + logger.setLevel(logging.INFO) + + TAG_NAME = "res:EnvironmentName" + + class CustomResourceResponse(TypedDict): + Status: str + Reason: str + PhysicalResourceId: str + StackId: str + RequestId: str + LogicalResourceId: str + + def send_response(url, response): + request = urllib.request.Request( + method="PUT", + url=url, + data=json.dumps(response).encode("utf-8"), + ) + urllib.request.urlopen(request) + + def get_cognito_data(cluster_name, region_name): + cognito_client = boto3.client("cognito-idp") + logger.info(f"Working on getting Cognito details") + userpool_pagintor = cognito_client.get_paginator("list_user_pools") + userpool_pages = map(lambda p: p.get("UserPools", []), userpool_pagintor.paginate(MaxResults=50)) + userpool_match_fn = lambda up: up.get("Name", "") == f"{cluster_name}-user-pool" and up.get("Id") + userpools = filter(userpool_match_fn, chain.from_iterable(userpool_pages)) + for userpool in userpools: + pool_name = userpool.get("Name") + pool_id = userpool.get("Id") + logger.info(f"Processing cognito pool: {pool_name} PoolId: {pool_id}") + describe_user_pool_result = cognito_client.describe_user_pool( + UserPoolId=pool_id + ) + tags = describe_user_pool_result.get("UserPool", {}).get( + "UserPoolTags", {} + ) + match_fn = lambda tg: tg[0] == TAG_NAME and tg[1] == cluster_name + if next(filter(match_fn, tags.items()), None): + logger.info(f"Found matching tags") + domain = describe_user_pool_result['UserPool']['Domain'] + saml_redirect_url = f'https://{domain}.auth.{region_name}.amazoncognito.com/saml2/idpresponse' + return pool_id, saml_redirect_url + else: + logger.info("No matching tags found") + + def get_alb_dns(cluster_name): + elbv2_client = boto3.client('elbv2') + logger.info(f"Working on getting load balancer DNS") + lb_paginator = elbv2_client.get_paginator("describe_load_balancers") + lb_pages = map(lambda p: p.get("LoadBalancers", []), lb_paginator.paginate()) + lb_match_fn = lambda lb: lb.get("LoadBalancerName", "") == f"{cluster_name}-external-alb" + load_balancers = filter(lb_match_fn, chain.from_iterable(lb_pages)) + for load_balancer in load_balancers: + load_balancer_arn = load_balancer.get("LoadBalancerArn", "") + load_balancer_name = load_balancer.get("LoadBalancerName", "") + logger.info(f"Processing load balancer: {load_balancer_name}") + tag_description = elbv2_client.describe_tags(ResourceArns=[load_balancer_arn]).get('TagDescriptions', [None])[0] + tags = tag_description.get('Tags', []) + match_fn = lambda t: t['Key'] == TAG_NAME and t['Value'] == cluster_name + if next(filter(match_fn, tags), None): + logger.info(f"Found matching tags") + return f'https://{load_balancer["DNSName"]}' + else: + logger.info("No matching tags found") + + def handler(event, _): + logger.info(f"Received event: {event}") + request_type = event["RequestType"] + response_url = event["ResponseURL"] + response = CustomResourceResponse( + Status="SUCCESS", + Reason="SUCCESS", + PhysicalResourceId=event["LogicalResourceId"], + StackId=event["StackId"], + RequestId=event["RequestId"], + LogicalResourceId=event["LogicalResourceId"], + Data={} + ) + if request_type == "Delete": + send_response(response_url, response) + return + + cluster_name = os.environ['CLUSTER_NAME'] + region_name = os.environ['AWS_REGION'] + + try: + user_pool_id, saml_redirect_url = get_cognito_data(cluster_name, region_name) + dns_name = get_alb_dns(cluster_name) + if not user_pool_id or not saml_redirect_url or not dns_name: + raise Exception(f"Unable to find matching cognito user pool, SAML redirect URL for the user pool, or load balancer. Response: {response}") + response["Data"]["UserPoolId"] = user_pool_id + response["Data"]["SAMLRedirectUrl"] = saml_redirect_url + response["Data"]["LoadBalancerDnsName"] = dns_name + except Exception as e: + logger.error(f"Error processing request {e}") + response["Status"] = "FAILED" + response["Reason"] = str(e) + finally: + logger.info(f"Sending response: {response}") + send_response(url=response_url, response=response) + Environment: + Variables: + CLUSTER_NAME: !Ref EnvironmentName + + DataGatherCustomResource: + Type: Custom::KeycloakDataGather + Properties: + ServiceToken: !GetAtt KeycloakDataGatherHandlerFunction.Arn + + InvokeConfigureSSOLambdaRole: + Type: 'AWS::IAM::Role' + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: lambda.amazonaws.com + Action: 'sts:AssumeRole' + Policies: + - PolicyName: InvokeConfigureSSOLambdaPolicy + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - lambda:InvokeFunction + Resource: + - !Sub arn:${AWS::Partition}:lambda:${AWS::Region}:${AWS::AccountId}:function:${EnvironmentName}-configure_sso + - Effect: Allow + Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogEvents + Resource: '*' + + InvokeConfigureSSOHandlerFunction: + Type: 'AWS::Lambda::Function' + DependsOn: + - InvokeConfigureSSOLambdaRole + - Keycloak + Properties: + Description: 'Invoke RES configure sso function' + FunctionName: !Sub InvokeConfigureSSOHandlerFunction-${EnvironmentName} + Timeout: 300 # 5 minutes + Role: !GetAtt InvokeConfigureSSOLambdaRole.Arn + Handler: index.handler + Runtime: python3.11 + Code: + ZipFile: | + import os + import boto3 + import urllib.error + import urllib.parse + import urllib.request + import json + from typing import Any, Dict, TypedDict, Union + + import boto3 + import botocore.exceptions + import base64 + import logging + from typing import TypedDict + + logger = logging.getLogger() + logger.setLevel(logging.INFO) + + class CustomResourceResponse(TypedDict): + Status: str + Reason: str + PhysicalResourceId: str + StackId: str + RequestId: str + LogicalResourceId: str + + def send_response(url, response): + request = urllib.request.Request( + method="PUT", + url=url, + data=json.dumps(response).encode("utf-8"), + ) + urllib.request.urlopen(request) + + def handler(event, _): + logger.info(f"Received event: {event}") + + try: + request_type = event["RequestType"] + response_url = event["ResponseURL"] + response = CustomResourceResponse( + Status="SUCCESS", + Reason="SUCCESS", + PhysicalResourceId=event["LogicalResourceId"], + StackId=event["StackId"], + RequestId=event["RequestId"], + LogicalResourceId=event["LogicalResourceId"], + Data={} + ) + if request_type == "Delete": + send_response(response_url, response) + return + lambda_name = os.environ['LAMBDA_NAME'] + region_name = os.environ['AWS_REGION'] + keycloak_url = os.environ['KEYCLOAK_URL'] + + #Get SAML metadata string from Keycloak + saml_metadata_url = f"{keycloak_url}/realms/res/protocol/saml/descriptor" + logger.info(f"SAML metadata url: {saml_metadata_url}") + + local_filename, headers = urllib.request.urlretrieve(saml_metadata_url) + saml_metadata = open(local_filename, "r").read() + saml_metadata_utf8encoded = saml_metadata.encode("utf-8") + saml_metadata_base64_bytes = base64.b64encode(saml_metadata_utf8encoded) + saml_metadata_base64_string = saml_metadata_base64_bytes.decode("utf-8") + + #Build payload + payload = json.dumps({ + 'configure_sso_request': { + 'provider_name': 'idc', + 'provider_type': 'SAML', + 'provider_email_attribute': 'email', + 'saml_metadata_file': saml_metadata_base64_string + } + }) + + #Invoke Lambda + logger.info(f"Invoking configure_sso lambda with payload : {payload}") + lambda_client = boto3.client("lambda") + lambda_response = lambda_client.invoke( + FunctionName=lambda_name, + Payload=payload + ) + + logger.info(f"Response from configure_sso lambda: lambda_response") + if 'FunctionError' in lambda_response: + response_payload = json.loads(response['Payload'].read()) + if 'errorMessage' in response_payload: + raise Exception(response_payload['errorMessage']) + raise Exception(lambda_response['FunctionError']) + except Exception as e: + logger.error(f"Error processing request {e}") + response["Status"] = "FAILED" + response["Reason"] = str(e) + finally: + logger.info(f"Sending response: {response}") + send_response(url=response_url, response=response) + Environment: + Variables: + LAMBDA_NAME: !Sub ${EnvironmentName}-configure_sso + KEYCLOAK_URL: !GetAtt [ Keycloak, Outputs.KeycloakUrl ] + + InvokeConfigureSSOCustomResource: + Type: Custom::InvokeConfigureSSO + Properties: + ServiceToken: !GetAtt InvokeConfigureSSOHandlerFunction.Arn + +Outputs: + KeycloakUrl: + Description: Keycloak Administrator Url + Value: !GetAtt [ Keycloak, Outputs.KeycloakUrl ] + KeycloakAdminPasswordSecretArn: + Description: Keycloak password for admin user + Value: !GetAtt [ Keycloak, Outputs.KeycloakAdminPasswordSecretArn ] + ApplicationUrl: + Description: RES application Url + Value: !GetAtt DataGatherCustomResource.LoadBalancerDnsName \ No newline at end of file diff --git a/res/res-demo-with-cidr/bi.yaml b/res/res-demo-with-cidr/bi.yaml index 3ac9d78c..ae841efc 100644 --- a/res/res-demo-with-cidr/bi.yaml +++ b/res/res-demo-with-cidr/bi.yaml @@ -3,6 +3,7 @@ Description: A set of external resources that can support a Research and Enginee Metadata: AWS::CloudFormation::Interface: ParameterGroups: + # change - Label: default: "Vpc Configuration" Parameters: @@ -13,6 +14,7 @@ Metadata: - VpcCidrPrivateSubnetA - VpcCidrPrivateSubnetB - VpcCidrPrivateSubnetC + # endchange - Label: default: "AD Configuration" Parameters: @@ -30,6 +32,7 @@ Metadata: - StopAdAdminInstances Parameters: + # change VpcCidrBlock: AllowedPattern: '((\d{1,3})\.){3}\d{1,3}/\d{1,2}' Default: 10.3.0.0/16 @@ -71,6 +74,7 @@ Parameters: Default: 10.3.160.0/20 Description: VPC CIDR Block for the Private Subnet C Type: String + # endchange DomainName: Description: Active Directory Domain Name. The supplied LDIF file which provides bootstrap users uses this domain. A different LDIF file needs to be provided for a different domain. @@ -88,21 +92,22 @@ Parameters: EnvironmentName: Description: (Optional) EnvironmentName must start with "res-"and should be less than or equal to 11 characters. Required to generate certificates. Type: String - AllowedPattern: ^res-[A-Za-z\-\_0-9]{0,7}$ + AllowedPattern: ^$|^res-[A-Za-z\-\_0-9]{0,7}$ + Default: res-demo AdminPassword: Description: Provide the Active Directory Administrator Account Password Directly or Resource ARN to Secret Containing Password. Type: String MinLength: 8 MaxLength: 2048 - AllowedPattern: (arn:(aws(-cn|-us-gov)?):secretsmanager:(us(-gov)?|ap|ca|cn|eu|sa)-(central|(north|south)?(east|west)?)-\d:\d{12}:secret:[a-zA-Z0-9/_+=.@-]+)|(?=^.{8,64}$)((?=.*\d)(?=.*[A-Z])(?=.*[a-z])|(?=.*\d)(?=.*[^A-Za-z0-9\s])(?=.*[a-z])|(?=.*[^A-Za-z0-9\s])(?=.*[A-Z])(?=.*[a-z])|(?=.*\d)(?=.*[A-Z])(?=.*[^A-Za-z0-9\s]))^.* + AllowedPattern: (arn:(aws(-cn|-us-gov)?):secretsmanager:(us(-gov)?|ap|ca|cn|eu|il|sa)-(central|(north|south)?(east|west)?)-\d:\d{12}:secret:[a-zA-Z0-9/_+=.@-]+)|(?=^.{8,64}$)((?=.*\d)(?=.*[A-Z])(?=.*[a-z])|(?=.*\d)(?=.*[^A-Za-z0-9\s])(?=.*[a-z])|(?=.*[^A-Za-z0-9\s])(?=.*[A-Z])(?=.*[a-z])|(?=.*\d)(?=.*[A-Z])(?=.*[^A-Za-z0-9\s]))^.* NoEcho: true ServiceAccountPassword: Description: Provide the Active Directory Service Account Password Directly or Resource ARN to Secret Containing Password. Type: String MinLength: 8 MaxLength: 2048 - AllowedPattern: (arn:(aws(-cn|-us-gov)?):secretsmanager:(us(-gov)?|ap|ca|cn|eu|sa)-(central|(north|south)?(east|west)?)-\d:\d{12}:secret:[a-zA-Z0-9/_+=.@-]+)|(?=^.{8,64}$)((?=.*\d)(?=.*[A-Z])(?=.*[a-z])|(?=.*\d)(?=.*[^A-Za-z0-9\s])(?=.*[a-z])|(?=.*[^A-Za-z0-9\s])(?=.*[A-Z])(?=.*[a-z])|(?=.*\d)(?=.*[A-Z])(?=.*[^A-Za-z0-9\s]))^.* + AllowedPattern: (arn:(aws(-cn|-us-gov)?):secretsmanager:(us(-gov)?|ap|ca|cn|eu|il|sa)-(central|(north|south)?(east|west)?)-\d:\d{12}:secret:[a-zA-Z0-9/_+=.@-]+)|(?=^.{8,64}$)((?=.*\d)(?=.*[A-Z])(?=.*[a-z])|(?=.*\d)(?=.*[^A-Za-z0-9\s])(?=.*[a-z])|(?=.*[^A-Za-z0-9\s])(?=.*[A-Z])(?=.*[a-z])|(?=.*\d)(?=.*[A-Z])(?=.*[^A-Za-z0-9\s]))^.* NoEcho: true LDIFS3Path: Description: (Optional) An S3 Path (without the s3://) to an LDIF file that will be used during stack creation. @@ -161,6 +166,7 @@ Resources: DeletionPolicy: !If [ RetainStorageAndNetworking, Retain, Delete ] Properties: Parameters: + # change CidrBlock: !Ref VpcCidrBlock CidrPublicSubnetA: !Ref VpcCidrPublicSubnetA CidrPublicSubnetB: !Ref VpcCidrPublicSubnetB @@ -168,6 +174,7 @@ Resources: CidrPrivateSubnetA: !Ref VpcCidrPrivateSubnetA CidrPrivateSubnetB: !Ref VpcCidrPrivateSubnetB CidrPrivateSubnetC: !Ref VpcCidrPrivateSubnetC + # endchange ProvisionSubnetsC: "False" TemplateURL: https://aws-hpc-recipes.s3.us-east-1.amazonaws.com/main/recipes/net/hpc_large_scale/assets/main.yaml @@ -541,7 +548,7 @@ Resources: response_data['Message'] = 'Resource creation successful!' physical_resource_id = create_physical_resource_id() - secretsmanager_arn_regex_pattern = r"(arn:(aws(-cn|-us-gov)?):secretsmanager:(us(-gov)?|ap|ca|cn|eu|sa)-(central|(north|south)?(east|west)?)-\d:\d{12}:secret:[a-zA-Z0-9/_+=.@-]+)" + secretsmanager_arn_regex_pattern = r"(arn:(aws(-cn|-us-gov)?):secretsmanager:(us(-gov)?|ap|ca|cn|eu|il|sa)-(central|(north|south)?(east|west)?)-\d:\d{12}:secret:[a-zA-Z0-9/_+=.@-]+)" admin_arn_match = re.search(secretsmanager_arn_regex_pattern, admin_password) service_account_arn_match = re.search(secretsmanager_arn_regex_pattern, service_account_password) @@ -606,10 +613,8 @@ Outputs: Value: !Sub - dc=${dc} - { dc: !Join [",dc=", !Split [".", !If [ SubDomainNotProvided, !Ref DomainName, !Join [ ".", [ !Ref SubDomain, !Ref DomainName] ] ] ]] } - ServiceAccountUsername: - Value: ServiceAccount - ServiceAccountPasswordSecretArn: - Value: !GetAtt [ DirectoryService, Outputs.PasswordSecretArn ] + ServiceAccountCredentialsSecretArn: + Value: !GetAtt [ DirectoryService, Outputs.CredentialsSecretArn ] ServiceAccountUserDN: Description: The Distinguished Name (DN) of the ServiceAccount user in your Active Directory Value: !Sub @@ -627,13 +632,9 @@ Outputs: Value: !Sub - OU=Users,OU=RES,OU=${ou},DC=${dc} - { dc: !Join [",DC=", !Split [".", !If [ SubDomainNotProvided, !Ref DomainName, !Join [ ".", [ !Ref SubDomain, !Ref DomainName]]]]], ou: !GetAtt [ DirectoryService, Outputs.DomainShortName ]} - SudoersOU: - Description: The OU for users who should have sudoers permission across all projects. The value provided here is based off of a supplied LDIF file. - Value: !Sub - - OU=Users,OU=RES,OU=${ou},DC=${dc} - - { dc: !Join [",DC=", !Split [".", !If [ SubDomainNotProvided, !Ref DomainName, !Join [ ".", [ !Ref SubDomain, !Ref DomainName]]]]], ou: !GetAtt [ DirectoryService, Outputs.DomainShortName ]} ComputersOU: Description: The OU for computers that join the AD. The value provided here is based off of a supplied LDIF file. Value: !Sub - OU=Computers,OU=RES,OU=${ou},DC=${dc} - { dc: !Join [",DC=", !Split [".", !If [ SubDomainNotProvided, !Ref DomainName, !Join [ ".", [ !Ref SubDomain, !Ref DomainName]]]]], ou: !GetAtt [ DirectoryService, Outputs.DomainShortName ]} + diff --git a/res/res-demo-with-cidr/keycloak.yaml b/res/res-demo-with-cidr/keycloak.yaml new file mode 100644 index 00000000..e36e2870 --- /dev/null +++ b/res/res-demo-with-cidr/keycloak.yaml @@ -0,0 +1,377 @@ +Description: Keycloak Server + +Metadata: + AWS::CloudFormation::Interface: + ParameterGroups: + - Label: + default: Keycloak Configuration + Parameters: + # change + - InstanceType + # endchange + - Keypair + - ServiceAccountCredentialsSecretArn + - VpcId + - PublicSubnet + - ServiceAccountUserDN + - UsersDN + - LDAPConnectionURI + - CogntioUserPoolId + - EnvironmentBaseURL + - SAMLRedirectUrl + +Parameters: + # change + InstanceType: + Description: EC2 instance type of keycloak server. + Type: String + Default: t3.small + # endchange + + Keypair: + Description: EC2 Keypair to access management instance. + Type: AWS::EC2::KeyPair::KeyName + + ServiceAccountCredentialsSecretArn: + Type: String + AllowedPattern: ^(?:arn:(?:aws|aws-us-gov|aws-cn):secretsmanager:[a-z0-9-]{1,20}:[0-9]{12}:secret:[A-Za-z0-9-_+=,\.@]{1,128})?$ + Description: Directory Service Service Account Credentials Secret ARN. The username and password for the Active Directory ServiceAccount user formatted as a username:password key/value pair. + + VpcId: + Type: AWS::EC2::VPC::Id + AllowedPattern: vpc-[0-9a-f]{17} + ConstraintDescription: VpcId must begin with 'vpc-', only contain letters (a-f) or numbers(0-9) and must be 21 characters in length + + PublicSubnet: + Type: AWS::EC2::Subnet::Id + AllowedPattern: subnet-.+ + Description: Select a public subnet from the already selected VPC + + ServiceAccountUserDN: + Type: String + AllowedPattern: .+ + Description: Provide the Distinguished name (DN) of the service account user in the Active Directory + + UsersDN: + Type: String + AllowedPattern: .+ + Description: Please provide Users Organization Unit in your active directory under which all of your users exist. For example, OU=Users,DC=RES,DC=example,DC=internal + + LDAPConnectionURI: + Type: String + AllowedPattern: .+ + Description: Please provide the active directory connection URI (e.g. ldap://www.example.com) + + CogntioUserPoolId: + Type: String + AllowedPattern: .+ + Description: Please provide the Cognito user pool id (e.g. us-east-1_ababab) + + EnvironmentBaseURL: + Type: String + AllowedPattern: https?://.+ + Description: Please provide your base URL for your environment + + SAMLRedirectUrl: + Type: String + AllowedPattern: https://.+\.amazoncognito\.com/saml2/idpresponse + Description: Please provide the SAML redirect URL + +Mappings: + Keycloak: + Config: + Version: "24.0.3" + + +Resources: + KeycloakSecret: + Type: AWS::SecretsManager::Secret + Properties: + Name: !Sub + - KeycloakSecret-${AWS::StackName}-${StackIdSuffix} + - StackIdSuffix: !Select [2, !Split ['/', !Ref 'AWS::StackId']] + Description: Keycloak secret + GenerateSecretString: + PasswordLength: 14 + ExcludePunctuation: true + + KeycloakEC2InstanceRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - ec2.amazonaws.com + Action: + - sts:AssumeRole + ManagedPolicyArns: + - !Sub arn:${AWS::Partition}:iam::aws:policy/AmazonSSMManagedInstanceCore + Policies: + - PolicyName: KeycloakEC2InstancePolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: secretsmanager:GetSecretValue + Resource: + - !Ref KeycloakSecret + - !Ref ServiceAccountCredentialsSecretArn + - Effect: Allow + Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogEvents + Resource: '*' + + KeycloakEC2InstanceProfile: + Type: AWS::IAM::InstanceProfile + Properties: + Roles: + - !Ref KeycloakEC2InstanceRole + + KeycloakSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + GroupDescription: Keycloak security group + VpcId: !Ref VpcId + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 80 + ToPort: 80 + CidrIp: "0.0.0.0/0" + + KeycloakEC2Instance: + Type: AWS::EC2::Instance + DependsOn: + - KeycloakSecurityGroup + - KeycloakEC2InstanceProfile + - KeycloakSecret + CreationPolicy: + ResourceSignal: + Timeout: PT15M + Properties: + # change +{%- raw %} + # endchange + ImageId: '{{resolve:ssm:/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-6.1-x86_64:75}}' + # change +{%- endraw %} + InstanceType: !Ref InstanceType + # endchange + KeyName: !Ref Keypair + IamInstanceProfile: !Ref KeycloakEC2InstanceProfile + SubnetId: !Ref PublicSubnet + SecurityGroupIds: + - !Ref KeycloakSecurityGroup + Tags: + - Key: Name + Value: !Sub keycloak-${AWS::StackName} + UserData: + Fn::Base64: + Fn::Sub: + - | + #!/bin/sh -x + mkdir -p /root/bootstrap && cd /root/bootstrap + mkdir -p /root/bootstrap/logs/ + exec > /root/bootstrap/logs/userdata.log 2>&1 + + # Create utils.sh script + echo -e "#!/bin/sh + wait_for_server() { + SERVER_URL=\$1 + MAX_ATTEMPTS=\$2 + RETRY_INTERVAL=\$3 + attempt=0 + while [ \$attempt -lt \$MAX_ATTEMPTS ]; do + response=\$(curl -s -o /dev/null -w \"%{http_code}\" \"\$SERVER_URL\") + if [ \"\$response\" == \"200\" ] || [ \"\$response\" == \"302\" ]; then + echo \"Server is up!\" + return 0 + else + echo \"Server is not yet up. Retrying in \$RETRY_INTERVAL seconds...\" + sleep \$RETRY_INTERVAL + ((attempt++)) + fi + done + echo \"Server is not up after \$MAX_ATTEMPTS attempts, exiting...\" + return 1 + } + " > /root/bootstrap/utils.sh + + #Install java17 + MAX_ATTEMPTS=5 + RETRY_INTERVAL=5 + attempt=0 + while [ $attempt -lt $MAX_ATTEMPTS ]; do + #Clean yum cache + sudo yum clean packages + #Install java-17 + sudo yum -y install java-17-amazon-corretto-headless + which java && break + sleep $RETRY_INTERVAL + ((attempt++)) + done + + export KEYCLOAK_VERSION=${KeycloakVersion} + wget https://github.com/keycloak/keycloak/releases/download/$KEYCLOAK_VERSION/keycloak-$KEYCLOAK_VERSION.zip + unzip keycloak-$KEYCLOAK_VERSION.zip + + cd keycloak-$KEYCLOAK_VERSION + + export KC_HTTP_PORT=80 + export KEYCLOAK_ADMIN=admin + set +x + export KEYCLOAK_ADMIN_PASSWORD=$(aws secretsmanager get-secret-value --secret-id ${KeycloakSecret} --query SecretString --region ${AWS::Region} --output text) + set -x + + # Start Keycloak + sudo -E nohup ./bin/kc.sh start-dev --http-port 80 > keycloak.log & + sleep 30 + + SERVER_URL="http://0.0.0.0:80" + MAX_ATTEMPTS=15 + RETRY_INTERVAL=10 + + # Initial setup to wait for the server to be up + . /root/bootstrap/utils.sh + wait_for_server "$SERVER_URL" $MAX_ATTEMPTS $RETRY_INTERVAL + if [ $? -ne 0 ]; then + /opt/aws/bin/cfn-signal -e 1 --stack "${AWS::StackName}" --resource "KeycloakEC2Instance" --region "${AWS::Region}" + sleep 30 + fi + + echo "Keycloak server is up" + # Login to Keycloak + set +x + ./bin/kcadm.sh config credentials --server $SERVER_URL --realm master --user admin --password $KEYCLOAK_ADMIN_PASSWORD + set -x + + # Create realm named 'res' + ./bin/kcadm.sh create realms -s realm=res -s id=res -s enabled=true -o + + # Set sslRequired to NONE + ./bin/kcadm.sh update realms/master -s sslRequired=NONE --server $SERVER_URL + ./bin/kcadm.sh update realms/res -s sslRequired=NONE --server $SERVER_URL + + #Configure Keycloak + #Get ServiceAccount passsword + set +x + serviceAccountPassword=$(aws secretsmanager get-secret-value --secret-id ${ServiceAccountCredentialsSecretArn} --query SecretString --region ${AWS::Region} --output text | jq -r 'to_entries[] | .value') + + #Create storage component to sync from AD + componentId=$(./bin/kcadm.sh create components -s name=ldap -s parentId=res -s providerId=ldap -s providerType=org.keycloak.storage.UserStorageProvider \ + -s 'config.authType=["simple"]' -s "config.bindCredential=[\"$serviceAccountPassword\"]" -s 'config.bindDn=["${ServiceAccountUserDN}"]' \ + -s 'config.connectionUrl=["${LDAPConnectionURI}"]' -s 'config.editMode=["READ_ONLY"]' -s 'config.enabled=["true"]' -s 'config.rdnLDAPAttribute=["cn"]' \ + -s 'config.searchScope=["2"]' -s 'config.usernameLDAPAttribute=["sAMAccountName"]' \ + -s 'config.usersDn=["${UsersDN}"]' -s 'config.uuidLDAPAttribute=["objectGUID"]' \ + -s 'config.vendor=["ad"]' -s 'config.userObjectClasses=["person, organizationalPerson, user"]' -r res -i) + set -x + + # Trigger user sync + ./bin/kcadm.sh create user-storage/$componentId/sync?action=triggerFullSync -r res + + #Create SSO SAML client for SSO + clientId=$(./bin/kcadm.sh create clients -r res -s baseUrl=${EnvironmentBaseURL} \ + -s clientId=urn:amazon:cognito:sp:${CogntioUserPoolId} -s name=saml -s protocol=saml -s 'redirectUris=["*"]' -s rootUrl=${EnvironmentBaseURL} \ + -s 'attributes.saml_name_id_format=email' -s 'attributes."post.logout.redirect.uris"=${EnvironmentBaseURL}' \ + -s 'attributes."saml.client.signature"=false' -s 'attributes."saml.force.post.binding"=true' -s 'attributes."saml.authnstatement"=true' \ + -s 'attributes."saml_assertion_consumer_url_post"=${SAMLRedirectUrl}' \ + -s 'attributes.saml_single_logout_service_url_redirect=${EnvironmentBaseURL}' -i) + + # Create email mapper + ./bin/kcadm.sh create clients/$clientId/protocol-mappers/models -s name=email_mapper -s protocol=saml -s protocolMapper=saml-user-property-mapper \ + -s 'config."attribute.name"=email' -s 'config."attribute.nameformat"=Unspecified' -s 'config."friendly.name"=email_mapper' -s 'config."user.attribute"=email' -r res + + ##Schedule crontabs + #Install crontab on al3 + sudo yum -y install cronie + sudo systemctl enable crond.service + sudo systemctl start crond.service + + #Crontab1 - service account password rotation - script + echo -e "#!/bin/sh -x + exec >> /root/bootstrap/logs/userdata.log 2>&1 + echo Updating service account password + cd /root/bootstrap/keycloak-$KEYCLOAK_VERSION + SERVER_URL=\"http://0.0.0.0:80\" + set +x + kc_admin_password=\$(aws secretsmanager get-secret-value --secret-id ${KeycloakSecret} --query SecretString --region ${AWS::Region} --output text) + serviceAccountPassword=\$(aws secretsmanager get-secret-value --secret-id ${ServiceAccountCredentialsSecretArn} --query SecretString --region ${AWS::Region} --output text | jq -r 'to_entries[] | .value') + ./bin/kcadm.sh config credentials --server \$SERVER_URL --realm master --user admin --password \$kc_admin_password + ./bin/kcadm.sh update components/$componentId -s name=ldap -s parentId=res -s providerId=ldap -s providerType=org.keycloak.storage.UserStorageProvider \\ + -s 'config.authType=[\"simple\"]' -s \"config.bindCredential=[\\\"\$serviceAccountPassword\\\"]\" -s 'config.bindDn=[\"${ServiceAccountUserDN}\"]' \\ + -s 'config.connectionUrl=[\"${LDAPConnectionURI}\"]' -s 'config.editMode=[\"READ_ONLY\"]' -s 'config.enabled=[\"true\"]' -s 'config.rdnLDAPAttribute=[\"cn\"]' \\ + -s 'config.searchScope=[\"2\"]' -s 'config.usernameLDAPAttribute=[\"sAMAccountName\"]' \\ + -s 'config.usersDn=[\"${UsersDN}\"]' -s 'config.uuidLDAPAttribute=[\"objectGUID\"]' \\ + -s 'config.vendor=[\"ad\"]' -s 'config.userObjectClasses=[\"person, organizationalPerson, user\"]' -r res + set -x + ./bin/kcadm.sh create user-storage/$componentId/sync?action=triggerFullSync -r res + " > /root/bootstrap/password_rotation.sh + chmod +x /root/bootstrap/password_rotation.sh + + #Crontab2 - user sync - script + echo -e "#!/bin/sh -x + exec >> /root/bootstrap/logs/userdata.log 2>&1 + echo Syncing users + cd /root/bootstrap/keycloak-$KEYCLOAK_VERSION + SERVER_URL=\"http://0.0.0.0:80\" + set +x + kc_admin_password=\$(aws secretsmanager get-secret-value --secret-id ${KeycloakSecret} --query SecretString --region ${AWS::Region} --output text) + ./bin/kcadm.sh config credentials --server \$SERVER_URL --realm master --user admin --password \$kc_admin_password + set -x + ./bin/kcadm.sh create user-storage/$componentId/sync?action=triggerFullSync -r res + " > /root/bootstrap/user_sync.sh + chmod +x /root/bootstrap/user_sync.sh + + (crontab -l; echo "*/30 * * * * /root/bootstrap/password_rotation.sh") | crontab - + (crontab -l; echo "*/5 * * * * /root/bootstrap/user_sync.sh") | crontab - + + # Monitoring script to restart Keycloak if it goes down + echo -e "#!/bin/sh -x + exec >> /root/bootstrap/logs/userdata.log 2>&1 + . /root/bootstrap/utils.sh + SERVER_URL=\"http://0.0.0.0:80\" + MAX_ATTEMPTS=15 + RETRY_INTERVAL=10 + + while true; do + echo \"Start monitoring keycloak server...\" + response=\$(curl -s -o /dev/null -w \"%{http_code}\" \"\$SERVER_URL\") + if [ \"\$response\" == \"200\" ] || [ \"\$response\" == \"302\" ]; then + echo \"Keycloak server is running.\" + else + # Check for running Keycloak processes and kill them if found + if pgrep -f \"keycloak\" > /dev/null; then + pkill -f \"keycloak\" + echo \"Killed existing Keycloak processes.\" + else + echo \"No Keycloak processes found.\" + fi + echo \"Keycloak server is down. Restarting...\" + + cd /root/bootstrap/keycloak-$KEYCLOAK_VERSION + sudo -E nohup ./bin/kc.sh start-dev --http-port 80 > keycloak.log & + wait_for_server \"\$SERVER_URL\" \$MAX_ATTEMPTS \$RETRY_INTERVAL + fi + sleep 60 + done + " > /root/bootstrap/monitor.sh + chmod +x /root/bootstrap/monitor.sh + + # Start the monitoring script in the background + nohup /root/bootstrap/monitor.sh & + + # Signal stack to continue based on last command output + /opt/aws/bin/cfn-signal -e $? --stack "${AWS::StackName}" --resource "KeycloakEC2Instance" --region "${AWS::Region}" + - KeycloakVersion: !FindInMap [Keycloak, Config, Version] + +Outputs: + KeycloakUrl: + Description: Keycloak administrator URL + Value: !Sub http://${KeycloakEC2Instance.PublicIp}:80 + KeycloakAdminPasswordSecretArn: + Description: Keycloak password for admin user + Value: !Sub ${KeycloakSecret} diff --git a/res/res-demo-with-cidr/res-bi-only.yaml b/res/res-demo-with-cidr/res-bi-only.yaml new file mode 100644 index 00000000..2a4179f5 --- /dev/null +++ b/res/res-demo-with-cidr/res-bi-only.yaml @@ -0,0 +1,513 @@ +Description: Research and Engineering Studio Batteries Included (BI). Can be used by the res-only.yml to create 1 or more RES stacks. + +Metadata: + AWS::CloudFormation::Interface: + ParameterGroups: + - Label: + # change + default: RES shared storage configuration + # endchange + Parameters: + - EnvironmentName + # change + # - AdministratorEmail + # - KeycloakInstanceType + # endchange + - Label: + default: Access Management + Parameters: + - Keypair + - ClientIpCidr + - InboundPrefixList + - Label: + default: AD user and group configuration + Parameters: + - LDIFS3Path + - Label: + default: Optional network configuration + Parameters: + - VpcCidrBlock + - VpcCidrPublicSubnetA + - VpcCidrPublicSubnetB + - VpcCidrPublicSubnetC + - VpcCidrPrivateSubnetA + - VpcCidrPrivateSubnetB + - VpcCidrPrivateSubnetC + +Parameters: + + Keypair: + Description: EC2 Keypair to access management instance. + Type: AWS::EC2::KeyPair::KeyName + Default: "" + + EnvironmentName: + # change + Description: Provide name of the RES Environment for the BI stack. Must be unique for your account and AWS Region. Used to tag the file system. Suggest using the StackName. + # endchange + Type: String + # change + Default: res-bi + # endchange + MinLength: 5 + MaxLength: 11 + AllowedPattern: ^res-[A-Za-z\-\_0-9]{0,7}$ + ConstraintDescription: EnvironmentName must start with "res-" and should be less than or equal to 11 characters. + + # change + # AdministratorEmail: + # Type: String + # AllowedPattern: ^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$ + + # KeycloakInstanceType: + # Type: String + # Default: m7a.medium + # endchange + + ClientIpCidr: + Description: Default IP(s) allowed to directly access the Web UI, SSH into the bastion host, and access the Windows AD admin host. We recommend that you restrict it with your own IP/subnet (x.x.x.x/32 for your own ip or x.x.x.x/24 for range. Replace x.x.x.x with your own PUBLIC IP. You can get your public IP using tools such as https://ifconfig.co/) + Type: String + AllowedPattern: (\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2}) + ConstraintDescription: Value must be a valid IP or network range of the form x.x.x.x/x. + + InboundPrefixList: + Description: (Optional) VPC Prefix List controlling inbound access to Web UI, bastion host, and Windows AD admin host. + Default: "" + Type: String + AllowedPattern: ^(pl-[a-z0-9]{8,20})?$ + ConstraintDescription: Must be a valid VPC Prefix List ID, which begins with `pl-` or be empty. + + LDIFS3Path: + Description: An S3 Path (without the s3://) to an LDIF file that will be used during stack creation. + Type: String + Default: {{ LDIFS3Path }} + + VpcCidrBlock: + AllowedPattern: '((\d{1,3})\.){3}\d{1,3}/\d{1,2}' + Default: 10.3.0.0/16 + Description: VPC CIDR Block (eg 10.3.0.0/16) + Type: String + + VpcCidrPublicSubnetA: + AllowedPattern: '((\d{1,3})\.){3}\d{1,3}/\d{1,2}' + Default: 10.3.0.0/20 + Description: VPC CIDR Block for the Public Subnet A + Type: String + + VpcCidrPublicSubnetB: + AllowedPattern: '((\d{1,3})\.){3}\d{1,3}/\d{1,2}' + Default: 10.3.16.0/20 + Description: VPC CIDR Block for the Public Subnet B + Type: String + + VpcCidrPublicSubnetC: + AllowedPattern: '((\d{1,3})\.){3}\d{1,3}/\d{1,2}' + Default: 10.3.32.0/20 + Description: VPC CIDR Block for the Public Subnet C + Type: String + + VpcCidrPrivateSubnetA: + AllowedPattern: '((\d{1,3})\.){3}\d{1,3}/\d{1,2}' + Default: 10.3.128.0/20 + Description: VPC CIDR Block for the Private Subnet A + Type: String + + VpcCidrPrivateSubnetB: + AllowedPattern: '((\d{1,3})\.){3}\d{1,3}/\d{1,2}' + Default: 10.3.144.0/20 + Description: VPC CIDR Block for the Private Subnet B + Type: String + + VpcCidrPrivateSubnetC: + AllowedPattern: '((\d{1,3})\.){3}\d{1,3}/\d{1,2}' + Default: 10.3.160.0/20 + Description: VPC CIDR Block for the Private Subnet C + Type: String + +Conditions: + UseEnvironmentName: !Not [!Equals [!Ref EnvironmentName, ""]] + +Resources: + + AdminPassword: + Type: AWS::SecretsManager::Secret + Properties: + Description: Active Directory Administrator Account Password. + Name: !Sub [ "res-AdminPassword-${StackName}-${StackId}", {StackName: !Select [1, !Split ['/', !Ref 'AWS::StackId']], StackId: !Select [2, !Split ['/', !Ref 'AWS::StackId']]}] + GenerateSecretString: + SecretStringTemplate: '{"username": "Admin"}' + GenerateStringKey: "password" + ExcludePunctuation: true + Tags: + - Key: res:Deployment + Value: "true" + - Key: res:EnvironmentName + Value: !Ref EnvironmentName + + ServiceAccountPassword: + Type: AWS::SecretsManager::Secret + Properties: + Description: Active Directory Service Account Password. + Name: !Sub [ "res-ServiceAccountPassword-${StackName}-${StackId}", {StackName: !Select [1, !Split ['/', !Ref 'AWS::StackId']], StackId: !Select [2, !Split ['/', !Ref 'AWS::StackId']]}] + GenerateSecretString: + SecretStringTemplate: '{"username": "ServiceAccount"}' + GenerateStringKey: "password" + ExcludePunctuation: true + Tags: + - Key: res:Deployment + Value: "true" + - Key: res:EnvironmentName + Value: !Ref EnvironmentName + + RESExternal: + Type: AWS::CloudFormation::Stack + Properties: + Parameters: + LDIFS3Path : !Ref LDIFS3Path + VpcCidrBlock: !Ref VpcCidrBlock + VpcCidrPublicSubnetA: !Ref VpcCidrPublicSubnetA + VpcCidrPublicSubnetB: !Ref VpcCidrPublicSubnetB + VpcCidrPublicSubnetC: !Ref VpcCidrPublicSubnetC + VpcCidrPrivateSubnetA: !Ref VpcCidrPrivateSubnetA + VpcCidrPrivateSubnetB: !Ref VpcCidrPrivateSubnetB + VpcCidrPrivateSubnetC: !Ref VpcCidrPrivateSubnetC + PortalDomainName: "" + Keypair: !Ref Keypair + EnvironmentName: !If [UseEnvironmentName, !Ref EnvironmentName, ""] + AdminPassword: !Ref AdminPassword + ServiceAccountPassword: !Ref ServiceAccountPassword + ClientIpCidr: !Ref ClientIpCidr + ClientPrefixList: !Ref InboundPrefixList + RetainStorageResources: "False" + #TemplateURL: https://aws-hpc-recipes.s3.us-east-1.amazonaws.com/main/recipes/res/res_demo_env/assets/bi.yaml + TemplateURL: https://{{TemplateBucket}}.s3.amazonaws.com/{{TemplateBaseKey}}/bi.yaml + + # change + # RES: + # Type: AWS::CloudFormation::Stack + # DependsOn: InvokeDeleteSharedStorageSecurityGroup + # Properties: + # Parameters: + # EnvironmentName: !Ref EnvironmentName + # AdministratorEmail: !Ref AdministratorEmail + # SSHKeyPair: !Ref Keypair + # ClientIp: !Ref ClientIpCidr + # ClientPrefixList: !Ref InboundPrefixList + # CustomDomainNameforWebApp: "" + # ACMCertificateARNforWebApp: "" + # CustomDomainNameforVDI: "" + # PrivateKeySecretARNforVDI: "" + # CertificateSecretARNforVDI: "" + # DomainTLSCertificateSecretArn: "" + # VpcId: !GetAtt [ RESExternal, Outputs.VpcId ] + # LoadBalancerSubnets: !GetAtt [ RESExternal, Outputs.PublicSubnets ] + # InfrastructureHostSubnets: !GetAtt [ RESExternal, Outputs.PrivateSubnets ] + # VdiSubnets: !GetAtt [ RESExternal, Outputs.PrivateSubnets ] + # IsLoadBalancerInternetFacing: "true" + # ActiveDirectoryName: !GetAtt [ RESExternal, Outputs.ActiveDirectoryName ] + # ADShortName: !GetAtt [ RESExternal, Outputs.ADShortName ] + # LDAPBase: !GetAtt [ RESExternal, Outputs.LDAPBase ] + # LDAPConnectionURI: !GetAtt [ RESExternal, Outputs.LDAPConnectionURI ] + # SudoersGroupName: RESAdministrators + # ServiceAccountCredentialsSecretArn: !GetAtt [ RESExternal, Outputs.ServiceAccountCredentialsSecretArn ] + # UsersOU: !GetAtt [ RESExternal, Outputs.UsersOU ] + # GroupsOU: !GetAtt [ RESExternal, Outputs.GroupsOU ] + # ComputersOU: !GetAtt [ RESExternal, Outputs.ComputersOU ] + # SharedHomeFileSystemId: !GetAtt [ RESExternal, Outputs.SharedHomeFilesystemId ] + # InfrastructureHostAMI: "" + # EnableLdapIDMapping: "True" + # IAMPermissionBoundary: "" + # DisableADJoin: "False" + # ServiceAccountUserDN: !GetAtt [ RESExternal, Outputs.ServiceAccountUserDN ] + # TemplateURL: https://research-engineering-studio-us-east-1.s3.amazonaws.com/releases/latest/ResearchAndEngineeringStudio.template.json + + # RESSsoKeycloak: + # Type: AWS::CloudFormation::Stack + # DependsOn: RES + # Properties: + # Parameters: + # EnvironmentName: !Ref EnvironmentName + # Keypair: !Ref Keypair + # ServiceAccountCredentialsSecretArn: !GetAtt [ RESExternal, Outputs.ServiceAccountCredentialsSecretArn ] + # VpcId: !GetAtt [ RESExternal, Outputs.VpcId ] + # PublicSubnet: !Select [0, !Split [",", !GetAtt RESExternal.Outputs.PublicSubnets]] + # ServiceAccountUserDN: !GetAtt [ RESExternal, Outputs.ServiceAccountUserDN ] + # UsersDN: !GetAtt [ RESExternal, Outputs.LDAPBase ] + # LDAPConnectionURI: !GetAtt [ RESExternal, Outputs.LDAPConnectionURI ] + # TemplateURL: https://aws-hpc-recipes.s3.us-east-1.amazonaws.com/main/recipes/res/res_demo_env/assets/res-sso-keycloak.yaml + # endchange + + InvokeDeleteSharedStorageSecurityGroupRole: + Type: 'AWS::IAM::Role' + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: lambda.amazonaws.com + Action: 'sts:AssumeRole' + Policies: + - PolicyName: InvokeConfigureSSOLambdaPolicy + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - lambda:InvokeFunction + Resource: + - !Sub arn:${AWS::Partition}:lambda:${AWS::Region}:${AWS::AccountId}:function:${EnvironmentName}-delete_shared_storage_security_group + - Effect: Allow + Action: + - ec2:DescribeSecurityGroups + - ec2:DeleteSecurityGroup + - ec2:DescribeNetworkInterfaces + Resource: '*' + + InvokeDeleteSharedSecurityGroupHandlerFunction: + Type: 'AWS::Lambda::Function' + DependsOn: + - InvokeDeleteSharedStorageSecurityGroupRole + Properties: + Description: 'Deletes the shared storage security group when the stack is deleted.' + FunctionName: !Sub InvokeDeleteSharedSecurityGroupHandlerFunction-${AWS::StackName} + Timeout: 360 # 6 minutes + Role: !GetAtt InvokeDeleteSharedStorageSecurityGroupRole.Arn + Handler: index.handler + Runtime: python3.11 + Code: + ZipFile: | + import boto3 + import os + import logging + import cfnresponse + + logger = logging.getLogger() + logger.setLevel(logging.INFO) + + def handler(event, context): + logger.info(f"Received event: {event}") + response = {} + + if event["RequestType"] == "Delete": + try: + ec2 = boto3.client("ec2") + sgResponse = ec2.describe_security_groups( + Filters=[ + { + 'Name': 'group-name', + 'Values': [ + f"{os.environ['ENVIRONMENT_NAME']}-shared-storage-security-group", + ] + } + ] + ) + + if len(sgResponse['SecurityGroups']) == 0: + response['Output'] = "Shared storage security group not found." + else: + ec2.delete_security_group(GroupId=sgResponse['SecurityGroups'][0]['GroupId']) + response['Output'] = "Shared storage security group deleted." + + cfnresponse.send(event, context, cfnresponse.SUCCESS, response) + except Exception as e: + logger.error(f"Error: Unable to delete shared storage security group: {e}") + response['Output'] = f"Error: Unable to delete shared storage security group: {e}" + cfnresponse.send(event, context, cfnresponse.FAILED, response) + else: + cfnresponse.send(event, context, cfnresponse.SUCCESS, response) + Environment: + Variables: + ENVIRONMENT_NAME: !Ref EnvironmentName + + InvokeDeleteSharedStorageSecurityGroup: + Type: Custom::DeleteSharedStorageSecurityGroup + Properties: + ServiceToken: !GetAtt InvokeDeleteSharedSecurityGroupHandlerFunction.Arn + + # change + # RESPostDeploymentConfiguationFunctionRole: + # Type: 'AWS::IAM::Role' + # DependsOn: RES + # Properties: + # AssumeRolePolicyDocument: + # Version: '2012-10-17' + # Statement: + # - Effect: Allow + # Principal: + # Service: lambda.amazonaws.com + # Action: 'sts:AssumeRole' + # Policies: + # - PolicyName: LogOutput + # PolicyDocument: + # Version: '2012-10-17' + # Statement: + # - Effect: Allow + # Action: + # - logs:CreateLogGroup + # - logs:CreateLogStream + # - logs:PutLogEvents + # Resource: '*' + # - PolicyName: DynamoDBReadWritePolicy + # PolicyDocument: + # Version: '2012-10-17' + # Statement: + # - Effect: Allow + # Action: + # - dynamodb:GetItem + # - dynamodb:UpdateItem + # Resource: + # - !Sub arn:${AWS::Partition}:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${EnvironmentName}.cluster-settings + # - !Sub arn:${AWS::Partition}:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${EnvironmentName}.cluster-settings/stream/* + # Condition: + # ForAllValues:StringLike: + # dynamodb:LeadingKeys: + # - shared-storage.* + + # RESPostDeploymentConfiguationFunction: + # Type: 'AWS::Lambda::Function' + # DependsOn: + # - RES + # - RESPostDeploymentConfiguationFunctionRole + # Properties: + # Description: 'Post configuration of RES for demo purposes' + # FunctionName: !Sub ${EnvironmentName}-RESPostDeploymentConfiguationFunction-${AWS::StackName} + # Timeout: 60 + # Role: !GetAtt RESPostDeploymentConfiguationFunctionRole.Arn + # Handler: index.handler + # Runtime: python3.11 + # Code: + # ZipFile: | + # import boto3 + # import os + # import logging + # import cfnresponse + + # logger = logging.getLogger() + # logger.setLevel(logging.INFO) + + # def handler(event, context): + # logger.info(f"Received event: {event}") + # response = {} + + # if event["RequestType"] == "Create": + # try: + # dynamodb = boto3.resource('dynamodb') + # cluster_settings_table = dynamodb.Table(f"{os.environ['ENVIRONMENT_NAME']}.cluster-settings") + + # demo_config = { + # 'shared-storage.enable_file_browser': True + # } + + # for key, value in demo_config.items(): + # item_response = cluster_settings_table.get_item( + # Key={ + # 'key': key + # } + # ) + + # if 'Item' in item_response: + # logger.info(f"Item found: {item_response['Item']}") + + # # Update the item + # update_response = cluster_settings_table.update_item( + # Key={ + # 'key': key + # }, + # UpdateExpression="SET #val = :val", + # ExpressionAttributeNames={ + # '#val': 'value' + # }, + # ExpressionAttributeValues={ + # ':val': value + # }, + # ReturnValues="UPDATED_NEW" + # ) + + # logger.info(f"Item updated: {update_response['Attributes']}") + # else: + # logger.info(f"Item with key '{key}' not found") + + # response['Output'] = 'RES demo environment has been pre-configured.' + # cfnresponse.send(event, context, cfnresponse.SUCCESS, response) + # except Exception as e: + # logger.error(f"Error: Unable to pre-configure RES demo environment: {e}") + # response['Output'] = f"Error: Unable to pre-configure RES demo environment: {e}" + # cfnresponse.send(event, context, cfnresponse.FAILED, response) + # else: + # cfnresponse.send(event, context, cfnresponse.SUCCESS, response) + # Environment: + # Variables: + # ENVIRONMENT_NAME: !Ref EnvironmentName + + # RESPostDeploymentConfiguation: + # Type: Custom::RESPostDeploymentConfiguation + # Properties: + # ServiceToken: !GetAtt RESPostDeploymentConfiguationFunction.Arn + # endchange + +Outputs: + # change + VpcId: + Description: VPC id + Value: !GetAtt [ RESExternal, Outputs.VpcId ] + Export: + Name: !Sub "${AWS::StackName}-VpcId" + PublicSubnets: + Description: Public subnets + Value: !GetAtt [ RESExternal, Outputs.PublicSubnets ] + Export: + Name: !Sub "${AWS::StackName}-PublicSubnets" + PrivateSubnets: + Description: Private subnets + Value: !GetAtt [ RESExternal, Outputs.PrivateSubnets ] + Export: + Name: !Sub "${AWS::StackName}-PrivateSubnets" + ActiveDirectoryName: + Description: Fully Qualified Domain Name (FQDN) for your Active Directory + Value: !GetAtt [ RESExternal, Outputs.ActiveDirectoryName ] + Export: + Name: !Sub "${AWS::StackName}-ActiveDirectoryName" + ADShortName: + Description: Please provide the short name in Active directory + Value: !GetAtt [ RESExternal, Outputs.ADShortName ] + Export: + Name: !Sub "${AWS::StackName}-ADShortName" + LDAPBase: + Value: !GetAtt [ RESExternal, Outputs.LDAPBase ] + Export: + Name: !Sub "${AWS::StackName}-LDAPBase" + LDAPConnectionURI: + Value: !GetAtt [ RESExternal, Outputs.LDAPConnectionURI ] + Export: + Name: !Sub "${AWS::StackName}-LDAPConnectionURI" + ServiceAccountCredentialsSecretArn: + Value: !GetAtt [ RESExternal, Outputs.ServiceAccountCredentialsSecretArn ] + Export: + Name: !Sub "${AWS::StackName}-ServiceAccountCredentialsSecretArn" + UsersOU: + Description: The OU for all users who might join the system. The value provided here is based off of a supplied LDIF file. + Value: !GetAtt [ RESExternal, Outputs.UsersOU ] + Export: + Name: !Sub "${AWS::StackName}-UsersOU" + GroupsOU: + Description: The OU for groups that users belong to who might join the system. The value provided here is based off of a supplied LDIF file. + Value: !GetAtt [ RESExternal, Outputs.GroupsOU ] + Export: + Name: !Sub "${AWS::StackName}-GroupsOU" + ComputersOU: + Description: The OU for computers that join the AD. The value provided here is based off of a supplied LDIF file. + Value: !GetAtt [ RESExternal, Outputs.ComputersOU ] + Export: + Name: !Sub "${AWS::StackName}-ComputersOU" + SharedHomeFilesystemId: + Value: !GetAtt [ RESExternal, Outputs.SharedHomeFilesystemId ] + Export: + Name: !Sub "${AWS::StackName}-SharedHomeFilesystemId" + ServiceAccountUserDN: + Description: The Distinguished Name (DN) of the ServiceAccount user in your Active Directory + Value: !GetAtt [ RESExternal, Outputs.ServiceAccountUserDN ] + Export: + Name: !Sub "${AWS::StackName}-ServiceAccountUserDN" + # endchange + diff --git a/res/res-demo-with-cidr/res-demo-stack.yaml b/res/res-demo-with-cidr/res-demo-stack.yaml index 1d3b26c1..f03018c9 100644 --- a/res/res-demo-with-cidr/res-demo-stack.yaml +++ b/res/res-demo-with-cidr/res-demo-stack.yaml @@ -8,12 +8,16 @@ Metadata: Parameters: - EnvironmentName - AdministratorEmail + # change + - KeycloakInstanceType + # endchange - Label: default: Access Management Parameters: - Keypair - ClientIpCidr - InboundPrefixList + # change - Label: default: AD user and group configuration Parameters: @@ -28,6 +32,7 @@ Metadata: - VpcCidrPrivateSubnetA - VpcCidrPrivateSubnetB - VpcCidrPrivateSubnetC + # endchange Parameters: @@ -40,8 +45,10 @@ Parameters: Description: Provide name of the RES Environment. Must be unique for your account and AWS Region. Type: String Default: res-demo + # change MinLength: 5 MaxLength: 11 + # endchange AllowedPattern: ^res-[A-Za-z\-\_0-9]{0,7}$ ConstraintDescription: EnvironmentName must start with "res-" and should be less than or equal to 11 characters. @@ -49,9 +56,14 @@ Parameters: Type: String AllowedPattern: ^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$ + # change + KeycloakInstanceType: + Type: String + Default: m7a.medium + # endchange + ClientIpCidr: Description: Default IP(s) allowed to directly access the Web UI, SSH into the bastion host, and access the Windows AD admin host. We recommend that you restrict it with your own IP/subnet (x.x.x.x/32 for your own ip or x.x.x.x/24 for range. Replace x.x.x.x with your own PUBLIC IP. You can get your public IP using tools such as https://ifconfig.co/) - Default: 0.0.0.0/0 Type: String AllowedPattern: (\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2}) ConstraintDescription: Value must be a valid IP or network range of the form x.x.x.x/x. @@ -63,6 +75,7 @@ Parameters: AllowedPattern: ^(pl-[a-z0-9]{8,20})?$ ConstraintDescription: Must be a valid VPC Prefix List ID, which begins with `pl-` or be empty. + # change LDIFS3Path: Description: An S3 Path (without the s3://) to an LDIF file that will be used during stack creation. Type: String @@ -109,6 +122,7 @@ Parameters: Default: 10.3.160.0/20 Description: VPC CIDR Block for the Private Subnet C Type: String + # endchange Conditions: UseEnvironmentName: !Not [!Equals [!Ref EnvironmentName, ""]] @@ -149,6 +163,7 @@ Resources: Type: AWS::CloudFormation::Stack Properties: Parameters: + # change LDIFS3Path : !Ref LDIFS3Path VpcCidrBlock: !Ref VpcCidrBlock VpcCidrPublicSubnetA: !Ref VpcCidrPublicSubnetA @@ -157,6 +172,7 @@ Resources: VpcCidrPrivateSubnetA: !Ref VpcCidrPrivateSubnetA VpcCidrPrivateSubnetB: !Ref VpcCidrPrivateSubnetB VpcCidrPrivateSubnetC: !Ref VpcCidrPrivateSubnetC + # endchange PortalDomainName: "" Keypair: !Ref Keypair EnvironmentName: !If [UseEnvironmentName, !Ref EnvironmentName, ""] @@ -165,8 +181,10 @@ Resources: ClientIpCidr: !Ref ClientIpCidr ClientPrefixList: !Ref InboundPrefixList RetainStorageResources: "False" + # change #TemplateURL: https://aws-hpc-recipes.s3.us-east-1.amazonaws.com/main/recipes/res/res_demo_env/assets/bi.yaml TemplateURL: https://{{TemplateBucket}}.s3.amazonaws.com/{{TemplateBaseKey}}/bi.yaml + # endchange RES: Type: AWS::CloudFormation::Stack @@ -194,11 +212,9 @@ Resources: LDAPBase: !GetAtt [ RESExternal, Outputs.LDAPBase ] LDAPConnectionURI: !GetAtt [ RESExternal, Outputs.LDAPConnectionURI ] SudoersGroupName: RESAdministrators - ServiceAccountUsername: !GetAtt [ RESExternal, Outputs.ServiceAccountUsername ] - ServiceAccountPasswordSecretArn: !GetAtt [ RESExternal, Outputs.ServiceAccountPasswordSecretArn ] + ServiceAccountCredentialsSecretArn: !GetAtt [ RESExternal, Outputs.ServiceAccountCredentialsSecretArn ] UsersOU: !GetAtt [ RESExternal, Outputs.UsersOU ] GroupsOU: !GetAtt [ RESExternal, Outputs.GroupsOU ] - SudoersOU: !GetAtt [ RESExternal, Outputs.SudoersOU ] ComputersOU: !GetAtt [ RESExternal, Outputs.ComputersOU ] SharedHomeFileSystemId: !GetAtt [ RESExternal, Outputs.SharedHomeFilesystemId ] InfrastructureHostAMI: "" @@ -213,15 +229,21 @@ Resources: DependsOn: RES Properties: Parameters: + # change + InstanceType: !Ref KeycloakInstanceType + # endchange EnvironmentName: !Ref EnvironmentName Keypair: !Ref Keypair - ServiceAccountPasswordSecretArn: !GetAtt [ RESExternal, Outputs.ServiceAccountPasswordSecretArn ] + ServiceAccountCredentialsSecretArn: !GetAtt [ RESExternal, Outputs.ServiceAccountCredentialsSecretArn ] VpcId: !GetAtt [ RESExternal, Outputs.VpcId ] PublicSubnet: !Select [0, !Split [",", !GetAtt RESExternal.Outputs.PublicSubnets]] ServiceAccountUserDN: !GetAtt [ RESExternal, Outputs.ServiceAccountUserDN ] UsersDN: !GetAtt [ RESExternal, Outputs.LDAPBase ] LDAPConnectionURI: !GetAtt [ RESExternal, Outputs.LDAPConnectionURI ] - TemplateURL: https://aws-hpc-recipes.s3.us-east-1.amazonaws.com/main/recipes/res/res_demo_env/assets/res-sso-keycloak.yaml + # change + #TemplateURL: https://aws-hpc-recipes.s3.us-east-1.amazonaws.com/main/recipes/res/res_demo_env/assets/res-sso-keycloak.yaml + TemplateURL: https://{{TemplateBucket}}.s3.amazonaws.com/{{TemplateBaseKey}}/res-sso-keycloak.yaml + # endchange InvokeDeleteSharedStorageSecurityGroupRole: Type: 'AWS::IAM::Role' @@ -311,6 +333,139 @@ Resources: Properties: ServiceToken: !GetAtt InvokeDeleteSharedSecurityGroupHandlerFunction.Arn + # change + RESPostDeploymentConfigurationFunctionRole: + # endchange + Type: 'AWS::IAM::Role' + DependsOn: RES + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: lambda.amazonaws.com + Action: 'sts:AssumeRole' + Policies: + - PolicyName: LogOutput + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogEvents + Resource: '*' + - PolicyName: DynamoDBReadWritePolicy + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - dynamodb:GetItem + - dynamodb:UpdateItem + Resource: + - !Sub arn:${AWS::Partition}:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${EnvironmentName}.cluster-settings + - !Sub arn:${AWS::Partition}:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${EnvironmentName}.cluster-settings/stream/* + Condition: + ForAllValues:StringLike: + dynamodb:LeadingKeys: + - shared-storage.* + + # change + RESPostDeploymentConfigurationFunction: + # endchange + Type: 'AWS::Lambda::Function' + DependsOn: + - RES + # change + - RESPostDeploymentConfigurationFunctionRole + # endchange + Properties: + Description: 'Post configuration of RES for demo purposes' + # change + FunctionName: !Sub ${EnvironmentName}-RESPostDeploymentConfigurationFunction-${AWS::StackName} + # endchange + Timeout: 60 + # change + Role: !GetAtt RESPostDeploymentConfigurationFunctionRole.Arn + # endchange + Handler: index.handler + Runtime: python3.11 + Code: + ZipFile: | + import boto3 + import os + import logging + import cfnresponse + + logger = logging.getLogger() + logger.setLevel(logging.INFO) + + def handler(event, context): + logger.info(f"Received event: {event}") + response = {} + + if event["RequestType"] == "Create": + try: + dynamodb = boto3.resource('dynamodb') + cluster_settings_table = dynamodb.Table(f"{os.environ['ENVIRONMENT_NAME']}.cluster-settings") + + demo_config = { + 'shared-storage.enable_file_browser': True + } + + for key, value in demo_config.items(): + item_response = cluster_settings_table.get_item( + Key={ + 'key': key + } + ) + + if 'Item' in item_response: + logger.info(f"Item found: {item_response['Item']}") + + # Update the item + update_response = cluster_settings_table.update_item( + Key={ + 'key': key + }, + UpdateExpression="SET #val = :val", + ExpressionAttributeNames={ + '#val': 'value' + }, + ExpressionAttributeValues={ + ':val': value + }, + ReturnValues="UPDATED_NEW" + ) + + logger.info(f"Item updated: {update_response['Attributes']}") + else: + logger.info(f"Item with key '{key}' not found") + + response['Output'] = 'RES demo environment has been pre-configured.' + cfnresponse.send(event, context, cfnresponse.SUCCESS, response) + except Exception as e: + logger.error(f"Error: Unable to pre-configure RES demo environment: {e}") + response['Output'] = f"Error: Unable to pre-configure RES demo environment: {e}" + cfnresponse.send(event, context, cfnresponse.FAILED, response) + else: + cfnresponse.send(event, context, cfnresponse.SUCCESS, response) + Environment: + Variables: + ENVIRONMENT_NAME: !Ref EnvironmentName + + # change + RESPostDeploymentConfiguration: + Type: Custom::RESPostDeploymentConfiguration + # endchange + Properties: + # change + ServiceToken: !GetAtt RESPostDeploymentConfigurationFunction.Arn + # endchange + Outputs: KeycloakUrl: Description: Keycloak Administrator Url diff --git a/res/res-demo-with-cidr/res-only.yaml b/res/res-demo-with-cidr/res-only.yaml new file mode 100644 index 00000000..b6a509ba --- /dev/null +++ b/res/res-demo-with-cidr/res-only.yaml @@ -0,0 +1,417 @@ +Description: Research and Engineering Studio on AWS environment. Requires the res-bi-only.yaml stack to have already been deployed. + +Metadata: + AWS::CloudFormation::Interface: + ParameterGroups: + - Label: + default: RES Configuration + Parameters: + - EnvironmentName + - AdministratorEmail + - KeycloakInstanceType + - Label: + default: Access Management + Parameters: + - Keypair + - ClientIpCidr + - InboundPrefixList + - Label: + # change + default: AD user and group and network configuration + # endchange + Parameters: + # change + - BiStackName + # endchange + +Parameters: + + Keypair: + Description: EC2 Keypair to access management instance. + Type: AWS::EC2::KeyPair::KeyName + Default: "" + + EnvironmentName: + Description: Provide name of the RES Environment. Must be unique for your account and AWS Region. + Type: String + Default: res-demo + MinLength: 5 + MaxLength: 11 + AllowedPattern: ^res-[A-Za-z\-\_0-9]{0,7}$ + ConstraintDescription: EnvironmentName must start with "res-" and should be less than or equal to 11 characters. + + AdministratorEmail: + Type: String + AllowedPattern: ^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$ + + KeycloakInstanceType: + Type: String + Default: m7a.medium + + ClientIpCidr: + Description: Default IP(s) allowed to directly access the Web UI, SSH into the bastion host, and access the Windows AD admin host. We recommend that you restrict it with your own IP/subnet (x.x.x.x/32 for your own ip or x.x.x.x/24 for range. Replace x.x.x.x with your own PUBLIC IP. You can get your public IP using tools such as https://ifconfig.co/) + Type: String + AllowedPattern: (\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2}) + ConstraintDescription: Value must be a valid IP or network range of the form x.x.x.x/x. + + InboundPrefixList: + Description: (Optional) VPC Prefix List controlling inbound access to Web UI, bastion host, and Windows AD admin host. + Default: "" + Type: String + AllowedPattern: ^(pl-[a-z0-9]{8,20})?$ + ConstraintDescription: Must be a valid VPC Prefix List ID, which begins with `pl-` or be empty. + + # change + # LDIFS3Path: + # Description: An S3 Path (without the s3://) to an LDIF file that will be used during stack creation. + # Type: String + # Default: {{LDIFS3Path}} + + BiStackName: + Description: RES Batteries included (BI) stack name + Type: String + # endchange + +Resources: + + # change + # AdminPassword: + # Type: AWS::SecretsManager::Secret + # Properties: + # Description: Active Directory Administrator Account Password. + # Name: !Sub [ "res-AdminPassword-${StackName}-${StackId}", {StackName: !Select [1, !Split ['/', !Ref 'AWS::StackId']], StackId: !Select [2, !Split ['/', !Ref 'AWS::StackId']]}] + # GenerateSecretString: + # SecretStringTemplate: '{"username": "Admin"}' + # GenerateStringKey: "password" + # ExcludePunctuation: true + # Tags: + # - Key: res:Deployment + # Value: "true" + # - Key: res:EnvironmentName + # Value: !Ref EnvironmentName + + # ServiceAccountPassword: + # Type: AWS::SecretsManager::Secret + # Properties: + # Description: Active Directory Service Account Password. + # Name: !Sub [ "res-ServiceAccountPassword-${StackName}-${StackId}", {StackName: !Select [1, !Split ['/', !Ref 'AWS::StackId']], StackId: !Select [2, !Split ['/', !Ref 'AWS::StackId']]}] + # GenerateSecretString: + # SecretStringTemplate: '{"username": "ServiceAccount"}' + # GenerateStringKey: "password" + # ExcludePunctuation: true + # Tags: + # - Key: res:Deployment + # Value: "true" + # - Key: res:EnvironmentName + # Value: !Ref EnvironmentName + + # RESExternal: + # Type: AWS::CloudFormation::Stack + # Properties: + # Parameters: + # LDIFS3Path : !Ref LDIFS3Path + # VpcCidrBlock: !Ref VpcCidrBlock + # VpcCidrPublicSubnetA: !Ref VpcCidrPublicSubnetA + # VpcCidrPublicSubnetB: !Ref VpcCidrPublicSubnetB + # VpcCidrPublicSubnetC: !Ref VpcCidrPublicSubnetC + # VpcCidrPrivateSubnetA: !Ref VpcCidrPrivateSubnetA + # VpcCidrPrivateSubnetB: !Ref VpcCidrPrivateSubnetB + # VpcCidrPrivateSubnetC: !Ref VpcCidrPrivateSubnetC + # PortalDomainName: "" + # Keypair: !Ref Keypair + # EnvironmentName: !If [UseEnvironmentName, !Ref EnvironmentName, ""] + # AdminPassword: !Ref AdminPassword + # ServiceAccountPassword: !Ref ServiceAccountPassword + # ClientIpCidr: !Ref ClientIpCidr + # ClientPrefixList: !Ref InboundPrefixList + # RetainStorageResources: "False" + # #TemplateURL: https://aws-hpc-recipes.s3.us-east-1.amazonaws.com/main/recipes/res/res_demo_env/assets/bi.yaml + # TemplateURL: https://{{TemplateBucket}}.s3.amazonaws.com/{{TemplateBaseKey}}/bi.yaml + # endchange + + RES: + Type: AWS::CloudFormation::Stack + # change + # DependsOn: InvokeDeleteSharedStorageSecurityGroup + # endchange + Properties: + Parameters: + EnvironmentName: !Ref EnvironmentName + AdministratorEmail: !Ref AdministratorEmail + SSHKeyPair: !Ref Keypair + ClientIp: !Ref ClientIpCidr + ClientPrefixList: !Ref InboundPrefixList + CustomDomainNameforWebApp: "" + ACMCertificateARNforWebApp: "" + CustomDomainNameforVDI: "" + PrivateKeySecretARNforVDI: "" + CertificateSecretARNforVDI: "" + DomainTLSCertificateSecretArn: "" + # change + VpcId: {"Fn::ImportValue": !Sub "${BiStackName}-VpcId"} + LoadBalancerSubnets: {"Fn::ImportValue": !Sub "${BiStackName}-PublicSubnets"} + InfrastructureHostSubnets: {"Fn::ImportValue": !Sub "${BiStackName}-PrivateSubnets"} + VdiSubnets: {"Fn::ImportValue": !Sub "${BiStackName}-PrivateSubnets"} + # endchange + IsLoadBalancerInternetFacing: "true" + # change + ActiveDirectoryName: {"Fn::ImportValue": !Sub "${BiStackName}-ActiveDirectoryName"} + ADShortName: {"Fn::ImportValue": !Sub "${BiStackName}-ADShortName"} + LDAPBase: {"Fn::ImportValue": !Sub "${BiStackName}-LDAPBase"} + LDAPConnectionURI: {"Fn::ImportValue": !Sub "${BiStackName}-LDAPConnectionURI"} + # endchange + SudoersGroupName: RESAdministrators + # change + ServiceAccountCredentialsSecretArn: {"Fn::ImportValue": !Sub "${BiStackName}-ServiceAccountCredentialsSecretArn"} + UsersOU: {"Fn::ImportValue": !Sub "${BiStackName}-UsersOU"} + GroupsOU: {"Fn::ImportValue": !Sub "${BiStackName}-GroupsOU"} + ComputersOU: {"Fn::ImportValue": !Sub "${BiStackName}-ComputersOU"} + SharedHomeFileSystemId: {"Fn::ImportValue": !Sub "${BiStackName}-SharedHomeFilesystemId"} + # endchange + InfrastructureHostAMI: "" + EnableLdapIDMapping: "True" + IAMPermissionBoundary: "" + DisableADJoin: "False" + # change + ServiceAccountUserDN: {"Fn::ImportValue": !Sub "${BiStackName}-ServiceAccountUserDN"} + # endchange + TemplateURL: https://research-engineering-studio-us-east-1.s3.amazonaws.com/releases/latest/ResearchAndEngineeringStudio.template.json + + RESSsoKeycloak: + Type: AWS::CloudFormation::Stack + DependsOn: RES + Properties: + Parameters: + InstanceType: !Ref KeycloakInstanceType + EnvironmentName: !Ref EnvironmentName + Keypair: !Ref Keypair + # change + ServiceAccountCredentialsSecretArn: {"Fn::ImportValue": !Sub "${BiStackName}-ServiceAccountCredentialsSecretArn"} + VpcId: {"Fn::ImportValue": !Sub "${BiStackName}-VpcId"} + PublicSubnet: !Select [0, !Split [",", {"Fn::ImportValue": !Sub "${BiStackName}-PublicSubnets"}]] + ServiceAccountUserDN: {"Fn::ImportValue": !Sub "${BiStackName}-ServiceAccountUserDN"} + UsersDN: {"Fn::ImportValue": !Sub "${BiStackName}-LDAPBase"} + LDAPConnectionURI: {"Fn::ImportValue": !Sub "${BiStackName}-LDAPConnectionURI"} + # endchange + #TemplateURL: https://aws-hpc-recipes.s3.us-east-1.amazonaws.com/main/recipes/res/res_demo_env/assets/res-sso-keycloak.yaml + TemplateURL: https://{{TemplateBucket}}.s3.amazonaws.com/{{TemplateBaseKey}}/res-sso-keycloak.yaml + + # change + # InvokeDeleteSharedStorageSecurityGroupRole: + # Type: 'AWS::IAM::Role' + # Properties: + # AssumeRolePolicyDocument: + # Version: '2012-10-17' + # Statement: + # - Effect: Allow + # Principal: + # Service: lambda.amazonaws.com + # Action: 'sts:AssumeRole' + # Policies: + # - PolicyName: InvokeConfigureSSOLambdaPolicy + # PolicyDocument: + # Version: '2012-10-17' + # Statement: + # - Effect: Allow + # Action: + # - lambda:InvokeFunction + # Resource: + # - !Sub arn:${AWS::Partition}:lambda:${AWS::Region}:${AWS::AccountId}:function:${EnvironmentName}-delete_shared_storage_security_group + # - Effect: Allow + # Action: + # - ec2:DescribeSecurityGroups + # - ec2:DeleteSecurityGroup + # - ec2:DescribeNetworkInterfaces + # Resource: '*' + + # InvokeDeleteSharedSecurityGroupHandlerFunction: + # Type: 'AWS::Lambda::Function' + # DependsOn: + # - InvokeDeleteSharedStorageSecurityGroupRole + # Properties: + # Description: 'Deletes the shared storage security group when the stack is deleted.' + # FunctionName: !Sub InvokeDeleteSharedSecurityGroupHandlerFunction-${AWS::StackName} + # Timeout: 360 # 6 minutes + # Role: !GetAtt InvokeDeleteSharedStorageSecurityGroupRole.Arn + # Handler: index.handler + # Runtime: python3.11 + # Code: + # ZipFile: | + # import boto3 + # import os + # import logging + # import cfnresponse + + # logger = logging.getLogger() + # logger.setLevel(logging.INFO) + + # def handler(event, context): + # logger.info(f"Received event: {event}") + # response = {} + + # if event["RequestType"] == "Delete": + # try: + # ec2 = boto3.client("ec2") + # sgResponse = ec2.describe_security_groups( + # Filters=[ + # { + # 'Name': 'group-name', + # 'Values': [ + # f"{os.environ['ENVIRONMENT_NAME']}-shared-storage-security-group", + # ] + # } + # ] + # ) + + # if len(sgResponse['SecurityGroups']) == 0: + # response['Output'] = "Shared storage security group not found." + # else: + # ec2.delete_security_group(GroupId=sgResponse['SecurityGroups'][0]['GroupId']) + # response['Output'] = "Shared storage security group deleted." + + # cfnresponse.send(event, context, cfnresponse.SUCCESS, response) + # except Exception as e: + # logger.error(f"Error: Unable to delete shared storage security group: {e}") + # response['Output'] = f"Error: Unable to delete shared storage security group: {e}" + # cfnresponse.send(event, context, cfnresponse.FAILED, response) + # else: + # cfnresponse.send(event, context, cfnresponse.SUCCESS, response) + # Environment: + # Variables: + # ENVIRONMENT_NAME: !Ref EnvironmentName + + # InvokeDeleteSharedStorageSecurityGroup: + # Type: Custom::DeleteSharedStorageSecurityGroup + # Properties: + # ServiceToken: !GetAtt InvokeDeleteSharedSecurityGroupHandlerFunction.Arn + # endchange + + RESPostDeploymentConfigurationFunctionRole: + Type: 'AWS::IAM::Role' + DependsOn: RES + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: lambda.amazonaws.com + Action: 'sts:AssumeRole' + Policies: + - PolicyName: LogOutput + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogEvents + Resource: '*' + - PolicyName: DynamoDBReadWritePolicy + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - dynamodb:GetItem + - dynamodb:UpdateItem + Resource: + - !Sub arn:${AWS::Partition}:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${EnvironmentName}.cluster-settings + - !Sub arn:${AWS::Partition}:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${EnvironmentName}.cluster-settings/stream/* + Condition: + ForAllValues:StringLike: + dynamodb:LeadingKeys: + - shared-storage.* + + RESPostDeploymentConfigurationFunction: + Type: 'AWS::Lambda::Function' + DependsOn: + - RES + - RESPostDeploymentConfigurationFunctionRole + Properties: + Description: 'Post configuration of RES for demo purposes' + FunctionName: !Sub ${EnvironmentName}-RESPostDeploymentConfigurationFunction-${AWS::StackName} + Timeout: 60 + Role: !GetAtt RESPostDeploymentConfigurationFunctionRole.Arn + Handler: index.handler + Runtime: python3.11 + Code: + ZipFile: | + import boto3 + import os + import logging + import cfnresponse + + logger = logging.getLogger() + logger.setLevel(logging.INFO) + + def handler(event, context): + logger.info(f"Received event: {event}") + response = {} + + if event["RequestType"] == "Create": + try: + dynamodb = boto3.resource('dynamodb') + cluster_settings_table = dynamodb.Table(f"{os.environ['ENVIRONMENT_NAME']}.cluster-settings") + + demo_config = { + 'shared-storage.enable_file_browser': True + } + + for key, value in demo_config.items(): + item_response = cluster_settings_table.get_item( + Key={ + 'key': key + } + ) + + if 'Item' in item_response: + logger.info(f"Item found: {item_response['Item']}") + + # Update the item + update_response = cluster_settings_table.update_item( + Key={ + 'key': key + }, + UpdateExpression="SET #val = :val", + ExpressionAttributeNames={ + '#val': 'value' + }, + ExpressionAttributeValues={ + ':val': value + }, + ReturnValues="UPDATED_NEW" + ) + + logger.info(f"Item updated: {update_response['Attributes']}") + else: + logger.info(f"Item with key '{key}' not found") + + response['Output'] = 'RES demo environment has been pre-configured.' + cfnresponse.send(event, context, cfnresponse.SUCCESS, response) + except Exception as e: + logger.error(f"Error: Unable to pre-configure RES demo environment: {e}") + response['Output'] = f"Error: Unable to pre-configure RES demo environment: {e}" + cfnresponse.send(event, context, cfnresponse.FAILED, response) + else: + cfnresponse.send(event, context, cfnresponse.SUCCESS, response) + Environment: + Variables: + ENVIRONMENT_NAME: !Ref EnvironmentName + + RESPostDeploymentConfiguration: + Type: Custom::RESPostDeploymentConfiguration + Properties: + ServiceToken: !GetAtt RESPostDeploymentConfigurationFunction.Arn + +Outputs: + KeycloakUrl: + Description: Keycloak Administrator Url + Value: !GetAtt [ RESSsoKeycloak, Outputs.KeycloakUrl ] + KeycloakAdminPasswordSecretArn: + Description: Keycloak password for admin user + Value: !GetAtt [ RESSsoKeycloak, Outputs.KeycloakAdminPasswordSecretArn ] + ApplicationUrl: + Description: RES application Url + Value: !GetAtt [ RESSsoKeycloak, Outputs.ApplicationUrl ] diff --git a/res/res-demo-with-cidr/res-sso-keycloak.yaml b/res/res-demo-with-cidr/res-sso-keycloak.yaml new file mode 100644 index 00000000..7f3bae8d --- /dev/null +++ b/res/res-demo-with-cidr/res-sso-keycloak.yaml @@ -0,0 +1,420 @@ +Description: Research and Engineering Studio SSO setup with Keycloak + +Metadata: + AWS::CloudFormation::Interface: + ParameterGroups: + - Label: + default: RES SSO Configuration + Parameters: + # change + - InstanceType + # endchange + - EnvironmentName + - Keypair + - ServiceAccountCredentialsSecretArn + - VpcId + - PublicSubnet + - ServiceAccountUserDN + - UsersDN + - LDAPConnectionURI + +Parameters: + # change + InstanceType: + Description: EC2 instance type of keycloak server. + Type: String + Default: t3.small + # endchange + + EnvironmentName: + Description: Provide name of the RES Environment. Must be unique for your account and AWS Region. + Type: String + Default: res-demo + AllowedPattern: ^res-[A-Za-z\-\_0-9]{0,7}$ + ConstraintDescription: EnvironmentName must start with "res-" and should be less than or equal to 11 characters. + + Keypair: + Description: EC2 Keypair to access management instance. + Type: AWS::EC2::KeyPair::KeyName + + ServiceAccountCredentialsSecretArn: + Type: String + AllowedPattern: ^(?:arn:(?:aws|aws-us-gov|aws-cn):secretsmanager:[a-z0-9-]{1,20}:[0-9]{12}:secret:[A-Za-z0-9-_+=,\.@]{1,128})?$ + Description: Directory Service Service Account Credentials Secret ARN. The username and password for the Active Directory ServiceAccount user formatted as a username:password key/value pair. + + VpcId: + Type: AWS::EC2::VPC::Id + AllowedPattern: vpc-[0-9a-f]{17} + ConstraintDescription: VpcId must begin with 'vpc-', only contain letters (a-f) or numbers(0-9) and must be 21 characters in length + + PublicSubnet: + Type: AWS::EC2::Subnet::Id + AllowedPattern: subnet-.+ + Description: Select a public subnet from the already selected VPC + + ServiceAccountUserDN: + Type: String + AllowedPattern: .+ + Description: Provide the Distinguished name (DN) of the service account user in the Active Directory + + UsersDN: + Type: String + AllowedPattern: .+ + Description: Please provide Users Organization Unit in your active directory under which all of your users exist. For example, OU=Users,DC=RES,DC=example,DC=internal + + LDAPConnectionURI: + Type: String + AllowedPattern: .+ + Description: Please provide the active directory connection URI (e.g. ldap://www.example.com) + +Resources: + + Keycloak: + Type: AWS::CloudFormation::Stack + Properties: + Parameters: + # change + InstanceType: !Ref InstanceType + # endchange + Keypair: !Ref Keypair + ServiceAccountCredentialsSecretArn: !Ref ServiceAccountCredentialsSecretArn + VpcId: !Ref VpcId + PublicSubnet: !Ref PublicSubnet + ServiceAccountUserDN: !Ref ServiceAccountUserDN + UsersDN: !Ref UsersDN + LDAPConnectionURI: !Ref LDAPConnectionURI + CogntioUserPoolId: !Sub ${DataGatherCustomResource.UserPoolId} + EnvironmentBaseURL: !Sub ${DataGatherCustomResource.LoadBalancerDnsName} + SAMLRedirectUrl: !Sub ${DataGatherCustomResource.SAMLRedirectUrl} + # change + #TemplateURL: https://aws-hpc-recipes.s3.us-east-1.amazonaws.com/main/recipes/res/res_demo_env/assets/keycloak.yaml + TemplateURL: https://{{TemplateBucket}}.s3.amazonaws.com/{{TemplateBaseKey}}/keycloak.yaml + # endchange + + KeycloakDataGatherLambdaExecutionRole: + Type: 'AWS::IAM::Role' + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: lambda.amazonaws.com + Action: 'sts:AssumeRole' + Policies: + - PolicyName: QueryCognitoAndELBv2 + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - elasticloadbalancing:DescribeLoadBalancers + - elasticloadbalancing:DescribeTags + - cognito-idp:ListUserPools + Resource: '*' + - Effect: Allow + Action: + - cognito-idp:DescribeUserPool + Resource: + - !Sub arn:${AWS::Partition}:cognito-idp:${AWS::Region}:${AWS::AccountId}:userpool/${AWS::Region}* + - Effect: Allow + Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogEvents + Resource: '*' + + KeycloakDataGatherHandlerFunction: + Type: 'AWS::Lambda::Function' + DependsOn: + - KeycloakDataGatherLambdaExecutionRole + Properties: + Description: 'Keycloak Data Gather Handler' + FunctionName: !Sub KeycloakDataGatherHandler-${EnvironmentName} + Timeout: 300 # 5 minutes + Role: !GetAtt KeycloakDataGatherLambdaExecutionRole.Arn + Handler: index.handler + Runtime: python3.11 + Code: + ZipFile: | + import os + import boto3 + import urllib.error + import urllib.parse + import urllib.request + import json + from typing import Any, Dict, TypedDict, Union + from itertools import chain + import boto3 + import botocore.exceptions + import logging + from typing import TypedDict + + logger = logging.getLogger() + logger.setLevel(logging.INFO) + + TAG_NAME = "res:EnvironmentName" + + class CustomResourceResponse(TypedDict): + Status: str + Reason: str + PhysicalResourceId: str + StackId: str + RequestId: str + LogicalResourceId: str + + def send_response(url, response): + request = urllib.request.Request( + method="PUT", + url=url, + data=json.dumps(response).encode("utf-8"), + ) + urllib.request.urlopen(request) + + def get_cognito_data(cluster_name, region_name): + cognito_client = boto3.client("cognito-idp") + logger.info(f"Working on getting Cognito details") + userpool_pagintor = cognito_client.get_paginator("list_user_pools") + userpool_pages = map(lambda p: p.get("UserPools", []), userpool_pagintor.paginate(MaxResults=50)) + userpool_match_fn = lambda up: up.get("Name", "") == f"{cluster_name}-user-pool" and up.get("Id") + userpools = filter(userpool_match_fn, chain.from_iterable(userpool_pages)) + for userpool in userpools: + pool_name = userpool.get("Name") + pool_id = userpool.get("Id") + logger.info(f"Processing cognito pool: {pool_name} PoolId: {pool_id}") + describe_user_pool_result = cognito_client.describe_user_pool( + UserPoolId=pool_id + ) + tags = describe_user_pool_result.get("UserPool", {}).get( + "UserPoolTags", {} + ) + match_fn = lambda tg: tg[0] == TAG_NAME and tg[1] == cluster_name + if next(filter(match_fn, tags.items()), None): + logger.info(f"Found matching tags") + domain = describe_user_pool_result['UserPool']['Domain'] + saml_redirect_url = f'https://{domain}.auth.{region_name}.amazoncognito.com/saml2/idpresponse' + return pool_id, saml_redirect_url + else: + logger.info("No matching tags found") + + def get_alb_dns(cluster_name): + elbv2_client = boto3.client('elbv2') + logger.info(f"Working on getting load balancer DNS") + lb_paginator = elbv2_client.get_paginator("describe_load_balancers") + lb_pages = map(lambda p: p.get("LoadBalancers", []), lb_paginator.paginate()) + lb_match_fn = lambda lb: lb.get("LoadBalancerName", "") == f"{cluster_name}-external-alb" + load_balancers = filter(lb_match_fn, chain.from_iterable(lb_pages)) + for load_balancer in load_balancers: + load_balancer_arn = load_balancer.get("LoadBalancerArn", "") + load_balancer_name = load_balancer.get("LoadBalancerName", "") + logger.info(f"Processing load balancer: {load_balancer_name}") + tag_description = elbv2_client.describe_tags(ResourceArns=[load_balancer_arn]).get('TagDescriptions', [None])[0] + tags = tag_description.get('Tags', []) + match_fn = lambda t: t['Key'] == TAG_NAME and t['Value'] == cluster_name + if next(filter(match_fn, tags), None): + logger.info(f"Found matching tags") + return f'https://{load_balancer["DNSName"]}' + else: + logger.info("No matching tags found") + + def handler(event, _): + logger.info(f"Received event: {event}") + request_type = event["RequestType"] + response_url = event["ResponseURL"] + response = CustomResourceResponse( + Status="SUCCESS", + Reason="SUCCESS", + PhysicalResourceId=event["LogicalResourceId"], + StackId=event["StackId"], + RequestId=event["RequestId"], + LogicalResourceId=event["LogicalResourceId"], + Data={} + ) + if request_type == "Delete": + send_response(response_url, response) + return + + cluster_name = os.environ['CLUSTER_NAME'] + region_name = os.environ['AWS_REGION'] + + try: + user_pool_id, saml_redirect_url = get_cognito_data(cluster_name, region_name) + dns_name = get_alb_dns(cluster_name) + if not user_pool_id or not saml_redirect_url or not dns_name: + raise Exception(f"Unable to find matching cognito user pool, SAML redirect URL for the user pool, or load balancer. Response: {response}") + response["Data"]["UserPoolId"] = user_pool_id + response["Data"]["SAMLRedirectUrl"] = saml_redirect_url + response["Data"]["LoadBalancerDnsName"] = dns_name + except Exception as e: + logger.error(f"Error processing request {e}") + response["Status"] = "FAILED" + response["Reason"] = str(e) + finally: + logger.info(f"Sending response: {response}") + send_response(url=response_url, response=response) + Environment: + Variables: + CLUSTER_NAME: !Ref EnvironmentName + + DataGatherCustomResource: + Type: Custom::KeycloakDataGather + Properties: + ServiceToken: !GetAtt KeycloakDataGatherHandlerFunction.Arn + + InvokeConfigureSSOLambdaRole: + Type: 'AWS::IAM::Role' + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: lambda.amazonaws.com + Action: 'sts:AssumeRole' + Policies: + - PolicyName: InvokeConfigureSSOLambdaPolicy + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - lambda:InvokeFunction + Resource: + - !Sub arn:${AWS::Partition}:lambda:${AWS::Region}:${AWS::AccountId}:function:${EnvironmentName}-configure_sso + - Effect: Allow + Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogEvents + Resource: '*' + + InvokeConfigureSSOHandlerFunction: + Type: 'AWS::Lambda::Function' + DependsOn: + - InvokeConfigureSSOLambdaRole + - Keycloak + Properties: + Description: 'Invoke RES configure sso function' + FunctionName: !Sub InvokeConfigureSSOHandlerFunction-${EnvironmentName} + Timeout: 300 # 5 minutes + Role: !GetAtt InvokeConfigureSSOLambdaRole.Arn + Handler: index.handler + Runtime: python3.11 + Code: + ZipFile: | + import os + import boto3 + import urllib.error + import urllib.parse + import urllib.request + import json + from typing import Any, Dict, TypedDict, Union + + import boto3 + import botocore.exceptions + import base64 + import logging + from typing import TypedDict + + logger = logging.getLogger() + logger.setLevel(logging.INFO) + + class CustomResourceResponse(TypedDict): + Status: str + Reason: str + PhysicalResourceId: str + StackId: str + RequestId: str + LogicalResourceId: str + + def send_response(url, response): + request = urllib.request.Request( + method="PUT", + url=url, + data=json.dumps(response).encode("utf-8"), + ) + urllib.request.urlopen(request) + + def handler(event, _): + logger.info(f"Received event: {event}") + + try: + request_type = event["RequestType"] + response_url = event["ResponseURL"] + response = CustomResourceResponse( + Status="SUCCESS", + Reason="SUCCESS", + PhysicalResourceId=event["LogicalResourceId"], + StackId=event["StackId"], + RequestId=event["RequestId"], + LogicalResourceId=event["LogicalResourceId"], + Data={} + ) + if request_type == "Delete": + send_response(response_url, response) + return + lambda_name = os.environ['LAMBDA_NAME'] + region_name = os.environ['AWS_REGION'] + keycloak_url = os.environ['KEYCLOAK_URL'] + + #Get SAML metadata string from Keycloak + saml_metadata_url = f"{keycloak_url}/realms/res/protocol/saml/descriptor" + logger.info(f"SAML metadata url: {saml_metadata_url}") + + local_filename, headers = urllib.request.urlretrieve(saml_metadata_url) + saml_metadata = open(local_filename, "r").read() + saml_metadata_utf8encoded = saml_metadata.encode("utf-8") + saml_metadata_base64_bytes = base64.b64encode(saml_metadata_utf8encoded) + saml_metadata_base64_string = saml_metadata_base64_bytes.decode("utf-8") + + #Build payload + payload = json.dumps({ + 'configure_sso_request': { + 'provider_name': 'idc', + 'provider_type': 'SAML', + 'provider_email_attribute': 'email', + 'saml_metadata_file': saml_metadata_base64_string + } + }) + + #Invoke Lambda + logger.info(f"Invoking configure_sso lambda with payload : {payload}") + lambda_client = boto3.client("lambda") + lambda_response = lambda_client.invoke( + FunctionName=lambda_name, + Payload=payload + ) + + logger.info(f"Response from configure_sso lambda: lambda_response") + if 'FunctionError' in lambda_response: + response_payload = json.loads(response['Payload'].read()) + if 'errorMessage' in response_payload: + raise Exception(response_payload['errorMessage']) + raise Exception(lambda_response['FunctionError']) + except Exception as e: + logger.error(f"Error processing request {e}") + response["Status"] = "FAILED" + response["Reason"] = str(e) + finally: + logger.info(f"Sending response: {response}") + send_response(url=response_url, response=response) + Environment: + Variables: + LAMBDA_NAME: !Sub ${EnvironmentName}-configure_sso + KEYCLOAK_URL: !GetAtt [ Keycloak, Outputs.KeycloakUrl ] + + InvokeConfigureSSOCustomResource: + Type: Custom::InvokeConfigureSSO + Properties: + ServiceToken: !GetAtt InvokeConfigureSSOHandlerFunction.Arn + +Outputs: + KeycloakUrl: + Description: Keycloak Administrator Url + Value: !GetAtt [ Keycloak, Outputs.KeycloakUrl ] + KeycloakAdminPasswordSecretArn: + Description: Keycloak password for admin user + Value: !GetAtt [ Keycloak, Outputs.KeycloakAdminPasswordSecretArn ] + ApplicationUrl: + Description: RES application Url + Value: !GetAtt DataGatherCustomResource.LoadBalancerDnsName diff --git a/res/upload-res-templates.py b/res/upload-res-templates.py index b9a9f4ef..011d66ca 100755 --- a/res/upload-res-templates.py +++ b/res/upload-res-templates.py @@ -31,7 +31,7 @@ import os import os.path from os.path import dirname, realpath -from shutil import copy +from shutil import copy, rmtree import sys script_path = os.path.dirname(os.path.abspath(__file__)) @@ -74,10 +74,17 @@ def main(self): src_dir = 'res-demo-with-cidr' dst_dir = 'rendered_templates' template_files = [ + 'bi.yaml', + 'keycloak.yaml', + 'res-bi-only.yaml', 'res-demo-stack.yaml', - 'bi.yaml' + 'res-only.yaml', + 'res-sso-keycloak.yaml', ] - os.makedirs(dst_dir, exist_ok=True) + if os.path.exists(dst_dir): + rmtree(dst_dir) + os.makedirs(dst_dir) + for template_file in template_files: src_file = f"{src_dir}/{template_file}" dst_file = f"{dst_dir}/{template_file}" @@ -99,8 +106,14 @@ def main(self): Body = open(local_file, 'r').read() ) + logger.info("") + logger.info("Use the following link to deploy RES BI using CloudFormation.") + logger.info(f"https://console.aws.amazon.com/cloudformation/home?region={args.region}#/stacks/quickcreate?templateURL=https://{args.s3_bucket}.s3.amazonaws.com/{args.s3_base_key}/res-bi-only.yaml") logger.info("") logger.info("Use the following link to deploy RES using CloudFormation.") + logger.info(f"https://console.aws.amazon.com/cloudformation/home?region={args.region}#/stacks/quickcreate?templateURL=https://{args.s3_bucket}.s3.amazonaws.com/{args.s3_base_key}/res-only.yaml") + logger.info("") + logger.info("Use the following link to deploy RES BI + RES using CloudFormation.") logger.info(f"https://console.aws.amazon.com/cloudformation/home?region={args.region}#/stacks/quickcreate?templateURL=https://{args.s3_bucket}.s3.amazonaws.com/{args.s3_base_key}/res-demo-stack.yaml") if __name__ == "__main__":