Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(stepfunctions-tasks): EMR createCluster command support OnDemandSpecification #27791

Merged
merged 30 commits into from
Dec 21, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
1a7a905
feat(stepfunctions-tasks): EMR createCluster command support OnDemand…
go-to-k Nov 1, 2023
97c9711
change required to optional
go-to-k Nov 1, 2023
5fd3528
change tests and rendering
go-to-k Nov 1, 2023
18d0ab7
unit test
go-to-k Nov 1, 2023
e3f4e40
wip in integ test
go-to-k Nov 1, 2023
b200d86
test: add integ tests
go-to-k Nov 1, 2023
e235589
chore: add validation for length of capacityReservationResourceGroupArn
go-to-k Nov 1, 2023
56ada60
chore: tweak
go-to-k Nov 1, 2023
904f7bf
style: new line
go-to-k Nov 1, 2023
8777b3c
chore: rendering props
go-to-k Nov 1, 2023
f558357
change integ tests
go-to-k Nov 1, 2023
5df65e7
test: change integ tests
go-to-k Nov 1, 2023
c0caee8
chore: change params and tests
go-to-k Nov 2, 2023
a3f29d2
chore: rm cache
go-to-k Nov 2, 2023
b716028
chore: rm integs
go-to-k Nov 2, 2023
4d5bd43
chore: rm a line
go-to-k Nov 2, 2023
76dd650
docs: change README
go-to-k Nov 2, 2023
31a1726
docs: fix codes in README
go-to-k Nov 2, 2023
c392bcf
docs: fix codes in README
go-to-k Nov 2, 2023
6510ab0
chore: allocationStrategy for on-demand to required
go-to-k Nov 2, 2023
18c92eb
fix: by the review
go-to-k Nov 2, 2023
7c660c8
Merge branch 'main' of https://github.com/go-to-k/aws-cdk into sfn-ta…
go-to-k Dec 6, 2023
dbe7c5d
Merge branch 'main' of https://github.com/go-to-k/aws-cdk into sfn-ta…
go-to-k Dec 6, 2023
40dcabc
use new module for apigatewayv2
go-to-k Dec 6, 2023
956ecb1
Merge branch 'main' into sfn-task-emr-ondemand
go-to-k Dec 6, 2023
4177095
Merge branch 'main' into sfn-task-emr-ondemand
kaizencc Dec 18, 2023
7c5a98e
modify by review
go-to-k Dec 19, 2023
68ca386
Merge branch 'main' into sfn-task-emr-ondemand
go-to-k Dec 19, 2023
45bd660
change snapshots
go-to-k Dec 19, 2023
8f34e1e
Merge branch 'main' into sfn-task-emr-ondemand
mergify[bot] Dec 21, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
@@ -0,0 +1,275 @@
{
"Resources": {
"EmrCreateClusterServiceRole5251910D": {
"Type": "AWS::IAM::Role",
"Properties": {
"AssumeRolePolicyDocument": {
"Statement": [
{
"Action": "sts:AssumeRole",
"Effect": "Allow",
"Principal": {
"Service": "elasticmapreduce.amazonaws.com"
}
}
],
"Version": "2012-10-17"
},
"ManagedPolicyArns": [
{
"Fn::Join": [
"",
[
"arn:",
{
"Ref": "AWS::Partition"
},
":iam::aws:policy/service-role/AmazonEMRServicePolicy_v2"
]
]
}
]
}
},
"EmrCreateClusterServiceRoleDefaultPolicyA8B4FA32": {
"Type": "AWS::IAM::Policy",
"Properties": {
"PolicyDocument": {
"Statement": [
{
"Action": "iam:PassRole",
"Effect": "Allow",
"Resource": {
"Fn::GetAtt": [
"EmrCreateClusterInstanceRoleC80466F5",
"Arn"
]
}
}
],
"Version": "2012-10-17"
},
"PolicyName": "EmrCreateClusterServiceRoleDefaultPolicyA8B4FA32",
"Roles": [
{
"Ref": "EmrCreateClusterServiceRole5251910D"
}
]
}
},
"EmrCreateClusterInstanceRoleC80466F5": {
"Type": "AWS::IAM::Role",
"Properties": {
"AssumeRolePolicyDocument": {
"Statement": [
{
"Action": "sts:AssumeRole",
"Effect": "Allow",
"Principal": {
"Service": "ec2.amazonaws.com"
}
}
],
"Version": "2012-10-17"
}
}
},
"EmrCreateClusterInstanceProfileC1729180": {
"Type": "AWS::IAM::InstanceProfile",
"Properties": {
"InstanceProfileName": {
"Ref": "EmrCreateClusterInstanceRoleC80466F5"
},
"Roles": [
{
"Ref": "EmrCreateClusterInstanceRoleC80466F5"
}
]
}
},
"SMRole49C19C48": {
"Type": "AWS::IAM::Role",
"Properties": {
"AssumeRolePolicyDocument": {
"Statement": [
{
"Action": "sts:AssumeRole",
"Effect": "Allow",
"Principal": {
"Service": "states.amazonaws.com"
}
}
],
"Version": "2012-10-17"
}
}
},
"SMRoleDefaultPolicy34CA15C7": {
"Type": "AWS::IAM::Policy",
"Properties": {
"PolicyDocument": {
"Statement": [
{
"Action": [
"elasticmapreduce:AddTags",
"elasticmapreduce:DescribeCluster",
"elasticmapreduce:RunJobFlow",
"elasticmapreduce:TerminateJobFlows"
],
"Effect": "Allow",
"Resource": "*"
},
{
"Action": "iam:PassRole",
"Effect": "Allow",
"Resource": [
{
"Fn::GetAtt": [
"EmrCreateClusterInstanceRoleC80466F5",
"Arn"
]
},
{
"Fn::GetAtt": [
"EmrCreateClusterServiceRole5251910D",
"Arn"
]
}
]
},
{
"Action": "elasticmapreduce:AddTags",
"Effect": "Allow",
"Resource": {
"Fn::Join": [
"",
[
"arn:",
{
"Ref": "AWS::Partition"
},
":elasticmapreduce:",
{
"Ref": "AWS::Region"
},
":",
{
"Ref": "AWS::AccountId"
},
":cluster/*"
]
]
}
},
{
"Action": [
"events:DescribeRule",
"events:PutRule",
"events:PutTargets"
],
"Effect": "Allow",
"Resource": {
"Fn::Join": [
"",
[
"arn:",
{
"Ref": "AWS::Partition"
},
":events:",
{
"Ref": "AWS::Region"
},
":",
{
"Ref": "AWS::AccountId"
},
":rule/StepFunctionsGetEventForEMRRunJobFlowRule"
]
]
}
}
],
"Version": "2012-10-17"
},
"PolicyName": "SMRoleDefaultPolicy34CA15C7",
"Roles": [
{
"Ref": "SMRole49C19C48"
}
]
}
},
"SM934E715A": {
"Type": "AWS::StepFunctions::StateMachine",
"Properties": {
"DefinitionString": {
"Fn::Join": [
"",
[
"{\"StartAt\":\"EmrCreateCluster\",\"States\":{\"EmrCreateCluster\":{\"End\":true,\"Type\":\"Task\",\"Resource\":\"arn:",
{
"Ref": "AWS::Partition"
},
":states:::elasticmapreduce:createCluster.sync\",\"Parameters\":{\"Instances\":{\"InstanceFleets\":[{\"InstanceFleetType\":\"MASTER\",\"InstanceTypeConfigs\":[{\"InstanceType\":\"m5.xlarge\",\"WeightedCapacity\":1}],\"LaunchSpecifications\":{\"OnDemandSpecification\":{\"AllocationStrategy\":\"lowest-price\"}},\"Name\":\"Master\",\"TargetOnDemandCapacity\":1}],\"KeepJobFlowAliveWhenNoSteps\":true},\"JobFlowRole\":\"",
{
"Ref": "EmrCreateClusterInstanceRoleC80466F5"
},
"\",\"Name\":\"Cluster\",\"ServiceRole\":\"",
{
"Ref": "EmrCreateClusterServiceRole5251910D"
},
"\",\"ReleaseLabel\":\"emr-5.36.1\",\"Tags\":[{\"Key\":\"Key\",\"Value\":\"Value\"},{\"Key\":\"for-use-with-amazon-emr-managed-policies\",\"Value\":\"true\"}],\"VisibleToAllUsers\":true}}}}"
]
]
},
"RoleArn": {
"Fn::GetAtt": [
"SMRole49C19C48",
"Arn"
]
}
},
"DependsOn": [
"SMRoleDefaultPolicy34CA15C7",
"SMRole49C19C48"
],
"UpdateReplacePolicy": "Delete",
"DeletionPolicy": "Delete"
}
},
"Parameters": {
"BootstrapVersion": {
"Type": "AWS::SSM::Parameter::Value<String>",
"Default": "/cdk-bootstrap/hnb659fds/version",
"Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store. [cdk:skip]"
}
},
"Rules": {
"CheckBootstrapVersion": {
"Assertions": [
{
"Assert": {
"Fn::Not": [
{
"Fn::Contains": [
[
"1",
"2",
"3",
"4",
"5"
],
{
"Ref": "BootstrapVersion"
}
]
}
]
},
"AssertDescription": "CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI."
}
]
}
}
}

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading