-
Notifications
You must be signed in to change notification settings - Fork 4k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat(stepfunctions): distributed map construct #28821
Changes from 1 commit
9592e28
701160d
7aab8a6
5780a74
f48f39e
3331868
ec0c1f7
17f53fc
c848a83
422ed73
171dd4d
7ec4ff6
b62c7a3
169c6dd
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Large diffs are not rendered by default.
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,338 @@ | ||
{ | ||
"Resources": { | ||
"Bucket83908E77": { | ||
"Type": "AWS::S3::Bucket", | ||
"Properties": { | ||
"Tags": [ | ||
{ | ||
"Key": "aws-cdk:auto-delete-objects", | ||
"Value": "true" | ||
} | ||
] | ||
}, | ||
"UpdateReplacePolicy": "Delete", | ||
"DeletionPolicy": "Delete" | ||
}, | ||
"BucketPolicyE9A3008A": { | ||
"Type": "AWS::S3::BucketPolicy", | ||
"Properties": { | ||
"Bucket": { | ||
"Ref": "Bucket83908E77" | ||
}, | ||
"PolicyDocument": { | ||
"Statement": [ | ||
{ | ||
"Action": [ | ||
"s3:DeleteObject*", | ||
"s3:GetBucket*", | ||
"s3:List*", | ||
"s3:PutBucketPolicy" | ||
], | ||
"Effect": "Allow", | ||
"Principal": { | ||
"AWS": { | ||
"Fn::GetAtt": [ | ||
"CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092", | ||
"Arn" | ||
] | ||
} | ||
}, | ||
"Resource": [ | ||
{ | ||
"Fn::GetAtt": [ | ||
"Bucket83908E77", | ||
"Arn" | ||
] | ||
}, | ||
{ | ||
"Fn::Join": [ | ||
"", | ||
[ | ||
{ | ||
"Fn::GetAtt": [ | ||
"Bucket83908E77", | ||
"Arn" | ||
] | ||
}, | ||
"/*" | ||
] | ||
] | ||
} | ||
] | ||
} | ||
], | ||
"Version": "2012-10-17" | ||
} | ||
} | ||
}, | ||
"BucketAutoDeleteObjectsCustomResourceBAFD23C2": { | ||
"Type": "Custom::S3AutoDeleteObjects", | ||
"Properties": { | ||
"ServiceToken": { | ||
"Fn::GetAtt": [ | ||
"CustomS3AutoDeleteObjectsCustomResourceProviderHandler9D90184F", | ||
"Arn" | ||
] | ||
}, | ||
"BucketName": { | ||
"Ref": "Bucket83908E77" | ||
} | ||
}, | ||
"DependsOn": [ | ||
"BucketPolicyE9A3008A" | ||
], | ||
"UpdateReplacePolicy": "Delete", | ||
"DeletionPolicy": "Delete" | ||
}, | ||
"CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092": { | ||
"Type": "AWS::IAM::Role", | ||
"Properties": { | ||
"AssumeRolePolicyDocument": { | ||
"Version": "2012-10-17", | ||
"Statement": [ | ||
{ | ||
"Action": "sts:AssumeRole", | ||
"Effect": "Allow", | ||
"Principal": { | ||
"Service": "lambda.amazonaws.com" | ||
} | ||
} | ||
] | ||
}, | ||
"ManagedPolicyArns": [ | ||
{ | ||
"Fn::Sub": "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" | ||
} | ||
] | ||
} | ||
}, | ||
"CustomS3AutoDeleteObjectsCustomResourceProviderHandler9D90184F": { | ||
"Type": "AWS::Lambda::Function", | ||
"Properties": { | ||
"Code": { | ||
"S3Bucket": { | ||
"Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" | ||
}, | ||
"S3Key": "2ec8ad9e91dcd6e7ad6a5c84ffc6c9c05c408aca3b26ceb2816d81043e6c4dc3.zip" | ||
}, | ||
"Timeout": 900, | ||
"MemorySize": 128, | ||
"Handler": "index.handler", | ||
"Role": { | ||
"Fn::GetAtt": [ | ||
"CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092", | ||
"Arn" | ||
] | ||
}, | ||
"Runtime": "nodejs18.x", | ||
"Description": { | ||
"Fn::Join": [ | ||
"", | ||
[ | ||
"Lambda function for auto-deleting objects in ", | ||
{ | ||
"Ref": "Bucket83908E77" | ||
}, | ||
" S3 bucket." | ||
] | ||
] | ||
} | ||
}, | ||
"DependsOn": [ | ||
"CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092" | ||
] | ||
}, | ||
"StateMachineRoleB840431D": { | ||
"Type": "AWS::IAM::Role", | ||
"Properties": { | ||
"AssumeRolePolicyDocument": { | ||
"Statement": [ | ||
{ | ||
"Action": "sts:AssumeRole", | ||
"Effect": "Allow", | ||
"Principal": { | ||
"Service": "states.amazonaws.com" | ||
} | ||
} | ||
], | ||
"Version": "2012-10-17" | ||
} | ||
} | ||
}, | ||
"StateMachineRoleDefaultPolicyDF1E6607": { | ||
"Type": "AWS::IAM::Policy", | ||
"Properties": { | ||
"PolicyDocument": { | ||
"Statement": [ | ||
{ | ||
"Action": [ | ||
"s3:AbortMultipartUpload", | ||
"s3:GetObject", | ||
"s3:ListMultipartUploadParts", | ||
"s3:PutObject" | ||
], | ||
"Effect": "Allow", | ||
"Resource": { | ||
"Fn::Join": [ | ||
"", | ||
[ | ||
"arn:", | ||
{ | ||
"Ref": "AWS::Partition" | ||
}, | ||
":s3:::", | ||
{ | ||
"Ref": "Bucket83908E77" | ||
}, | ||
"/*" | ||
] | ||
] | ||
} | ||
} | ||
], | ||
"Version": "2012-10-17" | ||
}, | ||
"PolicyName": "StateMachineRoleDefaultPolicyDF1E6607", | ||
"Roles": [ | ||
{ | ||
"Ref": "StateMachineRoleB840431D" | ||
} | ||
] | ||
} | ||
}, | ||
"StateMachine2E01A3A5": { | ||
"Type": "AWS::StepFunctions::StateMachine", | ||
"Properties": { | ||
"DefinitionString": { | ||
"Fn::Join": [ | ||
"", | ||
[ | ||
"{\"StartAt\":\"DistributedMap\",\"States\":{\"DistributedMap\":{\"Type\":\"Map\",\"End\":true,\"ItemProcessor\":{\"ProcessorConfig\":{\"Mode\":\"DISTRIBUTED\",\"ExecutionType\":\"STANDARD\"},\"StartAt\":\"Pass\",\"States\":{\"Pass\":{\"Type\":\"Pass\",\"End\":true}}},\"ItemReader\":{\"Resource\":\"arn:", | ||
{ | ||
"Ref": "AWS::Partition" | ||
}, | ||
":states:::s3:getObject\",\"ReaderConfig\":{\"InputType\":\"CSV\",\"CSVHeaderLocation\":\"FIRST_ROW\"},\"Parameters\":{\"Bucket\":\"", | ||
{ | ||
"Ref": "Bucket83908E77" | ||
}, | ||
"\",\"Key\":\"my-key.csv\"}},\"ResultWriter\":{\"Resource\":\"arn:", | ||
{ | ||
"Ref": "AWS::Partition" | ||
}, | ||
":states:::s3:putObject\",\"Parameters\":{\"Bucket\":\"", | ||
{ | ||
"Ref": "Bucket83908E77" | ||
}, | ||
"\",\"Prefix\":\"my-prefix\"}}}}}" | ||
] | ||
] | ||
}, | ||
"RoleArn": { | ||
"Fn::GetAtt": [ | ||
"StateMachineRoleB840431D", | ||
"Arn" | ||
] | ||
} | ||
}, | ||
"DependsOn": [ | ||
"StateMachineRoleDefaultPolicyDF1E6607", | ||
"StateMachineRoleB840431D" | ||
], | ||
"UpdateReplacePolicy": "Delete", | ||
"DeletionPolicy": "Delete" | ||
}, | ||
"StateMachineDistributedMapPolicy57C9D8C2": { | ||
"Type": "AWS::IAM::Policy", | ||
"Properties": { | ||
"PolicyDocument": { | ||
"Statement": [ | ||
{ | ||
"Action": "states:StartExecution", | ||
"Effect": "Allow", | ||
"Resource": { | ||
"Ref": "StateMachine2E01A3A5" | ||
} | ||
}, | ||
{ | ||
"Action": [ | ||
"states:DescribeExecution", | ||
"states:StopExecution" | ||
], | ||
"Effect": "Allow", | ||
"Resource": { | ||
"Fn::Join": [ | ||
"", | ||
[ | ||
{ | ||
"Ref": "StateMachine2E01A3A5" | ||
}, | ||
":*" | ||
] | ||
] | ||
} | ||
} | ||
], | ||
"Version": "2012-10-17" | ||
}, | ||
"PolicyName": "StateMachineDistributedMapPolicy57C9D8C2", | ||
"Roles": [ | ||
{ | ||
"Ref": "StateMachineRoleB840431D" | ||
} | ||
] | ||
} | ||
} | ||
}, | ||
"Outputs": { | ||
"ExportsOutputRefStateMachine2E01A3A5BA46F753": { | ||
"Value": { | ||
"Ref": "StateMachine2E01A3A5" | ||
}, | ||
"Export": { | ||
"Name": "aws-stepfunctions-map-integ:ExportsOutputRefStateMachine2E01A3A5BA46F753" | ||
} | ||
}, | ||
"ExportsOutputRefBucket83908E7781C90AC0": { | ||
"Value": { | ||
"Ref": "Bucket83908E77" | ||
}, | ||
"Export": { | ||
"Name": "aws-stepfunctions-map-integ:ExportsOutputRefBucket83908E7781C90AC0" | ||
} | ||
} | ||
}, | ||
"Parameters": { | ||
"BootstrapVersion": { | ||
"Type": "AWS::SSM::Parameter::Value<String>", | ||
"Default": "/cdk-bootstrap/hnb659fds/version", | ||
"Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store. [cdk:skip]" | ||
} | ||
}, | ||
"Rules": { | ||
"CheckBootstrapVersion": { | ||
"Assertions": [ | ||
{ | ||
"Assert": { | ||
"Fn::Not": [ | ||
{ | ||
"Fn::Contains": [ | ||
[ | ||
"1", | ||
"2", | ||
"3", | ||
"4", | ||
"5" | ||
], | ||
{ | ||
"Ref": "BootstrapVersion" | ||
} | ||
] | ||
} | ||
] | ||
}, | ||
"AssertDescription": "CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI." | ||
} | ||
] | ||
} | ||
} | ||
} |
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Large diffs are not rendered by default.
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,77 @@ | ||
import * as s3 from 'aws-cdk-lib/aws-s3'; | ||
import * as cdk from 'aws-cdk-lib/core'; | ||
import { ExpectedResult, IntegTest } from '@aws-cdk/integ-tests-alpha'; | ||
import * as sfn from 'aws-cdk-lib/aws-stepfunctions'; | ||
|
||
const CSV_KEY = 'my-key.csv'; | ||
|
||
class DistributedMapStack extends cdk.Stack { | ||
readonly bucket: s3.Bucket; | ||
readonly stateMachine: sfn.StateMachine; | ||
|
||
constructor(scope: cdk.App, id: string, props?: cdk.StackProps) { | ||
super(scope, id, props); | ||
|
||
this.bucket = new s3.Bucket(this, 'Bucket', { | ||
autoDeleteObjects: true, | ||
removalPolicy: cdk.RemovalPolicy.DESTROY, | ||
}); | ||
|
||
const distributedMap = new sfn.DistributedMap(this, 'DistributedMap', { | ||
itemReader: new sfn.S3CsvItemReader({ | ||
bucket: this.bucket, | ||
key: CSV_KEY, | ||
csvHeaders: sfn.CsvHeaders.useFirstRow(), | ||
}), | ||
resultWriter: new sfn.ResultWriter({ | ||
bucket: this.bucket, | ||
prefix: 'my-prefix', | ||
}), | ||
}); | ||
distributedMap.itemProcessor(new sfn.Pass(this, 'Pass')); | ||
|
||
this.stateMachine = new sfn.StateMachine(this, 'StateMachine', { | ||
definition: distributedMap, | ||
}); | ||
|
||
} | ||
} | ||
|
||
const app = new cdk.App(); | ||
const stack = new DistributedMapStack(app, 'aws-stepfunctions-map-integ'); | ||
|
||
const testCase = new IntegTest(app, 'DistributedMap', { | ||
testCases: [stack], | ||
}); | ||
|
||
testCase.assertions | ||
.awsApiCall('StepFunctions', 'describeStateMachine', { | ||
stateMachineArn: stack.stateMachine.stateMachineArn, | ||
}) | ||
.expect(ExpectedResult.objectLike({ status: 'ACTIVE' })); | ||
|
||
// Put an object in the bucket | ||
const putObject = testCase.assertions.awsApiCall('S3', 'putObject', { | ||
Bucket: stack.bucket.bucketName, | ||
Key: CSV_KEY, | ||
Body: 'a,b,c\n1,2,3\n4,5,6', | ||
}); | ||
|
||
// Start an execution | ||
const start = testCase.assertions.awsApiCall('StepFunctions', 'startExecution', { | ||
stateMachineArn: stack.stateMachine.stateMachineArn, | ||
}); | ||
putObject.next(start); | ||
|
||
// describe the results of the execution | ||
const describe = testCase.assertions.awsApiCall('StepFunctions', 'describeExecution', { | ||
executionArn: start.getAttString('executionArn'), | ||
}); | ||
start.next(describe); | ||
|
||
// assert the results | ||
describe.expect(ExpectedResult.objectLike({ | ||
status: 'SUCCEEDED', | ||
})); | ||
|
||
app.synth(); |
Large diffs are not rendered by default.
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,231 @@ | ||
import { Construct } from 'constructs'; | ||
import { StateType } from './private/state-type'; | ||
import { renderJsonPath, State } from './state'; | ||
import { Token } from '../../../core'; | ||
import { Chain } from '../chain'; | ||
import { FieldUtils } from '../fields'; | ||
import { StateGraph } from '../state-graph'; | ||
import { CatchProps, IChainable, INextable, ProcessorConfig, ProcessorMode, RetryProps } from '../types'; | ||
|
||
/** | ||
* Properties for defining a Map state | ||
*/ | ||
export interface BaseMapProps { | ||
/** | ||
* Optional name for this state | ||
* | ||
* @default - The construct ID will be used as state name | ||
*/ | ||
readonly stateName?: string; | ||
|
||
/** | ||
* An optional description for this state | ||
* | ||
* @default No comment | ||
*/ | ||
readonly comment?: string; | ||
|
||
/** | ||
* JSONPath expression to select part of the state to be the input to this state. | ||
* | ||
* May also be the special value JsonPath.DISCARD, which will cause the effective | ||
* input to be the empty object {}. | ||
* | ||
* @default $ | ||
*/ | ||
readonly inputPath?: string; | ||
|
||
/** | ||
* JSONPath expression to select part of the state to be the output to this state. | ||
* | ||
* May also be the special value JsonPath.DISCARD, which will cause the effective | ||
* output to be the empty object {}. | ||
* | ||
* @default $ | ||
*/ | ||
readonly outputPath?: string; | ||
|
||
/** | ||
* JSONPath expression to indicate where to inject the state's output | ||
* | ||
* May also be the special value JsonPath.DISCARD, which will cause the state's | ||
* input to become its output. | ||
* | ||
* @default $ | ||
*/ | ||
readonly resultPath?: string; | ||
|
||
/** | ||
* JSONPath expression to select the array to iterate over | ||
* | ||
* @default $ | ||
*/ | ||
readonly itemsPath?: string; | ||
|
||
/** | ||
* The JSON that you want to override your default iteration input (mutually exclusive with `parameters`). | ||
* | ||
* @see | ||
* https://docs.aws.amazon.com/step-functions/latest/dg/input-output-itemselector.html | ||
* | ||
* @default $ | ||
*/ | ||
readonly itemSelector?: { [key: string]: any }; | ||
|
||
/** | ||
* The JSON that will replace the state's raw result and become the effective | ||
* result before ResultPath is applied. | ||
* | ||
* You can use ResultSelector to create a payload with values that are static | ||
* or selected from the state's raw result. | ||
* | ||
* @see | ||
* https://docs.aws.amazon.com/step-functions/latest/dg/input-output-inputpath-params.html#input-output-resultselector | ||
* | ||
* @default - None | ||
*/ | ||
readonly resultSelector?: { [key: string]: any }; | ||
|
||
/** | ||
* MaxConcurrency | ||
* | ||
* An upper bound on the number of iterations you want running at once. | ||
* | ||
* @default - full concurrency | ||
*/ | ||
readonly maxConcurrency?: number; | ||
} | ||
|
||
/** | ||
* Returns true if the value passed is a positive integer | ||
* @param value the value ti validate | ||
abdelnn marked this conversation as resolved.
Show resolved
Hide resolved
|
||
*/ | ||
|
||
export const isPositiveInteger = (value: number) => { | ||
const isFloat = Math.floor(value) !== value; | ||
|
||
const isNotPositiveInteger = value < 0 || value > Number.MAX_SAFE_INTEGER; | ||
|
||
return !isFloat && !isNotPositiveInteger; | ||
}; | ||
|
||
/** | ||
* Define a Map state in the state machine | ||
* | ||
* A `Map` state can be used to run a set of steps for each element of an input array. | ||
* A Map state will execute the same steps for multiple entries of an array in the state input. | ||
* | ||
* While the Parallel state executes multiple branches of steps using the same input, a Map state | ||
* will execute the same steps for multiple entries of an array in the state input. | ||
* | ||
* @see https://docs.aws.amazon.com/step-functions/latest/dg/amazon-states-language-map-state.html | ||
*/ | ||
export abstract class MapBase extends State implements INextable { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Whats the motivation behind moving There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. With this PR, I wanted to do two things:
The way I decided to go about this was to abstract the common properties of both distributed and inline maps, and only support deprecated ASL fields ( Two other alternatives that I briefly considered but ultimately rejected:
|
||
public readonly endStates: INextable[]; | ||
|
||
private readonly maxConcurrency: number | undefined; | ||
abdelnn marked this conversation as resolved.
Show resolved
Hide resolved
|
||
protected readonly itemsPath?: string; | ||
protected readonly itemSelector?: { [key: string]: any }; | ||
|
||
constructor(scope: Construct, id: string, props: BaseMapProps = {}) { | ||
abdelnn marked this conversation as resolved.
Show resolved
Hide resolved
|
||
super(scope, id, props); | ||
this.endStates = [this]; | ||
this.maxConcurrency = props.maxConcurrency; | ||
this.itemsPath = props.itemsPath; | ||
this.itemSelector = props.itemSelector; | ||
} | ||
|
||
/** | ||
* Add retry configuration for this state | ||
* | ||
* This controls if and how the execution will be retried if a particular | ||
* error occurs. | ||
*/ | ||
public addRetry(props: RetryProps = {}): MapBase { | ||
super._addRetry(props); | ||
return this; | ||
} | ||
|
||
/** | ||
* Add a recovery handler for this state | ||
* | ||
* When a particular error occurs, execution will continue at the error | ||
* handler instead of failing the state machine execution. | ||
*/ | ||
public addCatch(handler: IChainable, props: CatchProps = {}): MapBase { | ||
super._addCatch(handler.startState, props); | ||
return this; | ||
} | ||
|
||
/** | ||
* Continue normal execution with the given state | ||
*/ | ||
public next(next: IChainable): Chain { | ||
super.makeNext(next.startState); | ||
return Chain.sequence(this, next); | ||
} | ||
|
||
/** | ||
* Define item processor in Map. | ||
* | ||
* A Map must either have a non-empty iterator or a non-empty item processor (mutually exclusive with `iterator`). | ||
*/ | ||
public itemProcessor(processor: IChainable, config: ProcessorConfig = {}): MapBase { | ||
const name = `Map ${this.stateId} Item Processor`; | ||
const stateGraph = new StateGraph(processor.startState, name); | ||
super.addItemProcessor(stateGraph, config); | ||
return this; | ||
} | ||
|
||
/** | ||
* Return the Amazon States Language object for this state | ||
*/ | ||
public toStateJson(): object { | ||
return { | ||
Type: StateType.MAP, | ||
Comment: this.comment, | ||
ResultPath: renderJsonPath(this.resultPath), | ||
...this.renderNextEnd(), | ||
...this.renderInputOutput(), | ||
...this.renderResultSelector(), | ||
...this.renderRetryCatch(), | ||
...this.renderItemsPath(), | ||
...this.renderItemSelector(), | ||
...this.renderItemProcessor(), | ||
MaxConcurrency: this.maxConcurrency, | ||
}; | ||
} | ||
|
||
/** | ||
* Validate this state | ||
*/ | ||
protected validateState(): string[] { | ||
const errors: string[] = []; | ||
|
||
if (this.processorConfig?.mode === ProcessorMode.DISTRIBUTED && !this.processorConfig?.executionType) { | ||
errors.push('You must specify an execution type for the distributed Map workflow'); | ||
} | ||
|
||
if (this.maxConcurrency && !Token.isUnresolved(this.maxConcurrency) && !isPositiveInteger(this.maxConcurrency)) { | ||
errors.push('maxConcurrency has to be a positive integer'); | ||
} | ||
|
||
return errors; | ||
} | ||
|
||
private renderItemsPath(): any { | ||
return { | ||
ItemsPath: renderJsonPath(this.itemsPath), | ||
}; | ||
} | ||
|
||
/** | ||
* Render ItemSelector in ASL JSON format | ||
*/ | ||
private renderItemSelector(): any { | ||
if (!this.itemSelector) return undefined; | ||
return FieldUtils.renderObject({ | ||
ItemSelector: this.itemSelector, | ||
}); | ||
} | ||
} |
abdelnn marked this conversation as resolved.
Show resolved
Hide resolved
|
Large diffs are not rendered by default.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@abdelnn is it possible that this is adding the role policies on the wrong place?
#29203
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
As soon as I remove this for and put it back where it was before (Only the for).
The bug disappears.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I believe you're right, this should be
this.role.addToPrincipalPolicy(statement)